Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7010: add MergePolicyWrapper

This commit is contained in:
Shai Erera 2016-02-03 14:49:03 +02:00
parent 62dfc815b0
commit 30455f728b
12 changed files with 147 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -137,6 +137,9 @@ New Features
* SOLR-4619: Added removeAllAttributes() to AttributeSource, which removes
all previously added attributes.
* LUCENE-7010: Added MergePolicyWrapper to allow easy wrapping of other policies.
(Shai Erera)
API Changes

6
lucene/core/src/java/org/apache/lucene/index/MergePolicy.java
View File

@ -468,7 +468,7 @@ public abstract class MergePolicy {
/** Returns current {@code noCFSRatio}.
*
* @see #setNoCFSRatio */
public final double getNoCFSRatio() {
public double getNoCFSRatio() {
return noCFSRatio;
}
@ -477,7 +477,7 @@ public abstract class MergePolicy {
* non-compound file even if compound file is enabled.
* Set to 1.0 to always use CFS regardless of merge
* size. */
public final void setNoCFSRatio(double noCFSRatio) {
public void setNoCFSRatio(double noCFSRatio) {
if (noCFSRatio < 0.0 || noCFSRatio > 1.0) {
throw new IllegalArgumentException("noCFSRatio must be 0.0 to 1.0 inclusive; got " + noCFSRatio);
}
@ -494,7 +494,7 @@ public abstract class MergePolicy {
* non-compound file even if compound file is enabled.
* Set this to Double.POSITIVE_INFINITY (default) and noCFSRatio to 1.0
* to always use CFS regardless of merge size. */
public final void setMaxCFSSegmentSizeMB(double v) {
public void setMaxCFSSegmentSizeMB(double v) {
if (v < 0.0) {
throw new IllegalArgumentException("maxCFSSegmentSizeMB must be >=0 (got " + v + ")");
}

86
lucene/core/src/java/org/apache/lucene/index/MergePolicyWrapper.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
import java.io.IOException;
import java.util.Map;
/**
* A wrapper for {@link MergePolicy} instances.
*
* @lucene.experimental
*/
public class MergePolicyWrapper extends MergePolicy {
protected final MergePolicy in;
/**
* Creates a new merge policy instance.
*/
public MergePolicyWrapper(MergePolicy in) {
this.in = in;
}
@Override
public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, IndexWriter writer)
throws IOException {
return in.findMerges(mergeTrigger, segmentInfos, writer);
}
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount,
Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer) throws IOException {
return in.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer);
}
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, IndexWriter writer) throws IOException {
return in.findForcedDeletesMerges(segmentInfos, writer);
}
@Override
public boolean useCompoundFile(SegmentInfos infos, SegmentCommitInfo mergedInfo, IndexWriter writer)
throws IOException {
return in.useCompoundFile(infos, mergedInfo, writer);
}
@Override
protected long size(SegmentCommitInfo info, IndexWriter writer) throws IOException {
return in.size(info, writer);
}
@Override
public double getNoCFSRatio() {
return in.getNoCFSRatio();
}
@Override
public final void setNoCFSRatio(double noCFSRatio) {
in.setNoCFSRatio(noCFSRatio);
}
@Override
public final void setMaxCFSSegmentSizeMB(double v) {
in.setMaxCFSSegmentSizeMB(v);
}
@Override
public String toString() {
return getClass().getSimpleName() + "(" + in + ")";
}
}

18
lucene/core/src/java/org/apache/lucene/index/NoMergePolicy.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Map;
/**
* A {@link MergePolicy} which never returns merges to execute. Use it if you
* want to prevent segment merges.
@ -53,7 +52,22 @@ public final class NoMergePolicy extends MergePolicy {
protected long size(SegmentCommitInfo info, IndexWriter writer) throws IOException {
return Long.MAX_VALUE;
}
@Override
public double getNoCFSRatio() {
return super.getNoCFSRatio();
}
@Override
public void setMaxCFSSegmentSizeMB(double v) {
super.setMaxCFSSegmentSizeMB(v);
}
@Override
public void setNoCFSRatio(double noCFSRatio) {
super.setNoCFSRatio(noCFSRatio);
}
@Override
public String toString() {
return "NoMergePolicy";

30
lucene/core/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java
View File

@ -48,15 +48,12 @@ import java.util.HashMap;
* @lucene.experimental
* @see IndexUpgrader
*/
public class UpgradeIndexMergePolicy extends MergePolicy {
/** Wrapped {@link MergePolicy}. */
protected final MergePolicy base;
public class UpgradeIndexMergePolicy extends MergePolicyWrapper {
/** Wrap the given {@link MergePolicy} and intercept forceMerge requests to
* only upgrade segments written with previous Lucene versions. */
public UpgradeIndexMergePolicy(MergePolicy base) {
this.base = base;
public UpgradeIndexMergePolicy(MergePolicy in) {
super(in);
}
/** Returns if the given segment should be upgraded. The default implementation
@ -70,7 +67,7 @@ public class UpgradeIndexMergePolicy extends MergePolicy {
@Override
public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, IndexWriter writer) throws IOException {
return base.findMerges(null, segmentInfos, writer);
return in.findMerges(null, segmentInfos, writer);
}
@Override
@ -91,7 +88,7 @@ public class UpgradeIndexMergePolicy extends MergePolicy {
if (oldSegments.isEmpty())
return null;
MergeSpecification spec = base.findForcedMerges(segmentInfos, maxSegmentCount, oldSegments, writer);
MergeSpecification spec = in.findForcedMerges(segmentInfos, maxSegmentCount, oldSegments, writer);
if (spec != null) {
// remove all segments that are in merge specification from oldSegments,
@ -104,7 +101,7 @@ public class UpgradeIndexMergePolicy extends MergePolicy {
if (!oldSegments.isEmpty()) {
if (verbose(writer)) {
message("findForcedMerges: " + base.getClass().getSimpleName() +
message("findForcedMerges: " + in.getClass().getSimpleName() +
" does not want to merge all old segments, merge remaining ones into new segment: " + oldSegments, writer);
}
final List<SegmentCommitInfo> newInfos = new ArrayList<>();
@ -123,21 +120,6 @@ public class UpgradeIndexMergePolicy extends MergePolicy {
return spec;
}
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, IndexWriter writer) throws IOException {
return base.findForcedDeletesMerges(segmentInfos, writer);
}
@Override
public boolean useCompoundFile(SegmentInfos segments, SegmentCommitInfo newSegment, IndexWriter writer) throws IOException {
return base.useCompoundFile(segments, newSegment, writer);
}
@Override
public String toString() {
return "[" + getClass().getSimpleName() + "->" + base + "]";
}
private boolean verbose(IndexWriter writer) {
return writer != null && writer.infoStream.isEnabled("UPGMP");
}

6
lucene/core/src/test/org/apache/lucene/index/TestDemoParallelLeafReader.java
View File

@ -505,7 +505,7 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
}
/** Just replaces the sub-readers with parallel readers, so reindexed fields are merged into new segments. */
private class ReindexingMergePolicy extends MergePolicy {
private class ReindexingMergePolicy extends MergePolicyWrapper {
class ReindexingOneMerge extends OneMerge {
@ -596,11 +596,9 @@ public class TestDemoParallelLeafReader extends LuceneTestCase {
return wrapped;
}
final MergePolicy in;
/** Create a new {@code MergePolicy} that sorts documents with the given {@code sort}. */
public ReindexingMergePolicy(MergePolicy in) {
this.in = in;
super(in);
}
@Override

31
solr/core/src/test/org/apache/solr/util/TestRandomForceMergePolicy.java → lucene/core/src/test/org/apache/lucene/index/TestMergePolicyWrapper.java Executable file → Normal file
View File

@ -1,10 +1,3 @@
package org.apache.solr.util;
import java.io.IOException;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.SegmentInfos;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -21,16 +14,24 @@ import org.apache.lucene.index.SegmentInfos;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.index;
public class TestRandomForceMergePolicy extends TestRandomMergePolicy {
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
/**
* Ensure it finds no merges
*/
public void testFindMerges() throws IOException {
MergePolicy mp = new RandomForceMergePolicy();
assertNull(mp.findMerges(null, (SegmentInfos)null, null));
import org.apache.lucene.util.LuceneTestCase;
public class TestMergePolicyWrapper extends LuceneTestCase {
public void testMethodsOverridden() throws Exception {
for (Method m : MergePolicy.class.getDeclaredMethods()) {
if (Modifier.isFinal(m.getModifiers())) continue;
try {
MergePolicyWrapper.class.getDeclaredMethod(m.getName(), m.getParameterTypes());
} catch (NoSuchMethodException e) {
fail("MergePolicyWrapper needs to override '" + m + "'");
}
}
}
}

17
lucene/misc/src/java/org/apache/lucene/index/SortingMergePolicy.java
View File

@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer; // javadocs
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MergePolicy;
@ -55,7 +54,7 @@ import org.apache.lucene.util.packed.PackedLongValues;
* the order of documents in a segment depend on the number of times the segment
* has been merged.
* @lucene.experimental */
public final class SortingMergePolicy extends MergePolicy {
public final class SortingMergePolicy extends MergePolicyWrapper {
/**
* Put in the {@link SegmentInfo#getDiagnostics() diagnostics} to denote that
@ -221,13 +220,12 @@ public final class SortingMergePolicy extends MergePolicy {
return sortingSpec;
}
final MergePolicy in;
final Sorter sorter;
final Sort sort;
/** Create a new {@code MergePolicy} that sorts documents with the given {@code sort}. */
public SortingMergePolicy(MergePolicy in, Sort sort) {
this.in = in;
super(in);
this.sorter = new Sorter(sort);
this.sort = sort;
}
@ -256,17 +254,6 @@ public final class SortingMergePolicy extends MergePolicy {
return sortedMergeSpecification(in.findForcedDeletesMerges(segmentInfos, writer), writer.infoStream);
}
@Override
public boolean useCompoundFile(SegmentInfos segments,
SegmentCommitInfo newSegment, IndexWriter writer) throws IOException {
return in.useCompoundFile(segments, newSegment, writer);
}
@Override
protected long size(SegmentCommitInfo info, IndexWriter writer) throws IOException {
return in.size(info, writer);
}
@Override
public String toString() {
return "SortingMergePolicy(" + in + ", sorter=" + sorter + ")";

17
lucene/misc/src/test/org/apache/lucene/index/TestSortingMergePolicy.java
View File

@ -78,12 +78,6 @@ public class TestSortingMergePolicy extends BaseMergePolicyTestCase {
return doc;
}
@Override
public void testForceMergeNotNeeded() throws IOException {
// This is a no-op until we figure out why the (super class) test fails.
// https://issues.apache.org/jira/browse/LUCENE-7008
}
public MergePolicy mergePolicy() {
return newSortingMergePolicy(sort);
}
@ -204,15 +198,4 @@ public class TestSortingMergePolicy extends BaseMergePolicyTestCase {
}
}
public void testMethodsOverridden() throws Exception {
for (Method m : MergePolicy.class.getDeclaredMethods()) {
if (Modifier.isFinal(m.getModifiers())) continue;
try {
SortingMergePolicy.class.getDeclaredMethod(m.getName(), m.getParameterTypes());
} catch (NoSuchMethodException e) {
fail("SortingMergePolicy needs to override '"+m+"'");
}
}
}
}

45
lucene/test-framework/src/java/org/apache/lucene/index/ForceMergePolicy.java
View File

@ -28,52 +28,23 @@ import org.apache.lucene.index.SegmentInfos;
/**
* A {@link MergePolicy} that only returns forced merges.
* <p><b>NOTE</b>: Use this policy if you wish to disallow background
* merges but wish to run optimize/forceMerge segment merges.
* <p>
* <b>NOTE</b>: Use this policy if you wish to disallow background merges but wish to run optimize/forceMerge segment
* merges.
*
* @lucene.experimental
* @lucene.experimental
*/
public final class ForceMergePolicy extends MergePolicy {
final MergePolicy in;
public final class ForceMergePolicy extends MergePolicyWrapper {
/** Create a new {@code ForceMergePolicy} around the given {@code MergePolicy} */
public ForceMergePolicy(MergePolicy in) {
this.in = in;
super(in);
}
@Override
public MergeSpecification findMerges(MergeTrigger mergeTrigger,
SegmentInfos segmentInfos, IndexWriter writer) throws IOException {
public MergeSpecification findMerges(MergeTrigger mergeTrigger, SegmentInfos segmentInfos, IndexWriter writer)
throws IOException {
return null;
}
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer)
throws IOException {
return in.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer);
}
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, IndexWriter writer)
throws IOException {
return in.findForcedDeletesMerges(segmentInfos, writer);
}
@Override
public boolean useCompoundFile(SegmentInfos segments,
SegmentCommitInfo newSegment, IndexWriter writer) throws IOException {
return in.useCompoundFile(segments, newSegment, writer);
}
@Override
protected long size(SegmentCommitInfo info, IndexWriter writer) throws IOException {
return in.size(info, writer);
}
@Override
public String toString() {
return "ForceMergePolicy(" + in + ")";
}
}

74
solr/core/src/test/org/apache/solr/util/TestRandomMergePolicy.java
View File

@ -1,74 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.InvocationTargetException;
/**
* A "test the test" sanity check using reflection to ensure that
* {@linke RandomMergePolicy} is working as expected
*/
@SuppressSysoutChecks(bugUrl = "Logs to JUL")
public class TestRandomMergePolicy extends LuceneTestCase {
/**
* Ensure every MP method is overridden by RMP
* (future proof ourselves against new methods being added to MP)
*/
public void testMethodOverride() {
Class rmp = RandomMergePolicy.class;
for (Method meth : rmp.getMethods()) {
if (// ignore things like hashCode, equals, etc...
meth.getDeclaringClass().equals(Object.class)
// can't do anything about it regardless of what class declares it
|| Modifier.isFinal(meth.getModifiers())) {
continue;
}
assertEquals("method not overridden by RandomMergePolicy: " +
meth.toGenericString(),
rmp, meth.getDeclaringClass());
}
}
/**
* Ensure any "getter" methods return the same value as
* the wrapped MP
* (future proof ourselves against new final getter/setter pairs being
* added to MP w/o dealing with them in the RMP Constructor)
*/
public void testGetters() throws IllegalAccessException, InvocationTargetException {
final int iters = atLeast(20);
for (int i = 0; i < iters; i++) {
RandomMergePolicy rmp = new RandomMergePolicy();
Class mp = MergePolicy.class;
for (Method meth : mp.getDeclaredMethods()) {
if (meth.getName().startsWith("get") &&
(0 == meth.getParameterTypes().length)) {
assertEquals("MergePolicy getter gave diff results for RandomMergePolicy and the policy it wrapped: " + meth.toGenericString(),
meth.invoke(rmp), meth.invoke(rmp.inner));
}
}
}
}
}

51
solr/test-framework/src/java/org/apache/solr/util/RandomMergePolicy.java
View File

@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergePolicy.MergeSpecification;
import org.apache.lucene.index.MergePolicyWrapper;
import org.apache.lucene.index.MergeTrigger;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfos;
@ -36,60 +38,17 @@ import org.slf4j.LoggerFactory;
* Solr tests utilizing the Lucene randomized test framework can refer
* to this class in solrconfig.xml to get a fully randomized merge policy.
*/
public class RandomMergePolicy extends MergePolicy {
public class RandomMergePolicy extends MergePolicyWrapper {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
* Not private so tests can inspect it,
*/
final MergePolicy inner;
public RandomMergePolicy() {
this(LuceneTestCase.newMergePolicy());
}
protected RandomMergePolicy(MergePolicy inner) {
super(inner.getNoCFSRatio(),
(long) (inner.getMaxCFSSegmentSizeMB() * 1024 * 1024));
this.inner = inner;
log.info("RandomMergePolicy wrapping {}: {}",
inner.getClass(), inner);
}
@Override
public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, IndexWriter writer)
throws IOException {
return inner.findForcedDeletesMerges(segmentInfos, writer);
}
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
int maxSegmentCount,
Map<SegmentCommitInfo,Boolean> segmentsToMerge,
IndexWriter writer)
throws IOException {
return inner.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer);
}
@Override
public MergeSpecification findMerges(MergeTrigger mergeTrigger,
SegmentInfos segmentInfos,
IndexWriter writer)
throws IOException {
return inner.findMerges(mergeTrigger, segmentInfos, writer);
}
@Override
public boolean useCompoundFile(SegmentInfos infos,
SegmentCommitInfo mergedInfo,
IndexWriter writer)
throws IOException {
return inner.useCompoundFile(infos, mergedInfo, writer);
super(inner);
log.info("RandomMergePolicy wrapping {}: {}", inner.getClass(), inner);
}
}