Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6666: Dynamic copy fields are considering all dynamic fields, causing a significant performance impact on indexing documents 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1649657 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erick Erickson 2015-01-05 20:50:39 +00:00
parent a13b71aa41
commit 14c8880729
4 changed files with 54 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
solr/CHANGES.txt
View File

@ -419,6 +419,10 @@ Optimizations
hl.usePhraseHighlighter, and can be more efficient handling data from term vectors.
(David Smiley)
* SOLR-6666: Dynamic copy fields are considering all dynamic fields, causing
a significant performance impact on indexing documents. (Liram Vardi via Erick
Erickson, Steve Rowe)
Other Changes
----------------------

40
solr/core/src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -901,37 +901,49 @@ public class IndexSchema {
String msg = "copyField dest :'" + dest + "' is not an explicit field and doesn't match a dynamicField.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
if (sourceIsDynamicFieldReference || sourceIsGlob) {
if (null != destDynamicField) { // source: glob or no-asterisk dynamic field ref; dest: dynamic field ref
if (sourceIsGlob) {
if (null != destDynamicField) { // source: glob ; dest: dynamic field ref
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
} else { // source: glob or no-asterisk dynamic field ref; dest: explicit field
} else { // source: glob ; dest: explicit field
destDynamicField = new DynamicField(destSchemaField);
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, null));
incrementCopyFieldTargetCount(destSchemaField);
}
} else {
if (null != destDynamicField) { // source: explicit field; dest: dynamic field reference
} else if (sourceIsDynamicFieldReference) {
if (null != destDynamicField) { // source: no-asterisk dynamic field ref ; dest: dynamic field ref
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
} else { // source: no-asterisk dynamic field ref ; dest: explicit field
sourceSchemaField = getField(source);
registerExplicitSrcAndDestFields(source, maxChars, destSchemaField, sourceSchemaField);
}
} else {
if (null != destDynamicField) { // source: explicit field ; dest: dynamic field reference
if (destDynamicField.pattern instanceof DynamicReplacement.DynamicPattern.NameEquals) {
// Dynamic dest with no asterisk is acceptable
registerDynamicCopyField(new DynamicCopy(source, destDynamicField, maxChars, sourceDynamicBase, destDynamicBase));
incrementCopyFieldTargetCount(destSchemaField);
} else {
} else { // source: explicit field ; dest: dynamic field with an asterisk
String msg = "copyField only supports a dynamic destination with an asterisk "
+ "if the source also has an asterisk";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
} else { // source & dest: explicit fields
List<CopyField> copyFieldList = copyFieldsMap.get(source);
if (copyFieldList == null) {
copyFieldList = new ArrayList<>();
copyFieldsMap.put(source, copyFieldList);
}
copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars));
incrementCopyFieldTargetCount(destSchemaField);
} else { // source & dest: explicit fields
registerExplicitSrcAndDestFields(source, maxChars, destSchemaField, sourceSchemaField);
}
}
}
private void registerExplicitSrcAndDestFields(String source, int maxChars, SchemaField destSchemaField, SchemaField sourceSchemaField) {
List<CopyField> copyFieldList = copyFieldsMap.get(source);
if (copyFieldList == null) {
copyFieldList = new ArrayList<>();
copyFieldsMap.put(source, copyFieldList);
}
copyFieldList.add(new CopyField(sourceSchemaField, destSchemaField, maxChars));
incrementCopyFieldTargetCount(destSchemaField);
}
private void incrementCopyFieldTargetCount(SchemaField dest) {
copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1);

29
solr/core/src/test/org/apache/solr/rest/schema/TestCopyFieldCollectionResource.java
View File

@ -23,7 +23,10 @@ public class TestCopyFieldCollectionResource extends SolrRestletTestBase {
@Test
public void testGetAllCopyFields() throws Exception {
assertQ("/schema/copyfields?indent=on&wt=xml",
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']"
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+" and str[@name='dest'][.='title']]",
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='title']"
+" and str[@name='dest'][.='title_stemmed']"
+" and int[@name='maxChars'][.='200']]",
@ -65,10 +68,6 @@ public class TestCopyFieldCollectionResource extends SolrRestletTestBase {
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+" and str[@name='sourceDynamicBase'][.='*_i']"
+" and str[@name='dest'][.='title']]",
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
+" and str[@name='sourceDynamicBase'][.='*_i']"
+" and str[@name='dest'][.='*_s']]",
"/response/arr[@name='copyFields']/lst[ str[@name='source'][.='src_sub_no_ast_i']"
@ -105,19 +104,19 @@ public class TestCopyFieldCollectionResource extends SolrRestletTestBase {
@Test
public void testJsonGetAllCopyFields() throws Exception {
assertJQ("/schema/copyfields?indent=on&wt=json",
"/copyFields/[6]=={'source':'title','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[1]=={'source':'src_sub_no_ast_i','dest':'title'}",
"/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[7]=={'source':'*_i','dest':'title'}",
"/copyFields/[8]=={'source':'*_i','dest':'*_s'}",
"/copyFields/[9]=={'source':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[10]=={'source':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[8]=={'source':'*_i','dest':'title'}",
"/copyFields/[9]=={'source':'*_i','dest':'*_s'}",
"/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[11]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'title'}",
"/copyFields/[12]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_s'}",
"/copyFields/[13]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[14]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[12]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'title'}",
"/copyFields/[13]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_s'}",
"/copyFields/[14]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[15]=={'source':'*_src_sub_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}",
"/copyFields/[15]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'title'}",
"/copyFields/[16]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_s'}",
"/copyFields/[17]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'*_dest_sub_s','destDynamicBase':'*_s'}",
"/copyFields/[18]=={'source':'src_sub_no_ast_i','sourceDynamicBase':'*_i','dest':'dest_sub_no_ast_s','destDynamicBase':'*_s'}");

20
solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java
View File

@ -131,19 +131,19 @@ public class TestSchemaResource extends SolrRestletTestBase {
"/schema/dynamicFields/[1]/name=='ignored_*'",
"/schema/dynamicFields/[2]/name=='*_mfacet'",
"/schema/copyFields/[6]=={'source':'title','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[1]=={'source':'src_sub_no_ast_i','dest':'title'}",
"/schema/copyFields/[7]=={'source':'*_i','dest':'title'}",
"/schema/copyFields/[8]=={'source':'*_i','dest':'*_s'}",
"/schema/copyFields/[9]=={'source':'*_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[10]=={'source':'*_i','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[7]=={'source':'title','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[8]=={'source':'*_i','dest':'title'}",
"/schema/copyFields/[9]=={'source':'*_i','dest':'*_s'}",
"/schema/copyFields/[10]=={'source':'*_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[11]=={'source':'*_i','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[11]=={'source':'*_src_sub_i','dest':'title'}",
"/schema/copyFields/[12]=={'source':'*_src_sub_i','dest':'*_s'}",
"/schema/copyFields/[13]=={'source':'*_src_sub_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[14]=={'source':'*_src_sub_i','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[12]=={'source':'*_src_sub_i','dest':'title'}",
"/schema/copyFields/[13]=={'source':'*_src_sub_i','dest':'*_s'}",
"/schema/copyFields/[14]=={'source':'*_src_sub_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[15]=={'source':'*_src_sub_i','dest':'dest_sub_no_ast_s'}",
"/schema/copyFields/[15]=={'source':'src_sub_no_ast_i','dest':'title'}",
"/schema/copyFields/[16]=={'source':'src_sub_no_ast_i','dest':'*_s'}",
"/schema/copyFields/[17]=={'source':'src_sub_no_ast_i','dest':'*_dest_sub_s'}",
"/schema/copyFields/[18]=={'source':'src_sub_no_ast_i','dest':'dest_sub_no_ast_s'}");