diff --git a/dev-tools/idea/solr/core/src/java/solr-core.iml b/dev-tools/idea/solr/core/src/java/solr-core.iml index 63426c37f94..563f572b661 100644 --- a/dev-tools/idea/solr/core/src/java/solr-core.iml +++ b/dev-tools/idea/solr/core/src/java/solr-core.iml @@ -25,5 +25,6 @@ + diff --git a/dev-tools/idea/solr/core/src/test/solr-core-tests.iml b/dev-tools/idea/solr/core/src/test/solr-core-tests.iml index ed9f86f7c9f..3e3c09540b6 100644 --- a/dev-tools/idea/solr/core/src/test/solr-core-tests.iml +++ b/dev-tools/idea/solr/core/src/test/solr-core-tests.iml @@ -27,5 +27,6 @@ + diff --git a/dev-tools/maven/solr/core/src/java/pom.xml.template b/dev-tools/maven/solr/core/src/java/pom.xml.template index 9e9885994c3..359158ca76c 100644 --- a/dev-tools/maven/solr/core/src/java/pom.xml.template +++ b/dev-tools/maven/solr/core/src/java/pom.xml.template @@ -93,6 +93,11 @@ lucene-misc ${project.version} + + org.apache.lucene + lucene-join + ${project.version} + org.apache.lucene lucene-queryparser diff --git a/dev-tools/maven/solr/core/src/test/pom.xml.template b/dev-tools/maven/solr/core/src/test/pom.xml.template index 1b2fa8e1c3f..452537358e5 100644 --- a/dev-tools/maven/solr/core/src/test/pom.xml.template +++ b/dev-tools/maven/solr/core/src/test/pom.xml.template @@ -198,6 +198,12 @@ jetty-util test + + dom4j + dom4j + 1.6.1 + test + diff --git a/lucene/module-build.xml b/lucene/module-build.xml index 572de31e1b0..6c6d97afcdf 100644 --- a/lucene/module-build.xml +++ b/lucene/module-build.xml @@ -132,6 +132,17 @@ + + + + + + + + + + + diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2a02593a81f..b9f95e895e3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -181,6 +181,7 @@ Upgrading from Solr 4.3.0 * LUCENE-5063: ByteField and ShortField have been deprecated and will be removed in 5.0. If you are still using these field types, you should migrate your fields to TrieIntField. + Detailed Change List ---------------------- @@ -261,6 +262,16 @@ New Features * SOLR-4943: Add a new system wide info admin handler that exposes the system info that could previously only be retrieved using a SolrCore. (Mark Miller) +* SOLR-3076: Block joins. Documents and their sub-documents must be indexed + as a block. + {!parent which=} takes in a query that matches child + documents and results in matches on their parents. + {!child of=} takes in a query that matches some parent + documents and results in matches on their children. + (Mikhail Khludnev, Vadim Kirilchuk, Alan Woodward, Tom Burton-West, Mike McCandless, + hossman, yonik) + + Bug Fixes ---------------------- diff --git a/solr/common-build.xml b/solr/common-build.xml index 57cfaf35571..3eaf3b3a705 100644 --- a/solr/common-build.xml +++ b/solr/common-build.xml @@ -82,6 +82,7 @@ + @@ -141,7 +142,7 @@ + jar-misc, jar-spatial, jar-grouping, jar-queries, jar-queryparser, jar-join"> diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 488266bbc90..1138eff2663 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -41,6 +41,7 @@ + diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index 6211322cef7..400a75337aa 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -16,50 +16,51 @@ package org.apache.solr.handler.loader; * limitations under the License. */ +import com.google.common.collect.Lists; +import org.apache.commons.io.IOUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.update.processor.UpdateRequestProcessor; -import org.apache.solr.update.AddUpdateCommand; -import org.apache.solr.update.CommitUpdateCommand; -import org.apache.solr.update.RollbackUpdateCommand; -import org.apache.solr.update.DeleteUpdateCommand; -import org.apache.solr.util.xslt.TransformerProvider; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.XMLErrorLogger; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; import org.apache.solr.core.SolrConfig; import org.apache.solr.handler.RequestHandlerUtils; import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.CommitUpdateCommand; +import org.apache.solr.update.DeleteUpdateCommand; +import org.apache.solr.update.RollbackUpdateCommand; +import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.util.EmptyEntityResolver; -import org.apache.commons.io.IOUtils; +import org.apache.solr.util.xslt.TransformerProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.xml.sax.InputSource; import org.xml.sax.XMLReader; -import javax.xml.stream.XMLStreamReader; -import javax.xml.stream.XMLStreamException; +import javax.xml.parsers.SAXParserFactory; import javax.xml.stream.FactoryConfigurationError; -import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.Transformer; import javax.xml.transform.TransformerException; import javax.xml.transform.dom.DOMResult; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.sax.SAXSource; -import javax.xml.parsers.SAXParserFactory; - import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -381,6 +382,7 @@ public class XMLLoader extends ContentStreamLoader { float boost = 1.0f; boolean isNull = false; String update = null; + Collection subDocs = null; Map> updateMap = null; boolean complete = false; while (!complete) { @@ -395,9 +397,14 @@ public class XMLLoader extends ContentStreamLoader { case XMLStreamConstants.END_ELEMENT: if ("doc".equals(parser.getLocalName())) { + if (subDocs != null && !subDocs.isEmpty()) { + doc.addChildDocuments(subDocs); + subDocs = null; + } complete = true; break; } else if ("field".equals(parser.getLocalName())) { + // should I warn in some text has been found too Object v = isNull ? null : text.toString(); if (update != null) { if (updateMap == null) updateMap = new HashMap>(); @@ -425,34 +432,43 @@ public class XMLLoader extends ContentStreamLoader { } doc.addField(name, v, boost); boost = 1.0f; + // field is over + name = null; } break; case XMLStreamConstants.START_ELEMENT: text.setLength(0); String localName = parser.getLocalName(); - if (!"field".equals(localName)) { - log.warn("unexpected XML tag doc/" + localName); - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "unexpected XML tag doc/" + localName); + if ("doc".equals(localName)) { + if (subDocs == null) + subDocs = Lists.newArrayList(); + subDocs.add(readDoc(parser)); } - boost = 1.0f; - update = null; - isNull = false; - String attrVal = ""; - for (int i = 0; i < parser.getAttributeCount(); i++) { - attrName = parser.getAttributeLocalName(i); - attrVal = parser.getAttributeValue(i); - if ("name".equals(attrName)) { - name = attrVal; - } else if ("boost".equals(attrName)) { - boost = Float.parseFloat(attrVal); - } else if ("null".equals(attrName)) { - isNull = StrUtils.parseBoolean(attrVal); - } else if ("update".equals(attrName)) { - update = attrVal; - } else { - log.warn("Unknown attribute doc/field/@" + attrName); + else { + if (!"field".equals(localName)) { + log.warn("unexpected XML tag doc/" + localName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "unexpected XML tag doc/" + localName); + } + boost = 1.0f; + update = null; + isNull = false; + String attrVal = ""; + for (int i = 0; i < parser.getAttributeCount(); i++) { + attrName = parser.getAttributeLocalName(i); + attrVal = parser.getAttributeValue(i); + if ("name".equals(attrName)) { + name = attrVal; + } else if ("boost".equals(attrName)) { + boost = Float.parseFloat(attrVal); + } else if ("null".equals(attrName)) { + isNull = StrUtils.parseBoolean(attrVal); + } else if ("update".equals(attrName)) { + update = attrVal; + } else { + log.warn("Unknown attribute doc/field/@" + attrName); + } } } break; diff --git a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java index e1acb1cbb14..e36759c74b9 100644 --- a/solr/core/src/java/org/apache/solr/search/QParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/QParserPlugin.java @@ -20,6 +20,8 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrInfoMBean; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.join.BlockJoinChildQParserPlugin; +import org.apache.solr.search.join.BlockJoinParentQParserPlugin; import org.apache.solr.util.plugin.NamedListInitializedPlugin; import java.net.URL; @@ -47,7 +49,9 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI JoinQParserPlugin.NAME, JoinQParserPlugin.class, SurroundQParserPlugin.NAME, SurroundQParserPlugin.class, SwitchQParserPlugin.NAME, SwitchQParserPlugin.class, - MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class + MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class, + BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class, + BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class }; /** return a {@link QParser} */ diff --git a/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java b/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java new file mode 100644 index 00000000000..b1d6224627b --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BitSetSlice.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.lucene.util.OpenBitSet; + +class BitSetSlice { + private final OpenBitSet obs; + private final int off; + private final int len; + + BitSetSlice(OpenBitSet obs, int off, int len) { + this.obs = obs; + this.off = off; + this.len = len; + } + + public boolean get(int pos) { + return obs.get(pos + off); + } + + public int prevSetBit(int pos) { + int result = obs.prevSetBit(pos + off) - off; + return (result < 0) ? -1 : result; + } + + public int nextSetBit(int pos) { + int result = obs.nextSetBit(pos + off) - off; + return (result < 0 || result >= len) ? -1 : result; + } +} diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java new file mode 100644 index 00000000000..d74650c9ecd --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.ToChildBlockJoinQuery; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; + +public class BlockJoinChildQParser extends BlockJoinParentQParser { + + public BlockJoinChildQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + protected Query createQuery(Query parentListQuery, Query query) { + return new ToChildBlockJoinQuery(query, getFilter(parentListQuery), false); + } + + @Override + protected String getParentFilterLocalParamName() { + return "of"; + } +} + + diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java new file mode 100644 index 00000000000..ed7600fa92a --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParserPlugin.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; + +/** + * Usage: {!child of="PARENT:true"}PARENT_PRICE:10 + * + **/ +public class BlockJoinChildQParserPlugin extends BlockJoinParentQParserPlugin { + public static String NAME = "child"; + + @Override + protected QParser createBJQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new BlockJoinChildQParser(qstr, localParams, params, req); + } +} + diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java new file mode 100644 index 00000000000..af9abaad4fc --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.lucene.search.CachingWrapperFilter; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.SolrCache; +import org.apache.solr.search.SolrConstantScoreQuery; +import org.apache.solr.search.SyntaxError; + +class BlockJoinParentQParser extends QParser { + /** implementation detail subject to change */ + public String CACHE_NAME="perSegFilter"; + + protected String getParentFilterLocalParamName() { + return "which"; + } + + BlockJoinParentQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(qstr, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + String filter = localParams.get(getParentFilterLocalParamName()); + QParser parentParser = subQuery(filter, null); + Query parentQ = parentParser.getQuery(); + + String queryText = localParams.get(QueryParsing.V); + // there is no child query, return parent filter from cache + if (queryText == null || queryText.length()==0) { + SolrConstantScoreQuery wrapped = new SolrConstantScoreQuery(getFilter(parentQ)); + wrapped.setCache(false); + return wrapped; + } + QParser childrenParser = subQuery(queryText, null); + Query childrenQuery = childrenParser.getQuery(); + return createQuery(parentQ, childrenQuery); + } + + protected Query createQuery(Query parentList, Query query) { + return new ToParentBlockJoinQuery(query, getFilter(parentList), ScoreMode.None); + } + + protected Filter getFilter(Query parentList) { + SolrCache parentCache = req.getSearcher().getCache(CACHE_NAME); + // lazily retrieve from solr cache + Filter filter = null; + if (parentCache != null) { + filter = (Filter) parentCache.get(parentList); + } + Filter result; + if (filter == null) { + result = createParentFilter(parentList); + if (parentCache != null) { + parentCache.put(parentList, result); + } + } else { + result = filter; + } + return result; + } + + protected Filter createParentFilter(Query parentQ) { + return new CachingWrapperFilter(new QueryWrapperFilter(parentQ)) { + }; + } +} + + + + + + diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java new file mode 100644 index 00000000000..4c16a9615d3 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParserPlugin.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; + +/** + * Usage: {!parent which="PARENT:true"}CHILD_PRICE:10 + * + **/ +public class BlockJoinParentQParserPlugin extends QParserPlugin { + public static String NAME = "parent"; + + @Override + public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + QParser parser = createBJQParser(qstr, localParams, params, req); + return parser; + } + + protected QParser createBJQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new BlockJoinParentQParser(qstr, localParams, params, req); + } + + @Override + public void init(NamedList args) { + } +} + diff --git a/solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java b/solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java new file mode 100644 index 00000000000..7e057fe5756 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Set; + +public class IgnoreAcceptDocsQuery extends Query { + private final Query q; + + public IgnoreAcceptDocsQuery(Query q) { + this.q = q; + } + + @Override + public void setBoost(float b) { + q.setBoost(b); + } + + @Override + public float getBoost() { + return q.getBoost(); + } + + @Override + public String toString() { + return q.toString(); + } + + @Override + public Weight createWeight(IndexSearcher searcher) throws IOException { + Weight inner = q.createWeight(searcher); + return new IADWeight(inner); + } + + private class IADWeight extends Weight { + Weight w; + + IADWeight(Weight delegate) { + this.w = delegate; + } + + @Override + public Explanation explain(AtomicReaderContext context, int doc) throws IOException { + return w.explain(context, doc); + } + + @Override + public Query getQuery() { + return q; + } + + @Override + public float getValueForNormalization() throws IOException { + return w.getValueForNormalization(); + } + + @Override + public void normalize(float norm, float topLevelBoost) { + w.normalize(norm, topLevelBoost); + } + + @Override + public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException { + return w.scorer(context, scoreDocsInOrder, topScorer, null); + } + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query n = q.rewrite(reader); + if (q == n) return this; + return new IgnoreAcceptDocsQuery(n); + } + + @Override + public void extractTerms(Set terms) { + q.extractTerms(terms); + } + + @Override + public int hashCode() { + return q.hashCode()*31; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof IgnoreAcceptDocsQuery)) return false; + IgnoreAcceptDocsQuery other = (IgnoreAcceptDocsQuery)o; + return q.equals(other.q); + } + + @Override + public String toString(String field) { + return "IgnoreAcceptDocs(" + q + ")"; + } +} diff --git a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java index 86cd9928b73..d1d8175fe0f 100644 --- a/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/AddUpdateCommand.java @@ -18,6 +18,7 @@ package org.apache.solr.update; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexDocument; import org.apache.lucene.index.Term; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; @@ -27,10 +28,15 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + /** * */ -public class AddUpdateCommand extends UpdateCommand { +public class AddUpdateCommand extends UpdateCommand implements Iterable { // optional id in "internal" indexed form... if it is needed and not supplied, // it will be obtained from the doc. private BytesRef indexedId; @@ -143,8 +149,64 @@ public class AddUpdateCommand extends UpdateCommand { } return id; } - - @Override + + public boolean isBlock() { + return solrDoc.hasChildDocuments(); + } + + @Override + public Iterator iterator() { + return new Iterator() { + Iterator iter; + + { + List all = flatten(solrDoc); + + SchemaField uniq = req.getSchema().getUniqueKeyField(); + String idField = getHashableId(); + + for (SolrInputDocument sdoc : all) { + sdoc.setField("_root_", idField); // should this be a string or the same type as the ID? + // TODO: if possible concurrent modification exception (if SolrInputDocument not cloned and is being forwarded to replicas) + // then we could add this field to the generated lucene document instead. + } + + iter = all.iterator(); + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public IndexDocument next() { + return DocumentBuilder.toDocument(iter.next(), req.getSchema()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + private List flatten(SolrInputDocument root) { + List unwrappedDocs = new ArrayList(); + recUnwrapp(unwrappedDocs, root); + Collections.reverse(unwrappedDocs); + return unwrappedDocs; + } + + private void recUnwrapp(List unwrappedDocs, SolrInputDocument currentDoc) { + unwrappedDocs.add(currentDoc); + for (SolrInputDocument child : currentDoc.getChildDocuments()) { + recUnwrapp(unwrappedDocs, child); + } + } + + + @Override public String toString() { StringBuilder sb = new StringBuilder(super.toString()); sb.append(",id=").append(getPrintableId()); @@ -153,4 +215,6 @@ public class AddUpdateCommand extends UpdateCommand { sb.append('}'); return sb.toString(); } - } + + +} diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index a2b253c5cd9..f1daaf0f97a 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -20,16 +20,6 @@ package org.apache.solr.update; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -56,12 +46,22 @@ import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.FunctionRangeQuery; import org.apache.solr.search.QParser; -import org.apache.solr.search.SyntaxError; import org.apache.solr.search.QueryUtils; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SyntaxError; import org.apache.solr.search.function.ValueSourceRangeFilter; import org.apache.solr.util.RefCounted; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; + /** * DirectUpdateHandler2 implements an UpdateHandler where documents are added * directly to the main Lucene index as opposed to adding to a separate smaller index. @@ -199,19 +199,23 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState // normal update Term updateTerm; - Term idTerm = new Term(idField.getName(), cmd.getIndexedId()); + Term idTerm = new Term(cmd.isBlock() ? "_root_" : idField.getName(), cmd.getIndexedId()); boolean del = false; if (cmd.updateTerm == null) { updateTerm = idTerm; } else { + // this is only used by the dedup update processor del = true; updateTerm = cmd.updateTerm; } - - Document luceneDocument = cmd.getLuceneDocument(); - // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer); - writer.updateDocument(updateTerm, luceneDocument, - schema.getAnalyzer()); + + if (cmd.isBlock()) { + writer.updateDocuments(updateTerm, cmd, schema.getAnalyzer()); + } else { + Document luceneDocument = cmd.getLuceneDocument(); + // SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer); + writer.updateDocument(updateTerm, luceneDocument, schema.getAnalyzer()); + } // SolrCore.verbose("updateDocument",updateTerm,"DONE"); if (del) { // ensure id remains unique @@ -234,7 +238,12 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState } else { // allow duplicates - writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer()); + if (cmd.isBlock()) { + writer.addDocuments(cmd, schema.getAnalyzer()); + } else { + writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer()); + } + if (ulog != null) ulog.add(cmd); } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index 3d0068412ff..a22844de0c4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -441,6 +441,8 @@ + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index 59e3bcba6bc..b05e1a7ce9e 100755 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -523,13 +523,15 @@ - - - - + + + + + + @@ -560,6 +562,7 @@ + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index f36e4742630..da6bb36929a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -120,6 +120,12 @@ initialSize="512" autowarmCount="0"/> + + true @@ -553,3 +559,4 @@ + diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java index 8ed0fa23f36..d8e6dd229c3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java @@ -121,6 +121,8 @@ public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase results = query(cloudClient); assertEquals(2, results.getResults().getNumFound()); + docId = testIndexQueryDeleteHierarchical(docId); + testIndexingWithSuss(); // TODO: testOptimisticUpdate(results); @@ -235,6 +237,75 @@ public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase assertEquals(0, query(cloudClient).getResults().getNumFound()); } + private long testIndexQueryDeleteHierarchical(long docId) throws Exception { + //index + int topDocsNum = atLeast(10); + int childsNum = atLeast(10); + for (int i = 0; i < topDocsNum; ++i) { + UpdateRequest uReq = new UpdateRequest(); + SolrInputDocument topDocument = new SolrInputDocument(); + topDocument.addField("id", docId++); + topDocument.addField("type_s", "parent"); + topDocument.addField(i + "parent_f1_s", "v1"); + topDocument.addField(i + "parent_f2_s", "v2"); + + + for (int index = 0; index < childsNum; ++index) { + docId = addChildren("child", topDocument, index, false, docId); + } + + uReq.add(topDocument); + uReq.process(cloudClient); + uReq.process(controlClient); + } + + commit(); + checkShardConsistency(); + assertDocCounts(VERBOSE); + + //query + // parents + SolrQuery query = new SolrQuery("type_s:parent"); + QueryResponse results = cloudClient.query(query); + assertEquals(topDocsNum, results.getResults().getNumFound()); + + //childs + query = new SolrQuery("type_s:child"); + results = cloudClient.query(query); + assertEquals(topDocsNum * childsNum, results.getResults().getNumFound()); + + //grandchilds + query = new SolrQuery("type_s:grand"); + results = cloudClient.query(query); + //each topDoc has t childs where each child has x = 0 + 2 + 4 + ..(t-1)*2 grands + //x = 2 * (1 + 2 + 3 +.. (t-1)) => arithmetic summ of t-1 + //x = 2 * ((t-1) * t / 2) = t * (t - 1) + assertEquals(topDocsNum * childsNum * (childsNum - 1), results.getResults().getNumFound()); + + //delete + del("*:*"); + commit(); + + return docId; + } + + private long addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel, long docId) { + SolrInputDocument childDocument = new SolrInputDocument(); + childDocument.addField("id", docId++); + childDocument.addField("type_s", prefix); + for (int index = 0; index < childIndex; ++index) { + childDocument.addField(childIndex + prefix + index + "_s", childIndex + "value"+ index); + } + + if (!lastLevel) { + for (int i = 0; i < childIndex * 2; ++i) { + docId = addChildren("grand", childDocument, i, true, docId); + } + } + topDocument.addChildDocument(childDocument); + return docId; + } + private void testIndexingWithSuss() throws Exception { ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer( ((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1); diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java index 0a3ebe266d2..57fd43b6df6 100644 --- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java +++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java @@ -298,6 +298,13 @@ public class QueryEqualityTest extends SolrTestCaseJ4 { } } + public void testBlockJoin() throws Exception { + assertQueryEquals("parent", "{!parent which=foo_s:parent}dude", + "{!parent which=foo_s:parent}dude"); + assertQueryEquals("child", "{!child of=foo_s:parent}dude", + "{!child of=foo_s:parent}dude"); + } + public void testQuerySurround() throws Exception { assertQueryEquals("surround", "{!surround}and(apache,solr)", "and(apache,solr)", "apache AND solr"); diff --git a/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java new file mode 100644 index 00000000000..18310b1554c --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/join/BJQParserTest.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.join; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.QParser; +import org.apache.solr.search.SolrCache; +import org.apache.solr.search.SyntaxError; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.ListIterator; + +public class BJQParserTest extends SolrTestCaseJ4 { + + private static final String[] klm = new String[] {"k", "l", "m"}; + private static final List xyz = Arrays.asList("x", "y", "z"); + private static final String[] abcdef = new String[] {"a", "b", "c", "d", "e", "f"}; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema15.xml"); + createIndex(); + } + + public static void createIndex() throws IOException, Exception { + int i = 0; + List> blocks = createBlocks(); + for (List block : blocks) { + for (String[] doc : block) { + String[] idDoc = Arrays.copyOf(doc,doc.length+2); + idDoc[doc.length]="id"; + idDoc[doc.length+1]=Integer.toString(i); + assertU(add(doc(idDoc))); + i++; + } + if (random().nextBoolean()) { + assertU(commit()); + // force empty segment (actually, this will no longer create an empty segment, only a new segments_n) + if (random().nextBoolean()) { + assertU(commit()); + } + } + } + assertU(commit()); + assertQ(req("q", "*:*"), "//*[@numFound='" + i + "']"); + /* + * dump docs well System.out.println(h.query(req("q","*:*", + * "sort","_docid_ asc", "fl", + * "parent_s,child_s,parentchild_s,grand_s,grand_child_s,grand_parentchild_s" + * , "wt","csv", "rows","1000"))); / + */ + } + + private static int id=0; + private static List> createBlocks() { + List> blocks = new ArrayList>(); + for (String parent : abcdef) { + List block = createChildrenBlock(parent); + block.add(new String[] {"parent_s", parent}); + blocks.add(block); + } + Collections.shuffle(blocks, random()); + return blocks; + } + + private static List createChildrenBlock(String parent) { + List block = new ArrayList(); + for (String child : klm) { + block + .add(new String[] {"child_s", child, "parentchild_s", parent + child}); + } + Collections.shuffle(block, random()); + addGrandChildren(block); + return block; + } + + private static void addGrandChildren(List block) { + List grandChildren = new ArrayList(xyz); + // add grandchildren after children + for (ListIterator iter = block.listIterator(); iter.hasNext();) { + String[] child = iter.next(); + String child_s = child[1]; + String parentchild_s = child[3]; + int grandChildPos = 0; + boolean lastLoopButStillHasGrCh = !iter.hasNext() + && !grandChildren.isEmpty(); + while (!grandChildren.isEmpty() + && ((grandChildPos = random().nextInt(grandChildren.size() * 2)) < grandChildren + .size() || lastLoopButStillHasGrCh)) { + grandChildPos = grandChildPos >= grandChildren.size() ? 0 + : grandChildPos; + iter.add(new String[] {"grand_s", grandChildren.remove(grandChildPos), + "grand_child_s", child_s, "grand_parentchild_s", parentchild_s}); + } + } + // and reverse after that + Collections.reverse(block); + } + + @Test + public void testFull() throws IOException, Exception { + String childb = "{!parent which=\"parent_s:[* TO *]\"}child_s:l"; + assertQ(req("q", childb), sixParents); + } + + private static final String sixParents[] = new String[] { + "//*[@numFound='6']", "//doc/arr[@name=\"parent_s\"]/str='a'", + "//doc/arr[@name=\"parent_s\"]/str='b'", + "//doc/arr[@name=\"parent_s\"]/str='c'", + "//doc/arr[@name=\"parent_s\"]/str='d'", + "//doc/arr[@name=\"parent_s\"]/str='e'", + "//doc/arr[@name=\"parent_s\"]/str='f'"}; + + @Test + public void testJustParentsFilter() throws IOException { + assertQ(req("q", "{!parent which=\"parent_s:[* TO *]\"}"), sixParents); + } + + private final static String beParents[] = new String[] {"//*[@numFound='2']", + "//doc/arr[@name=\"parent_s\"]/str='b'", + "//doc/arr[@name=\"parent_s\"]/str='e'"}; + + @Test + public void testIntersectBqBjq() { + + assertQ( + req("q", "+parent_s:(e b) +_query_:\"{!parent which=$pq v=$chq}\"", + "chq", "child_s:l", "pq", "parent_s:[* TO *]"), beParents); + assertQ( + req("fq", "{!parent which=$pq v=$chq}\"", "q", "parent_s:(e b)", "chq", + "child_s:l", "pq", "parent_s:[* TO *]"), beParents); + + assertQ( + req("q", "*:*", "fq", "{!parent which=$pq v=$chq}\"", "fq", + "parent_s:(e b)", "chq", "child_s:l", "pq", "parent_s:[* TO *]"), + beParents); + } + + @Test + public void testFq() { + assertQ( + req("q", "{!parent which=$pq v=$chq}", "fq", "parent_s:(e b)", "chq", + "child_s:l", "pq", "parent_s:[* TO *]"// ,"debugQuery","on" + ), beParents); + + boolean qfq = random().nextBoolean(); + assertQ( + req(qfq ? "q" : "fq", "parent_s:(a e b)", (!qfq) ? "q" : "fq", + "{!parent which=$pq v=$chq}", "chq", "parentchild_s:(bm ek cl)", + "pq", "parent_s:[* TO *]"), beParents); + + } + + @Test + public void testIntersectParentBqChildBq() throws IOException { + + assertQ( + req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"", + "chq", "parentchild_s:(bm ek cl)", "pq", "parent_s:[* TO *]"), + beParents); + } + + @Test + public void testGrandChildren() throws IOException { + assertQ( + req("q", "{!parent which=$parentfilter v=$children}", "children", + "{!parent which=$childrenfilter v=$grandchildren}", + "grandchildren", "grand_s:" + "x", "parentfilter", + "parent_s:[* TO *]", "childrenfilter", "child_s:[* TO *]"), + sixParents); + // int loops = atLeast(1); + String grandChildren = xyz.get(random().nextInt(xyz.size())); + assertQ( + req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"", + "chq", "{!parent which=$childfilter v=$grandchq}", "grandchq", + "+grand_s:" + grandChildren + " +grand_parentchild_s:(b* e* c*)", + "pq", "parent_s:[* TO *]", "childfilter", "child_s:[* TO *]"), + beParents); + } + + @Test + public void testChildrenParser() { + assertQ( + req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:a", "fq", + "NOT grand_s:[* TO *]"), "//*[@numFound='3']", + "//doc/arr[@name=\"child_s\"]/str='k'", + "//doc/arr[@name=\"child_s\"]/str='l'", + "//doc/arr[@name=\"child_s\"]/str='m'"); + assertQ( + req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:b", "fq", + "-parentchild_s:bm", "fq", "-grand_s:*"), "//*[@numFound='2']", + "//doc/arr[@name=\"child_s\"]/str='k'", + "//doc/arr[@name=\"child_s\"]/str='l'"); + } + + @Test + public void testCacheHit() throws IOException { + + SolrCache parentFilterCache = (SolrCache) h.getCore().getInfoRegistry() + .get("perSegFilter"); + + SolrCache filterCache = (SolrCache) h.getCore().getInfoRegistry() + .get("filterCache"); + + NamedList parentsBefore = parentFilterCache.getStatistics(); + + NamedList filtersBefore = filterCache.getStatistics(); + + // it should be weird enough to be uniq + String parentFilter = "parent_s:([a TO c] [d TO f])"; + + assertQ("search by parent filter", + req("q", "{!parent which=\"" + parentFilter + "\"}"), + "//*[@numFound='6']"); + + assertQ("filter by parent filter", + req("q", "*:*", "fq", "{!parent which=\"" + parentFilter + "\"}"), + "//*[@numFound='6']"); + + assertEquals("didn't hit fqCache yet ", 0L, + delta("hits", filterCache.getStatistics(), filtersBefore)); + + assertQ( + "filter by join", + req("q", "*:*", "fq", "{!parent which=\"" + parentFilter + + "\"}child_s:l"), "//*[@numFound='6']"); + + assertEquals("in cache mode every request lookups", 3, + delta("lookups", parentFilterCache.getStatistics(), parentsBefore)); + assertEquals("last two lookups causes hits", 2, + delta("hits", parentFilterCache.getStatistics(), parentsBefore)); + assertEquals("the first lookup gets insert", 1, + delta("inserts", parentFilterCache.getStatistics(), parentsBefore)); + + + assertEquals("true join query is cached in fqCache", 1L, + delta("lookups", filterCache.getStatistics(), filtersBefore)); + } + + private long delta(String key, NamedList a, NamedList b) { + return (Long) a.get(key) - (Long) b.get(key); + } + + + @Test + public void nullInit() { + new BlockJoinParentQParserPlugin().init(null); + } + +} + diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java new file mode 100644 index 00000000000..a273d882ffb --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -0,0 +1,601 @@ +package org.apache.solr.update; + +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeFilter; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.search.join.ToParentBlockJoinQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.handler.loader.XMLLoader; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.util.DefaultSolrThreadFactory; +import org.apache.solr.util.RefCounted; +import org.dom4j.Document; +import org.dom4j.DocumentHelper; +import org.dom4j.Element; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.xml.sax.SAXException; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + + + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +public class AddBlockUpdateTest extends SolrTestCaseJ4 { + + private static final String child = "child_s"; + private static final String parent = "parent_s"; + private static final String type = "type_s"; + + private static ExecutorService exe; + private static AtomicInteger counter = new AtomicInteger(); + private static boolean cachedMode; + + private static XMLInputFactory inputFactory; + + private RefCounted searcherRef; + private SolrIndexSearcher _searcher; + + @BeforeClass + public static void beforeClass() throws Exception { + String oldCacheNamePropValue = System + .getProperty("blockJoinParentFilterCache"); + System.setProperty("blockJoinParentFilterCache", (cachedMode = random() + .nextBoolean()) ? "blockJoinParentFilterCache" : "don't cache"); + if (oldCacheNamePropValue != null) { + System.setProperty("blockJoinParentFilterCache", oldCacheNamePropValue); + } + inputFactory = XMLInputFactory.newInstance(); + + exe = // Executors.newSingleThreadExecutor(); + rarely() ? Executors.newFixedThreadPool(atLeast(2), new DefaultSolrThreadFactory("AddBlockUpdateTest")) : Executors + .newCachedThreadPool(new DefaultSolrThreadFactory("AddBlockUpdateTest")); + + + initCore("solrconfig.xml", "schema15.xml"); + } + + @Before + public void prepare() { + // assertU(""); + assertU(delQ("*:*")); + assertU(commit("expungeDeletes", "true")); + + } + + private SolrIndexSearcher getSearcher() { + if (_searcher == null) { + searcherRef = h.getCore().getSearcher(); + _searcher = searcherRef.get(); + } + return _searcher; + } + + @After + public void cleanup() { + if (searcherRef != null || _searcher != null) { + searcherRef.decref(); + searcherRef = null; + _searcher = null; + } + } + + @AfterClass + public static void afterClass() throws Exception { + inputFactory = null; + exe.shutdownNow(); + } + + @Test + public void testBasics() throws Exception { + List blocks = new ArrayList(Arrays.asList( + block("abcD"), + block("efgH"), + merge(block("ijkL"), block("mnoP")), + merge(block("qrsT"), block("uvwX")), + block("Y"), + block("Z"))); + + Collections.shuffle(blocks); + + log.trace("{}", blocks); + + for (Future f : exe.invokeAll(callables(blocks))) { + f.get(); // exceptions? + } + + assertU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + // final String resp = h.query(req("q","*:*", "sort","_docid_ asc", "rows", + // "10000")); + // log.trace(resp); + int parentsNum = "DHLPTXYZ".length(); + assertQ(req(parent + ":[* TO *]"), "//*[@numFound='" + parentsNum + "']"); + assertQ(req(child + ":[* TO *]"), "//*[@numFound='" + + (('z' - 'a' + 1) - parentsNum) + "']"); + assertQ(req("*:*"), "//*[@numFound='" + ('z' - 'a' + 1) + "']"); + assertSingleParentOf(searcher, one("abc"), "D"); + assertSingleParentOf(searcher, one("efg"), "H"); + assertSingleParentOf(searcher, one("ijk"), "L"); + assertSingleParentOf(searcher, one("mno"), "P"); + assertSingleParentOf(searcher, one("qrs"), "T"); + assertSingleParentOf(searcher, one("uvw"), "X"); + } + + /*** + @Test + public void testSmallBlockDirect() throws Exception { + final AddBlockUpdateCommand cmd = new AddBlockUpdateCommand(req("*:*")); + final List docs = Arrays.asList(new SolrInputDocument() { + { + addField("id", id()); + addField(child, "a"); + } + }, new SolrInputDocument() { + { + addField("id", id()); + addField(parent, "B"); + } + }); + cmd.setDocs(docs); + assertEquals(2, h.getCore().getUpdateHandler().addBlock(cmd)); + assertU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + assertQ(req("*:*"), "//*[@numFound='2']"); + assertSingleParentOf(searcher, one("a"), "B"); + } + + @Test + public void testEmptyDirect() throws Exception { + final AddBlockUpdateCommand cmd = new AddBlockUpdateCommand(req("*:*")); + // let's add empty one + cmd.setDocs(Collections. emptyList()); + assertEquals(0, + ((DirectUpdateHandler2) h.getCore().getUpdateHandler()).addBlock(cmd)); + assertU(commit()); + + assertQ(req("*:*"), "//*[@numFound='0']"); + } + ***/ + + @Test + public void testExceptionThrown() throws Exception { + final String abcD = block("abcD").asXML(); + log.info(abcD); + assertBlockU(abcD); + + Document docToFail = DocumentHelper.createDocument(); + Element root = docToFail.addElement("add"); + Element doc1 = root.addElement("doc"); + attachField(doc1, "id", id()); + attachField(doc1, parent, "Y"); + attachField(doc1, "sample_i", "notanumber"); + Element subDoc1 = doc1.addElement("doc"); + attachField(subDoc1, "id", id()); + attachField(subDoc1, child, "x"); + Element doc2 = root.addElement("doc"); + attachField(doc2, "id", id()); + attachField(doc2, parent, "W"); + + assertFailedBlockU(docToFail.asXML()); + + assertBlockU(block("efgH").asXML()); + assertBlockU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + assertQ(req("q","*:*","indent","true", "fl","id,parent_s,child_s"), "//*[@numFound='" + "abcDefgH".length() + "']"); + assertSingleParentOf(searcher, one("abc"), "D"); + assertSingleParentOf(searcher, one("efg"), "H"); + + assertQ(req(child + ":x"), "//*[@numFound='0']"); + assertQ(req(parent + ":Y"), "//*[@numFound='0']"); + assertQ(req(parent + ":W"), "//*[@numFound='0']"); + } + + @SuppressWarnings("serial") + @Test + public void testSolrJXML() throws IOException { + UpdateRequest req = new UpdateRequest(); + + List docs = new ArrayList(); + + SolrInputDocument document1 = new SolrInputDocument() { + { + final String id = id(); + addField("id", id); + addField("parent_s", "X"); + + ArrayList ch1 = new ArrayList( + Arrays.asList(new SolrInputDocument() { + { + addField("id", id()); + addField("child_s", "y"); + } + }, new SolrInputDocument() { + { + addField("id", id()); + addField("child_s", "z"); + } + })); + + Collections.shuffle(ch1, random()); + addChildDocuments(ch1); + } + }; + + SolrInputDocument document2 = new SolrInputDocument() { + { + final String id = id(); + addField("id", id); + addField("parent_s", "A"); + addChildDocument(new SolrInputDocument() { + { + addField("id", id()); + addField("child_s", "b"); + } + }); + addChildDocument(new SolrInputDocument() { + { + addField("id", id()); + addField("child_s", "c"); + } + }); + } + }; + + docs.add(document1); + docs.add(document2); + + Collections.shuffle(docs, random()); + req.add(docs); + + RequestWriter requestWriter = new RequestWriter(); + OutputStream os = new ByteArrayOutputStream(); + requestWriter.write(req, os); + assertBlockU(os.toString()); + assertU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + assertSingleParentOf(searcher, one("yz"), "X"); + assertSingleParentOf(searcher, one("bc"), "A"); + } + //This is the same as testSolrJXML above but uses the XMLLoader + // to illustrate the structure of the XML documents + @Test + public void testXML() throws IOException, XMLStreamException { + UpdateRequest req = new UpdateRequest(); + + List docs = new ArrayList(); + + + String xml_doc1 = + "" + + " 1" + + " X" + + " " + + " 2" + + " y" + + ""+ + " " + + " 3" + + " z" + + ""+ + ""; + + String xml_doc2 = + "" + + " 4" + + " A" + + " " + + " 5" + + " b" + + ""+ + " " + + " 6" + + " c" + + ""+ + ""; + + + XMLStreamReader parser = + inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) ); + parser.next(); // read the START document... + //null for the processor is all right here + XMLLoader loader = new XMLLoader(); + SolrInputDocument document1 = loader.readDoc( parser ); + + XMLStreamReader parser2 = + inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) ); + parser2.next(); // read the START document... + //null for the processor is all right here + //XMLLoader loader = new XMLLoader(); + SolrInputDocument document2 = loader.readDoc( parser2 ); + + + docs.add(document1); + docs.add(document2); + + Collections.shuffle(docs, random()); + req.add(docs); + + RequestWriter requestWriter = new RequestWriter(); + OutputStream os = new ByteArrayOutputStream(); + requestWriter.write(req, os); + assertBlockU(os.toString()); + assertU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + assertSingleParentOf(searcher, one("yz"), "X"); + assertSingleParentOf(searcher, one("bc"), "A"); + + } + + + @Test + public void testJavaBinCodec() throws IOException { //actually this test must be in other test class + SolrInputDocument topDocument = new SolrInputDocument(); + topDocument.addField("parent_f1", "v1"); + topDocument.addField("parent_f2", "v2"); + + int childsNum = atLeast(10); + for (int index = 0; index < childsNum; ++index) { + addChildren("child", topDocument, index, false); + } + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + new JavaBinCodec().marshal(topDocument, os); + byte[] buffer = os.toByteArray(); + //now read the Object back + InputStream is = new ByteArrayInputStream(buffer); + SolrInputDocument result = (SolrInputDocument) new JavaBinCodec().unmarshal(is); + assertEquals(2, result.size()); + assertEquals("v1", result.getFieldValue("parent_f1")); + assertEquals("v2", result.getFieldValue("parent_f2")); + + List resultChilds = result.getChildDocuments(); + assertEquals(childsNum, resultChilds.size()); + + for (int childIndex = 0; childIndex < childsNum; ++childIndex) { + SolrInputDocument child = resultChilds.get(childIndex); + for (int fieldNum = 0; fieldNum < childIndex; ++fieldNum) { + assertEquals(childIndex + "value" + fieldNum, child.getFieldValue(childIndex + "child" + fieldNum)); + } + + List grandChilds = child.getChildDocuments(); + assertEquals(childIndex * 2, grandChilds.size()); + for (int grandIndex = 0; grandIndex < childIndex * 2; ++grandIndex) { + SolrInputDocument grandChild = grandChilds.get(grandIndex); + assertFalse(grandChild.hasChildDocuments()); + for (int fieldNum = 0; fieldNum < grandIndex; ++fieldNum) { + assertEquals(grandIndex + "value" + fieldNum, grandChild.getFieldValue(grandIndex + "grand" + fieldNum)); + } + } + } + } + + private void addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel) { + SolrInputDocument childDocument = new SolrInputDocument(); + for (int index = 0; index < childIndex; ++index) { + childDocument.addField(childIndex + prefix + index, childIndex + "value"+ index); + } + + if (!lastLevel) { + for (int i = 0; i < childIndex * 2; ++i) { + addChildren("grand", childDocument, i, true); + } + } + topDocument.addChildDocument(childDocument); + } + + /** + * on the given abcD it generates one parent doc, taking D from the tail and + * two subdocs relaitons ab and c uniq ids are supplied also + * + *
+   * {@code
+   * 
+   *  
+   *    D 
+   *     
+   *        a
+   *        1
+   *     
+   *     
+   *        b 
+   *        1
+   *     
+   *     
+   *        c
+   *        2 
+   *     
+   *   
+   * 
+   * }
+   * 
+ * */ + private Document block(String string) { + Document document = DocumentHelper.createDocument(); + Element root = document.addElement("add"); + Element doc = root.addElement("doc"); + + if (string.length() > 0) { + // last character is a top parent + attachField(doc, parent, + String.valueOf(string.charAt(string.length() - 1))); + attachField(doc, "id", id()); + + // add subdocs + int type = 1; + for (int i = 0; i < string.length() - 1; i += 2) { + String relation = string.substring(i, + Math.min(i + 2, string.length() - 1)); + attachSubDocs(doc, relation, type); + type++; + } + } + + return document; + } + + private void attachSubDocs(Element parent, String relation, int typeValue) { + for (int j = 0; j < relation.length(); j++) { + Element document = parent.addElement("doc"); + attachField(document, child, String.valueOf(relation.charAt(j))); + attachField(document, "id", id()); + attachField(document, type, String.valueOf(typeValue)); + } + } + + /** + * Merges two documents like + * + *
+   * {@code ... + ... = ... + ...}
+   * 
+ * + * @param doc1 + * first document + * @param doc2 + * second document + * @return merged document + */ + private Document merge(Document doc1, Document doc2) { + List list = doc2.getRootElement().elements(); + for (Element element : list) { + doc1.getRootElement().add(element.detach()); + } + + return doc1; + } + + private void attachField(Element root, String fieldName, String value) { + Element field = root.addElement("field"); + field.addAttribute("name", fieldName); + field.addText(value); + } + + private static String id() { + return "" + counter.incrementAndGet(); + } + + private String one(String string) { + return "" + string.charAt(random().nextInt(string.length())); + } + + protected void assertSingleParentOf(final SolrIndexSearcher searcher, + final String childTerm, String parentExp) throws IOException { + final TopDocs docs = searcher.search(join(childTerm), 10); + assertEquals(1, docs.totalHits); + final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent); + assertEquals(parentExp, pAct); + } + + protected ToParentBlockJoinQuery join(final String childTerm) { + return new ToParentBlockJoinQuery( + new TermQuery(new Term(child, childTerm)), new TermRangeFilter(parent, + null, null, false, false), ScoreMode.None); + } + + private Collection> callables(List blocks) { + final List> rez = new ArrayList>(); + for (Document block : blocks) { + final String msg = block.asXML(); + if (msg.length() > 0) { + rez.add(new Callable() { + @Override + public Void call() { + assertBlockU(msg); + return null; + } + + }); + if (rarely()) { + rez.add(new Callable() { + @Override + public Void call() { + assertBlockU(commit()); + return null; + } + + }); + } + } + } + return rez; + } + + private void assertBlockU(final String msg) { + assertBlockU(msg, "0"); + } + + private void assertFailedBlockU(final String msg) { + try { + assertBlockU(msg, "1"); + fail("expecting fail"); + } catch (Exception e) { + // gotcha + } + } + + private void assertBlockU(final String msg, String expected) { + try { + String res = h.checkUpdateStatus(msg, expected); + if (res != null) { + fail("update was not successful: " + res + " expected: " + expected); + } + } catch (SAXException e) { + throw new RuntimeException("Invalid XML", e); + } + } +} + diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 95a29b59c0a..2385512a1a0 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -133,6 +133,11 @@ public class ClientUtils } } } + + for (SolrInputDocument childDocument : doc.getChildDocuments()) { + writeXML(childDocument, writer); + } + writer.write(""); } diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java index 78aa5301aff..0fdf78b7a39 100644 --- a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java +++ b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java @@ -18,9 +18,12 @@ package org.apache.solr.common; import java.io.Serializable; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -36,13 +39,16 @@ public class SolrInputDocument implements Map, Iterable _fields; private float _documentBoost = 1.0f; - + private List _childDocuments; + public SolrInputDocument() { _fields = new LinkedHashMap(); + _childDocuments = new ArrayList(); } public SolrInputDocument(Map fields) { _fields = fields; + _childDocuments = new ArrayList(); } /** @@ -52,7 +58,10 @@ public class SolrInputDocument implements Map, Iterable, Iterable, Iterable(_childDocuments.size()); + for (SolrInputDocument child : _childDocuments) { + clone._childDocuments.add(child.deepCopy()); + } + return clone; } @@ -260,4 +275,23 @@ public class SolrInputDocument implements Map, Iterable values() { return _fields.values(); } + + public void addChildDocument(SolrInputDocument child) { + _childDocuments.add(child); + } + + public void addChildDocuments(Collection childs) { + for (SolrInputDocument child : childs) { + addChildDocument(child); + } + } + + public List getChildDocuments() { + return _childDocuments; + } + + public boolean hasChildDocuments() { + boolean isEmpty = (_childDocuments == null || _childDocuments.isEmpty()); + return !isEmpty; + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java index d65bdc304eb..583c33eb35b 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java @@ -62,6 +62,7 @@ public class JavaBinCodec { END = 15, SOLRINPUTDOC = 16, + SOLRINPUTDOC_CHILDS = 17, // types that combine tag + length (or other info) in a single byte TAG_AND_LEN = (byte) (1 << 5), @@ -358,6 +359,8 @@ public class JavaBinCodec { public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException { int sz = readVInt(dis); + dis.readByte(); // skip childDocuments tag + int childsSize = readVInt(dis); float docBoost = (Float)readVal(dis); SolrInputDocument sdoc = new SolrInputDocument(); sdoc.setDocumentBoost(docBoost); @@ -374,11 +377,17 @@ public class JavaBinCodec { Object fieldVal = readVal(dis); sdoc.setField(fieldName, fieldVal, boost); } + for (int i = 0; i < childsSize; i++) { + dis.readByte(); // skip solrinputdoc tag + SolrInputDocument child = readSolrInputDocument(dis); + sdoc.addChildDocument(child); + } return sdoc; } public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException { writeTag(SOLRINPUTDOC, sdoc.size()); + writeTag(SOLRINPUTDOC_CHILDS, sdoc.getChildDocuments().size()); writeFloat(sdoc.getDocumentBoost()); for (SolrInputField inputField : sdoc.values()) { if (inputField.getBoost() != 1.0f) { @@ -387,6 +396,9 @@ public class JavaBinCodec { writeExternString(inputField.getName()); writeVal(inputField.getValue()); } + for (SolrInputDocument child : sdoc.getChildDocuments()) { + writeSolrInputDocument(child); + } } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index b0288861147..35f48d06619 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -72,6 +72,7 @@ import java.io.File; import java.io.IOException; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -969,6 +970,10 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { return sd; } + public static List sdocs(SolrInputDocument... docs) { + return Arrays.asList(docs); + } + /** Converts "test JSON" and returns standard JSON. * Currently this only consists of changing unescaped single quotes to double quotes, * and escaped single quotes to single quotes. @@ -1527,7 +1532,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase { } /** Return a Map from field value to a list of document ids */ - Map> invertField(Map model, String field) { + public Map> invertField(Map model, String field) { Map> value_to_id = new HashMap>(); // invert field