mirror of https://github.com/apache/lucene.git
SOLR-3076: block join parent and child queries
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1513290 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley
parent
d4c20941c9
commit
f28864213f
|
|
@ -25,5 +25,6 @@
|
|||
<orderEntry type="module" module-name="analysis-common" />
|
||||
<orderEntry type="module" module-name="lucene-core" />
|
||||
<orderEntry type="module" module-name="queryparser" />
|
||||
<orderEntry type="module" module-name="join" />
|
||||
</component>
|
||||
</module>
|
||||
|
|
|
|||
|
|
@ -27,5 +27,6 @@
|
|||
<orderEntry type="module" scope="TEST" module-name="suggest" />
|
||||
<orderEntry type="module" scope="TEST" module-name="spatial" />
|
||||
<orderEntry type="module" scope="TEST" module-name="misc" />
|
||||
<orderEntry type="module" scope="TEST" module-name="join" />
|
||||
</component>
|
||||
</module>
|
||||
|
|
|
|||
|
|
@ -93,6 +93,11 @@
|
|||
<artifactId>lucene-misc</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-join</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-queryparser</artifactId>
|
||||
|
|
|
|||
|
|
@ -198,6 +198,12 @@
|
|||
<artifactId>jetty-util</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dom4j</groupId>
|
||||
<artifactId>dom4j</artifactId>
|
||||
<version>1.6.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<build>
|
||||
<sourceDirectory/>
|
||||
|
|
|
|||
|
|
@ -132,6 +132,17 @@
|
|||
</ant>
|
||||
<property name="queryparser-javadocs.uptodate" value="true"/>
|
||||
</target>
|
||||
|
||||
<property name="join.jar" value="${common.dir}/build/join/lucene-join-${version}.jar"/>
|
||||
<target name="check-join-uptodate" unless="join.uptodate">
|
||||
<module-uptodate name="join" jarfile="${join.jar}" property="join.uptodate"/>
|
||||
</target>
|
||||
<target name="jar-join" unless="join.uptodate" depends="check-join-uptodate">
|
||||
<ant dir="${common.dir}/join" target="jar-core" inheritAll="false">
|
||||
<propertyset refid="uptodate.and.compiled.properties"/>
|
||||
</ant>
|
||||
<property name="join.uptodate" value="true"/>
|
||||
</target>
|
||||
|
||||
<property name="analyzers-common.jar" value="${common.dir}/build/analysis/common/lucene-analyzers-common-${version}.jar"/>
|
||||
<target name="check-analyzers-common-uptodate" unless="analyzers-common.uptodate">
|
||||
|
|
|
|||
|
|
@ -181,6 +181,7 @@ Upgrading from Solr 4.3.0
|
|||
* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed
|
||||
in 5.0. If you are still using these field types, you should migrate your
|
||||
fields to TrieIntField.
|
||||
|
||||
|
||||
Detailed Change List
|
||||
----------------------
|
||||
|
|
@ -261,6 +262,16 @@ New Features
|
|||
* SOLR-4943: Add a new system wide info admin handler that exposes the system info
|
||||
that could previously only be retrieved using a SolrCore. (Mark Miller)
|
||||
|
||||
* SOLR-3076: Block joins. Documents and their sub-documents must be indexed
|
||||
as a block.
|
||||
{!parent which=<allParents>}<someChildren> takes in a query that matches child
|
||||
documents and results in matches on their parents.
|
||||
{!child of=<allParents>}<someParents> takes in a query that matches some parent
|
||||
documents and results in matches on their children.
|
||||
(Mikhail Khludnev, Vadim Kirilchuk, Alan Woodward, Tom Burton-West, Mike McCandless,
|
||||
hossman, yonik)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@
|
|||
<pathelement location="${grouping.jar}"/>
|
||||
<pathelement location="${queries.jar}"/>
|
||||
<pathelement location="${queryparser.jar}"/>
|
||||
<pathelement location="${join.jar}"/>
|
||||
</path>
|
||||
|
||||
<path id="solr.base.classpath">
|
||||
|
|
@ -141,7 +142,7 @@
|
|||
|
||||
<target name="prep-lucene-jars"
|
||||
depends="jar-lucene-core, jar-analyzers-phonetic, jar-analyzers-kuromoji, jar-codecs, jar-suggest, jar-highlighter, jar-memory,
|
||||
jar-misc, jar-spatial, jar-grouping, jar-queries, jar-queryparser">
|
||||
jar-misc, jar-spatial, jar-grouping, jar-queries, jar-queryparser, jar-join">
|
||||
<property name="solr.deps.compiled" value="true"/>
|
||||
</target>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@
|
|||
<dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" conf="compile->*"/>
|
||||
<dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" conf="compile->*"/>
|
||||
<dependency org="joda-time" name="joda-time" rev="2.2" conf="compile->*"/>
|
||||
<dependency org="dom4j" name="dom4j" rev="1.6.1" transitive="false"/>
|
||||
|
||||
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" conf="test->*"/>
|
||||
<dependency org="org.easymock" name="easymock" rev="3.0" conf="test->*"/>
|
||||
|
|
|
|||
|
|
@ -16,50 +16,51 @@ package org.apache.solr.handler.loader;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.update.processor.UpdateRequestProcessor;
|
||||
import org.apache.solr.update.AddUpdateCommand;
|
||||
import org.apache.solr.update.CommitUpdateCommand;
|
||||
import org.apache.solr.update.RollbackUpdateCommand;
|
||||
import org.apache.solr.update.DeleteUpdateCommand;
|
||||
import org.apache.solr.util.xslt.TransformerProvider;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.UpdateParams;
|
||||
import org.apache.solr.common.util.ContentStream;
|
||||
import org.apache.solr.common.util.ContentStreamBase;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.common.util.XMLErrorLogger;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.UpdateParams;
|
||||
import org.apache.solr.core.SolrConfig;
|
||||
import org.apache.solr.handler.RequestHandlerUtils;
|
||||
import org.apache.solr.handler.UpdateRequestHandler;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.update.AddUpdateCommand;
|
||||
import org.apache.solr.update.CommitUpdateCommand;
|
||||
import org.apache.solr.update.DeleteUpdateCommand;
|
||||
import org.apache.solr.update.RollbackUpdateCommand;
|
||||
import org.apache.solr.update.processor.UpdateRequestProcessor;
|
||||
import org.apache.solr.util.EmptyEntityResolver;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.solr.util.xslt.TransformerProvider;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.xml.sax.InputSource;
|
||||
import org.xml.sax.XMLReader;
|
||||
|
||||
import javax.xml.stream.XMLStreamReader;
|
||||
import javax.xml.stream.XMLStreamException;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
import javax.xml.stream.FactoryConfigurationError;
|
||||
import javax.xml.stream.XMLStreamConstants;
|
||||
import javax.xml.stream.XMLInputFactory;
|
||||
import javax.xml.stream.XMLStreamConstants;
|
||||
import javax.xml.stream.XMLStreamException;
|
||||
import javax.xml.stream.XMLStreamReader;
|
||||
import javax.xml.transform.Transformer;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import javax.xml.transform.dom.DOMResult;
|
||||
import javax.xml.transform.dom.DOMSource;
|
||||
import javax.xml.transform.sax.SAXSource;
|
||||
import javax.xml.parsers.SAXParserFactory;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -381,6 +382,7 @@ public class XMLLoader extends ContentStreamLoader {
|
|||
float boost = 1.0f;
|
||||
boolean isNull = false;
|
||||
String update = null;
|
||||
Collection<SolrInputDocument> subDocs = null;
|
||||
Map<String, Map<String, Object>> updateMap = null;
|
||||
boolean complete = false;
|
||||
while (!complete) {
|
||||
|
|
@ -395,9 +397,14 @@ public class XMLLoader extends ContentStreamLoader {
|
|||
|
||||
case XMLStreamConstants.END_ELEMENT:
|
||||
if ("doc".equals(parser.getLocalName())) {
|
||||
if (subDocs != null && !subDocs.isEmpty()) {
|
||||
doc.addChildDocuments(subDocs);
|
||||
subDocs = null;
|
||||
}
|
||||
complete = true;
|
||||
break;
|
||||
} else if ("field".equals(parser.getLocalName())) {
|
||||
// should I warn in some text has been found too
|
||||
Object v = isNull ? null : text.toString();
|
||||
if (update != null) {
|
||||
if (updateMap == null) updateMap = new HashMap<String, Map<String, Object>>();
|
||||
|
|
@ -425,34 +432,43 @@ public class XMLLoader extends ContentStreamLoader {
|
|||
}
|
||||
doc.addField(name, v, boost);
|
||||
boost = 1.0f;
|
||||
// field is over
|
||||
name = null;
|
||||
}
|
||||
break;
|
||||
|
||||
case XMLStreamConstants.START_ELEMENT:
|
||||
text.setLength(0);
|
||||
String localName = parser.getLocalName();
|
||||
if (!"field".equals(localName)) {
|
||||
log.warn("unexpected XML tag doc/" + localName);
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"unexpected XML tag doc/" + localName);
|
||||
if ("doc".equals(localName)) {
|
||||
if (subDocs == null)
|
||||
subDocs = Lists.newArrayList();
|
||||
subDocs.add(readDoc(parser));
|
||||
}
|
||||
boost = 1.0f;
|
||||
update = null;
|
||||
isNull = false;
|
||||
String attrVal = "";
|
||||
for (int i = 0; i < parser.getAttributeCount(); i++) {
|
||||
attrName = parser.getAttributeLocalName(i);
|
||||
attrVal = parser.getAttributeValue(i);
|
||||
if ("name".equals(attrName)) {
|
||||
name = attrVal;
|
||||
} else if ("boost".equals(attrName)) {
|
||||
boost = Float.parseFloat(attrVal);
|
||||
} else if ("null".equals(attrName)) {
|
||||
isNull = StrUtils.parseBoolean(attrVal);
|
||||
} else if ("update".equals(attrName)) {
|
||||
update = attrVal;
|
||||
} else {
|
||||
log.warn("Unknown attribute doc/field/@" + attrName);
|
||||
else {
|
||||
if (!"field".equals(localName)) {
|
||||
log.warn("unexpected XML tag doc/" + localName);
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"unexpected XML tag doc/" + localName);
|
||||
}
|
||||
boost = 1.0f;
|
||||
update = null;
|
||||
isNull = false;
|
||||
String attrVal = "";
|
||||
for (int i = 0; i < parser.getAttributeCount(); i++) {
|
||||
attrName = parser.getAttributeLocalName(i);
|
||||
attrVal = parser.getAttributeValue(i);
|
||||
if ("name".equals(attrName)) {
|
||||
name = attrVal;
|
||||
} else if ("boost".equals(attrName)) {
|
||||
boost = Float.parseFloat(attrVal);
|
||||
} else if ("null".equals(attrName)) {
|
||||
isNull = StrUtils.parseBoolean(attrVal);
|
||||
} else if ("update".equals(attrName)) {
|
||||
update = attrVal;
|
||||
} else {
|
||||
log.warn("Unknown attribute doc/field/@" + attrName);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import org.apache.solr.common.params.SolrParams;
|
|||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.SolrInfoMBean;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.search.join.BlockJoinChildQParserPlugin;
|
||||
import org.apache.solr.search.join.BlockJoinParentQParserPlugin;
|
||||
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
|
||||
|
||||
import java.net.URL;
|
||||
|
|
@ -47,7 +49,9 @@ public abstract class QParserPlugin implements NamedListInitializedPlugin, SolrI
|
|||
JoinQParserPlugin.NAME, JoinQParserPlugin.class,
|
||||
SurroundQParserPlugin.NAME, SurroundQParserPlugin.class,
|
||||
SwitchQParserPlugin.NAME, SwitchQParserPlugin.class,
|
||||
MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class
|
||||
MaxScoreQParserPlugin.NAME, MaxScoreQParserPlugin.class,
|
||||
BlockJoinParentQParserPlugin.NAME, BlockJoinParentQParserPlugin.class,
|
||||
BlockJoinChildQParserPlugin.NAME, BlockJoinChildQParserPlugin.class
|
||||
};
|
||||
|
||||
/** return a {@link QParser} */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.lucene.util.OpenBitSet;
|
||||
|
||||
class BitSetSlice {
|
||||
private final OpenBitSet obs;
|
||||
private final int off;
|
||||
private final int len;
|
||||
|
||||
BitSetSlice(OpenBitSet obs, int off, int len) {
|
||||
this.obs = obs;
|
||||
this.off = off;
|
||||
this.len = len;
|
||||
}
|
||||
|
||||
public boolean get(int pos) {
|
||||
return obs.get(pos + off);
|
||||
}
|
||||
|
||||
public int prevSetBit(int pos) {
|
||||
int result = obs.prevSetBit(pos + off) - off;
|
||||
return (result < 0) ? -1 : result;
|
||||
}
|
||||
|
||||
public int nextSetBit(int pos) {
|
||||
int result = obs.nextSetBit(pos + off) - off;
|
||||
return (result < 0 || result >= len) ? -1 : result;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.join.ToChildBlockJoinQuery;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
|
||||
public class BlockJoinChildQParser extends BlockJoinParentQParser {
|
||||
|
||||
public BlockJoinChildQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
super(qstr, localParams, params, req);
|
||||
}
|
||||
|
||||
protected Query createQuery(Query parentListQuery, Query query) {
|
||||
return new ToChildBlockJoinQuery(query, getFilter(parentListQuery), false);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getParentFilterLocalParamName() {
|
||||
return "of";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.search.QParser;
|
||||
|
||||
/**
|
||||
* Usage: {!child of="PARENT:true"}PARENT_PRICE:10
|
||||
*
|
||||
**/
|
||||
public class BlockJoinChildQParserPlugin extends BlockJoinParentQParserPlugin {
|
||||
public static String NAME = "child";
|
||||
|
||||
@Override
|
||||
protected QParser createBJQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
return new BlockJoinChildQParser(qstr, localParams, params, req);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.lucene.search.CachingWrapperFilter;
|
||||
import org.apache.lucene.search.ConstantScoreQuery;
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.QueryWrapperFilter;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.QueryParsing;
|
||||
import org.apache.solr.search.SolrCache;
|
||||
import org.apache.solr.search.SolrConstantScoreQuery;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
|
||||
class BlockJoinParentQParser extends QParser {
|
||||
/** implementation detail subject to change */
|
||||
public String CACHE_NAME="perSegFilter";
|
||||
|
||||
protected String getParentFilterLocalParamName() {
|
||||
return "which";
|
||||
}
|
||||
|
||||
BlockJoinParentQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
super(qstr, localParams, params, req);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query parse() throws SyntaxError {
|
||||
String filter = localParams.get(getParentFilterLocalParamName());
|
||||
QParser parentParser = subQuery(filter, null);
|
||||
Query parentQ = parentParser.getQuery();
|
||||
|
||||
String queryText = localParams.get(QueryParsing.V);
|
||||
// there is no child query, return parent filter from cache
|
||||
if (queryText == null || queryText.length()==0) {
|
||||
SolrConstantScoreQuery wrapped = new SolrConstantScoreQuery(getFilter(parentQ));
|
||||
wrapped.setCache(false);
|
||||
return wrapped;
|
||||
}
|
||||
QParser childrenParser = subQuery(queryText, null);
|
||||
Query childrenQuery = childrenParser.getQuery();
|
||||
return createQuery(parentQ, childrenQuery);
|
||||
}
|
||||
|
||||
protected Query createQuery(Query parentList, Query query) {
|
||||
return new ToParentBlockJoinQuery(query, getFilter(parentList), ScoreMode.None);
|
||||
}
|
||||
|
||||
protected Filter getFilter(Query parentList) {
|
||||
SolrCache parentCache = req.getSearcher().getCache(CACHE_NAME);
|
||||
// lazily retrieve from solr cache
|
||||
Filter filter = null;
|
||||
if (parentCache != null) {
|
||||
filter = (Filter) parentCache.get(parentList);
|
||||
}
|
||||
Filter result;
|
||||
if (filter == null) {
|
||||
result = createParentFilter(parentList);
|
||||
if (parentCache != null) {
|
||||
parentCache.put(parentList, result);
|
||||
}
|
||||
} else {
|
||||
result = filter;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
protected Filter createParentFilter(Query parentQ) {
|
||||
return new CachingWrapperFilter(new QueryWrapperFilter(parentQ)) {
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.QParserPlugin;
|
||||
|
||||
/**
|
||||
* Usage: {!parent which="PARENT:true"}CHILD_PRICE:10
|
||||
*
|
||||
**/
|
||||
public class BlockJoinParentQParserPlugin extends QParserPlugin {
|
||||
public static String NAME = "parent";
|
||||
|
||||
@Override
|
||||
public QParser createParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
QParser parser = createBJQParser(qstr, localParams, params, req);
|
||||
return parser;
|
||||
}
|
||||
|
||||
protected QParser createBJQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
|
||||
return new BlockJoinParentQParser(qstr, localParams, params, req);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(NamedList args) {
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.Explanation;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.Scorer;
|
||||
import org.apache.lucene.search.Weight;
|
||||
import org.apache.lucene.util.Bits;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Set;
|
||||
|
||||
public class IgnoreAcceptDocsQuery extends Query {
|
||||
private final Query q;
|
||||
|
||||
public IgnoreAcceptDocsQuery(Query q) {
|
||||
this.q = q;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setBoost(float b) {
|
||||
q.setBoost(b);
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getBoost() {
|
||||
return q.getBoost();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return q.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Weight createWeight(IndexSearcher searcher) throws IOException {
|
||||
Weight inner = q.createWeight(searcher);
|
||||
return new IADWeight(inner);
|
||||
}
|
||||
|
||||
private class IADWeight extends Weight {
|
||||
Weight w;
|
||||
|
||||
IADWeight(Weight delegate) {
|
||||
this.w = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
|
||||
return w.explain(context, doc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query getQuery() {
|
||||
return q;
|
||||
}
|
||||
|
||||
@Override
|
||||
public float getValueForNormalization() throws IOException {
|
||||
return w.getValueForNormalization();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void normalize(float norm, float topLevelBoost) {
|
||||
w.normalize(norm, topLevelBoost);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException {
|
||||
return w.scorer(context, scoreDocsInOrder, topScorer, null);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Query rewrite(IndexReader reader) throws IOException {
|
||||
Query n = q.rewrite(reader);
|
||||
if (q == n) return this;
|
||||
return new IgnoreAcceptDocsQuery(n);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void extractTerms(Set<Term> terms) {
|
||||
q.extractTerms(terms);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return q.hashCode()*31;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (!(o instanceof IgnoreAcceptDocsQuery)) return false;
|
||||
IgnoreAcceptDocsQuery other = (IgnoreAcceptDocsQuery)o;
|
||||
return q.equals(other.q);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString(String field) {
|
||||
return "IgnoreAcceptDocs(" + q + ")";
|
||||
}
|
||||
}
|
||||
|
|
@ -18,6 +18,7 @@
|
|||
package org.apache.solr.update;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.IndexDocument;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.common.SolrException;
|
||||
|
|
@ -27,10 +28,15 @@ import org.apache.solr.request.SolrQueryRequest;
|
|||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class AddUpdateCommand extends UpdateCommand {
|
||||
public class AddUpdateCommand extends UpdateCommand implements Iterable<IndexDocument> {
|
||||
// optional id in "internal" indexed form... if it is needed and not supplied,
|
||||
// it will be obtained from the doc.
|
||||
private BytesRef indexedId;
|
||||
|
|
@ -143,8 +149,64 @@ public class AddUpdateCommand extends UpdateCommand {
|
|||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
public boolean isBlock() {
|
||||
return solrDoc.hasChildDocuments();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<IndexDocument> iterator() {
|
||||
return new Iterator<IndexDocument>() {
|
||||
Iterator<SolrInputDocument> iter;
|
||||
|
||||
{
|
||||
List<SolrInputDocument> all = flatten(solrDoc);
|
||||
|
||||
SchemaField uniq = req.getSchema().getUniqueKeyField();
|
||||
String idField = getHashableId();
|
||||
|
||||
for (SolrInputDocument sdoc : all) {
|
||||
sdoc.setField("_root_", idField); // should this be a string or the same type as the ID?
|
||||
// TODO: if possible concurrent modification exception (if SolrInputDocument not cloned and is being forwarded to replicas)
|
||||
// then we could add this field to the generated lucene document instead.
|
||||
}
|
||||
|
||||
iter = all.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return iter.hasNext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public IndexDocument next() {
|
||||
return DocumentBuilder.toDocument(iter.next(), req.getSchema());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove() {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private List<SolrInputDocument> flatten(SolrInputDocument root) {
|
||||
List<SolrInputDocument> unwrappedDocs = new ArrayList<SolrInputDocument>();
|
||||
recUnwrapp(unwrappedDocs, root);
|
||||
Collections.reverse(unwrappedDocs);
|
||||
return unwrappedDocs;
|
||||
}
|
||||
|
||||
private void recUnwrapp(List<SolrInputDocument> unwrappedDocs, SolrInputDocument currentDoc) {
|
||||
unwrappedDocs.add(currentDoc);
|
||||
for (SolrInputDocument child : currentDoc.getChildDocuments()) {
|
||||
recUnwrapp(unwrappedDocs, child);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder(super.toString());
|
||||
sb.append(",id=").append(getPrintableId());
|
||||
|
|
@ -153,4 +215,6 @@ public class AddUpdateCommand extends UpdateCommand {
|
|||
sb.append('}');
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,16 +20,6 @@
|
|||
|
||||
package org.apache.solr.update;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
|
|
@ -56,12 +46,22 @@ import org.apache.solr.schema.IndexSchema;
|
|||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.FunctionRangeQuery;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.apache.solr.search.QueryUtils;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.apache.solr.search.function.ValueSourceRangeFilter;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* <code>DirectUpdateHandler2</code> implements an UpdateHandler where documents are added
|
||||
* directly to the main Lucene index as opposed to adding to a separate smaller index.
|
||||
|
|
@ -199,19 +199,23 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
|
|||
// normal update
|
||||
|
||||
Term updateTerm;
|
||||
Term idTerm = new Term(idField.getName(), cmd.getIndexedId());
|
||||
Term idTerm = new Term(cmd.isBlock() ? "_root_" : idField.getName(), cmd.getIndexedId());
|
||||
boolean del = false;
|
||||
if (cmd.updateTerm == null) {
|
||||
updateTerm = idTerm;
|
||||
} else {
|
||||
// this is only used by the dedup update processor
|
||||
del = true;
|
||||
updateTerm = cmd.updateTerm;
|
||||
}
|
||||
|
||||
Document luceneDocument = cmd.getLuceneDocument();
|
||||
// SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
|
||||
writer.updateDocument(updateTerm, luceneDocument,
|
||||
schema.getAnalyzer());
|
||||
|
||||
if (cmd.isBlock()) {
|
||||
writer.updateDocuments(updateTerm, cmd, schema.getAnalyzer());
|
||||
} else {
|
||||
Document luceneDocument = cmd.getLuceneDocument();
|
||||
// SolrCore.verbose("updateDocument",updateTerm,luceneDocument,writer);
|
||||
writer.updateDocument(updateTerm, luceneDocument, schema.getAnalyzer());
|
||||
}
|
||||
// SolrCore.verbose("updateDocument",updateTerm,"DONE");
|
||||
|
||||
if (del) { // ensure id remains unique
|
||||
|
|
@ -234,7 +238,12 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState
|
|||
|
||||
} else {
|
||||
// allow duplicates
|
||||
writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer());
|
||||
if (cmd.isBlock()) {
|
||||
writer.addDocuments(cmd, schema.getAnalyzer());
|
||||
} else {
|
||||
writer.addDocument(cmd.getLuceneDocument(), schema.getAnalyzer());
|
||||
}
|
||||
|
||||
if (ulog != null) ulog.add(cmd);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -441,6 +441,8 @@
|
|||
|
||||
<fields>
|
||||
<field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
|
||||
<field name="_root_" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
|
||||
|
||||
<field name="signatureField" type="string" indexed="true" stored="false"/>
|
||||
<field name="uuid" type="uuid" stored="true" />
|
||||
<field name="name" type="nametext" indexed="true" stored="true"/>
|
||||
|
|
|
|||
|
|
@ -523,13 +523,15 @@
|
|||
<field name="uniq3" type="string" indexed="true" stored="true"/>
|
||||
<field name="nouniq" type="string" indexed="true" stored="true" multiValued="true"/>
|
||||
|
||||
<!-- for versioning -->
|
||||
<field name="_version_" type="long" indexed="true" stored="true"/>
|
||||
|
||||
|
||||
|
||||
<field name="copyfield_source" type="string" indexed="true" stored="true" multiValued="true"/>
|
||||
|
||||
<!-- for versioning -->
|
||||
<field name="_version_" type="long" indexed="true" stored="true"/>
|
||||
<!-- points to the root document of a block of nested documents -->
|
||||
<field name="_root_" type="string" indexed="true" stored="true"/>
|
||||
|
||||
|
||||
|
||||
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false"/>
|
||||
|
||||
|
|
@ -560,6 +562,7 @@
|
|||
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_tt" type="text" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_ws" type="nametext" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_f" type="float" indexed="true" stored="true"/>
|
||||
<dynamicField name="*_d" type="double" indexed="true" stored="true"/>
|
||||
|
|
|
|||
|
|
@ -120,6 +120,12 @@
|
|||
initialSize="512"
|
||||
autowarmCount="0"/>
|
||||
|
||||
<cache name="perSegFilter"
|
||||
class="solr.search.LRUCache"
|
||||
size="10"
|
||||
initialSize="0"
|
||||
autowarmCount="10" />
|
||||
|
||||
<!-- If true, stored fields that are not requested will be loaded lazily.
|
||||
-->
|
||||
<enableLazyFieldLoading>true</enableLazyFieldLoading>
|
||||
|
|
@ -553,3 +559,4 @@
|
|||
</updateRequestProcessorChain>
|
||||
|
||||
</config>
|
||||
|
||||
|
|
|
|||
|
|
@ -121,6 +121,8 @@ public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase
|
|||
results = query(cloudClient);
|
||||
assertEquals(2, results.getResults().getNumFound());
|
||||
|
||||
docId = testIndexQueryDeleteHierarchical(docId);
|
||||
|
||||
testIndexingWithSuss();
|
||||
|
||||
// TODO: testOptimisticUpdate(results);
|
||||
|
|
@ -235,6 +237,75 @@ public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase
|
|||
assertEquals(0, query(cloudClient).getResults().getNumFound());
|
||||
}
|
||||
|
||||
private long testIndexQueryDeleteHierarchical(long docId) throws Exception {
|
||||
//index
|
||||
int topDocsNum = atLeast(10);
|
||||
int childsNum = atLeast(10);
|
||||
for (int i = 0; i < topDocsNum; ++i) {
|
||||
UpdateRequest uReq = new UpdateRequest();
|
||||
SolrInputDocument topDocument = new SolrInputDocument();
|
||||
topDocument.addField("id", docId++);
|
||||
topDocument.addField("type_s", "parent");
|
||||
topDocument.addField(i + "parent_f1_s", "v1");
|
||||
topDocument.addField(i + "parent_f2_s", "v2");
|
||||
|
||||
|
||||
for (int index = 0; index < childsNum; ++index) {
|
||||
docId = addChildren("child", topDocument, index, false, docId);
|
||||
}
|
||||
|
||||
uReq.add(topDocument);
|
||||
uReq.process(cloudClient);
|
||||
uReq.process(controlClient);
|
||||
}
|
||||
|
||||
commit();
|
||||
checkShardConsistency();
|
||||
assertDocCounts(VERBOSE);
|
||||
|
||||
//query
|
||||
// parents
|
||||
SolrQuery query = new SolrQuery("type_s:parent");
|
||||
QueryResponse results = cloudClient.query(query);
|
||||
assertEquals(topDocsNum, results.getResults().getNumFound());
|
||||
|
||||
//childs
|
||||
query = new SolrQuery("type_s:child");
|
||||
results = cloudClient.query(query);
|
||||
assertEquals(topDocsNum * childsNum, results.getResults().getNumFound());
|
||||
|
||||
//grandchilds
|
||||
query = new SolrQuery("type_s:grand");
|
||||
results = cloudClient.query(query);
|
||||
//each topDoc has t childs where each child has x = 0 + 2 + 4 + ..(t-1)*2 grands
|
||||
//x = 2 * (1 + 2 + 3 +.. (t-1)) => arithmetic summ of t-1
|
||||
//x = 2 * ((t-1) * t / 2) = t * (t - 1)
|
||||
assertEquals(topDocsNum * childsNum * (childsNum - 1), results.getResults().getNumFound());
|
||||
|
||||
//delete
|
||||
del("*:*");
|
||||
commit();
|
||||
|
||||
return docId;
|
||||
}
|
||||
|
||||
private long addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel, long docId) {
|
||||
SolrInputDocument childDocument = new SolrInputDocument();
|
||||
childDocument.addField("id", docId++);
|
||||
childDocument.addField("type_s", prefix);
|
||||
for (int index = 0; index < childIndex; ++index) {
|
||||
childDocument.addField(childIndex + prefix + index + "_s", childIndex + "value"+ index);
|
||||
}
|
||||
|
||||
if (!lastLevel) {
|
||||
for (int i = 0; i < childIndex * 2; ++i) {
|
||||
docId = addChildren("grand", childDocument, i, true, docId);
|
||||
}
|
||||
}
|
||||
topDocument.addChildDocument(childDocument);
|
||||
return docId;
|
||||
}
|
||||
|
||||
private void testIndexingWithSuss() throws Exception {
|
||||
ConcurrentUpdateSolrServer suss = new ConcurrentUpdateSolrServer(
|
||||
((HttpSolrServer) clients.get(0)).getBaseURL(), 3, 1);
|
||||
|
|
|
|||
|
|
@ -298,6 +298,13 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
|
|||
}
|
||||
}
|
||||
|
||||
public void testBlockJoin() throws Exception {
|
||||
assertQueryEquals("parent", "{!parent which=foo_s:parent}dude",
|
||||
"{!parent which=foo_s:parent}dude");
|
||||
assertQueryEquals("child", "{!child of=foo_s:parent}dude",
|
||||
"{!child of=foo_s:parent}dude");
|
||||
}
|
||||
|
||||
public void testQuerySurround() throws Exception {
|
||||
assertQueryEquals("surround", "{!surround}and(apache,solr)",
|
||||
"and(apache,solr)", "apache AND solr");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.search.join;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.search.QParser;
|
||||
import org.apache.solr.search.SolrCache;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
|
||||
public class BJQParserTest extends SolrTestCaseJ4 {
|
||||
|
||||
private static final String[] klm = new String[] {"k", "l", "m"};
|
||||
private static final List<String> xyz = Arrays.asList("x", "y", "z");
|
||||
private static final String[] abcdef = new String[] {"a", "b", "c", "d", "e", "f"};
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("solrconfig.xml", "schema15.xml");
|
||||
createIndex();
|
||||
}
|
||||
|
||||
public static void createIndex() throws IOException, Exception {
|
||||
int i = 0;
|
||||
List<List<String[]>> blocks = createBlocks();
|
||||
for (List<String[]> block : blocks) {
|
||||
for (String[] doc : block) {
|
||||
String[] idDoc = Arrays.copyOf(doc,doc.length+2);
|
||||
idDoc[doc.length]="id";
|
||||
idDoc[doc.length+1]=Integer.toString(i);
|
||||
assertU(add(doc(idDoc)));
|
||||
i++;
|
||||
}
|
||||
if (random().nextBoolean()) {
|
||||
assertU(commit());
|
||||
// force empty segment (actually, this will no longer create an empty segment, only a new segments_n)
|
||||
if (random().nextBoolean()) {
|
||||
assertU(commit());
|
||||
}
|
||||
}
|
||||
}
|
||||
assertU(commit());
|
||||
assertQ(req("q", "*:*"), "//*[@numFound='" + i + "']");
|
||||
/*
|
||||
* dump docs well System.out.println(h.query(req("q","*:*",
|
||||
* "sort","_docid_ asc", "fl",
|
||||
* "parent_s,child_s,parentchild_s,grand_s,grand_child_s,grand_parentchild_s"
|
||||
* , "wt","csv", "rows","1000"))); /
|
||||
*/
|
||||
}
|
||||
|
||||
private static int id=0;
|
||||
private static List<List<String[]>> createBlocks() {
|
||||
List<List<String[]>> blocks = new ArrayList<List<String[]>>();
|
||||
for (String parent : abcdef) {
|
||||
List<String[]> block = createChildrenBlock(parent);
|
||||
block.add(new String[] {"parent_s", parent});
|
||||
blocks.add(block);
|
||||
}
|
||||
Collections.shuffle(blocks, random());
|
||||
return blocks;
|
||||
}
|
||||
|
||||
private static List<String[]> createChildrenBlock(String parent) {
|
||||
List<String[]> block = new ArrayList<String[]>();
|
||||
for (String child : klm) {
|
||||
block
|
||||
.add(new String[] {"child_s", child, "parentchild_s", parent + child});
|
||||
}
|
||||
Collections.shuffle(block, random());
|
||||
addGrandChildren(block);
|
||||
return block;
|
||||
}
|
||||
|
||||
private static void addGrandChildren(List<String[]> block) {
|
||||
List<String> grandChildren = new ArrayList<String>(xyz);
|
||||
// add grandchildren after children
|
||||
for (ListIterator<String[]> iter = block.listIterator(); iter.hasNext();) {
|
||||
String[] child = iter.next();
|
||||
String child_s = child[1];
|
||||
String parentchild_s = child[3];
|
||||
int grandChildPos = 0;
|
||||
boolean lastLoopButStillHasGrCh = !iter.hasNext()
|
||||
&& !grandChildren.isEmpty();
|
||||
while (!grandChildren.isEmpty()
|
||||
&& ((grandChildPos = random().nextInt(grandChildren.size() * 2)) < grandChildren
|
||||
.size() || lastLoopButStillHasGrCh)) {
|
||||
grandChildPos = grandChildPos >= grandChildren.size() ? 0
|
||||
: grandChildPos;
|
||||
iter.add(new String[] {"grand_s", grandChildren.remove(grandChildPos),
|
||||
"grand_child_s", child_s, "grand_parentchild_s", parentchild_s});
|
||||
}
|
||||
}
|
||||
// and reverse after that
|
||||
Collections.reverse(block);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFull() throws IOException, Exception {
|
||||
String childb = "{!parent which=\"parent_s:[* TO *]\"}child_s:l";
|
||||
assertQ(req("q", childb), sixParents);
|
||||
}
|
||||
|
||||
private static final String sixParents[] = new String[] {
|
||||
"//*[@numFound='6']", "//doc/arr[@name=\"parent_s\"]/str='a'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='b'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='c'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='d'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='e'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='f'"};
|
||||
|
||||
@Test
|
||||
public void testJustParentsFilter() throws IOException {
|
||||
assertQ(req("q", "{!parent which=\"parent_s:[* TO *]\"}"), sixParents);
|
||||
}
|
||||
|
||||
private final static String beParents[] = new String[] {"//*[@numFound='2']",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='b'",
|
||||
"//doc/arr[@name=\"parent_s\"]/str='e'"};
|
||||
|
||||
@Test
|
||||
public void testIntersectBqBjq() {
|
||||
|
||||
assertQ(
|
||||
req("q", "+parent_s:(e b) +_query_:\"{!parent which=$pq v=$chq}\"",
|
||||
"chq", "child_s:l", "pq", "parent_s:[* TO *]"), beParents);
|
||||
assertQ(
|
||||
req("fq", "{!parent which=$pq v=$chq}\"", "q", "parent_s:(e b)", "chq",
|
||||
"child_s:l", "pq", "parent_s:[* TO *]"), beParents);
|
||||
|
||||
assertQ(
|
||||
req("q", "*:*", "fq", "{!parent which=$pq v=$chq}\"", "fq",
|
||||
"parent_s:(e b)", "chq", "child_s:l", "pq", "parent_s:[* TO *]"),
|
||||
beParents);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFq() {
|
||||
assertQ(
|
||||
req("q", "{!parent which=$pq v=$chq}", "fq", "parent_s:(e b)", "chq",
|
||||
"child_s:l", "pq", "parent_s:[* TO *]"// ,"debugQuery","on"
|
||||
), beParents);
|
||||
|
||||
boolean qfq = random().nextBoolean();
|
||||
assertQ(
|
||||
req(qfq ? "q" : "fq", "parent_s:(a e b)", (!qfq) ? "q" : "fq",
|
||||
"{!parent which=$pq v=$chq}", "chq", "parentchild_s:(bm ek cl)",
|
||||
"pq", "parent_s:[* TO *]"), beParents);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIntersectParentBqChildBq() throws IOException {
|
||||
|
||||
assertQ(
|
||||
req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"",
|
||||
"chq", "parentchild_s:(bm ek cl)", "pq", "parent_s:[* TO *]"),
|
||||
beParents);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGrandChildren() throws IOException {
|
||||
assertQ(
|
||||
req("q", "{!parent which=$parentfilter v=$children}", "children",
|
||||
"{!parent which=$childrenfilter v=$grandchildren}",
|
||||
"grandchildren", "grand_s:" + "x", "parentfilter",
|
||||
"parent_s:[* TO *]", "childrenfilter", "child_s:[* TO *]"),
|
||||
sixParents);
|
||||
// int loops = atLeast(1);
|
||||
String grandChildren = xyz.get(random().nextInt(xyz.size()));
|
||||
assertQ(
|
||||
req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"",
|
||||
"chq", "{!parent which=$childfilter v=$grandchq}", "grandchq",
|
||||
"+grand_s:" + grandChildren + " +grand_parentchild_s:(b* e* c*)",
|
||||
"pq", "parent_s:[* TO *]", "childfilter", "child_s:[* TO *]"),
|
||||
beParents);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testChildrenParser() {
|
||||
assertQ(
|
||||
req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:a", "fq",
|
||||
"NOT grand_s:[* TO *]"), "//*[@numFound='3']",
|
||||
"//doc/arr[@name=\"child_s\"]/str='k'",
|
||||
"//doc/arr[@name=\"child_s\"]/str='l'",
|
||||
"//doc/arr[@name=\"child_s\"]/str='m'");
|
||||
assertQ(
|
||||
req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:b", "fq",
|
||||
"-parentchild_s:bm", "fq", "-grand_s:*"), "//*[@numFound='2']",
|
||||
"//doc/arr[@name=\"child_s\"]/str='k'",
|
||||
"//doc/arr[@name=\"child_s\"]/str='l'");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCacheHit() throws IOException {
|
||||
|
||||
SolrCache parentFilterCache = (SolrCache) h.getCore().getInfoRegistry()
|
||||
.get("perSegFilter");
|
||||
|
||||
SolrCache filterCache = (SolrCache) h.getCore().getInfoRegistry()
|
||||
.get("filterCache");
|
||||
|
||||
NamedList parentsBefore = parentFilterCache.getStatistics();
|
||||
|
||||
NamedList filtersBefore = filterCache.getStatistics();
|
||||
|
||||
// it should be weird enough to be uniq
|
||||
String parentFilter = "parent_s:([a TO c] [d TO f])";
|
||||
|
||||
assertQ("search by parent filter",
|
||||
req("q", "{!parent which=\"" + parentFilter + "\"}"),
|
||||
"//*[@numFound='6']");
|
||||
|
||||
assertQ("filter by parent filter",
|
||||
req("q", "*:*", "fq", "{!parent which=\"" + parentFilter + "\"}"),
|
||||
"//*[@numFound='6']");
|
||||
|
||||
assertEquals("didn't hit fqCache yet ", 0L,
|
||||
delta("hits", filterCache.getStatistics(), filtersBefore));
|
||||
|
||||
assertQ(
|
||||
"filter by join",
|
||||
req("q", "*:*", "fq", "{!parent which=\"" + parentFilter
|
||||
+ "\"}child_s:l"), "//*[@numFound='6']");
|
||||
|
||||
assertEquals("in cache mode every request lookups", 3,
|
||||
delta("lookups", parentFilterCache.getStatistics(), parentsBefore));
|
||||
assertEquals("last two lookups causes hits", 2,
|
||||
delta("hits", parentFilterCache.getStatistics(), parentsBefore));
|
||||
assertEquals("the first lookup gets insert", 1,
|
||||
delta("inserts", parentFilterCache.getStatistics(), parentsBefore));
|
||||
|
||||
|
||||
assertEquals("true join query is cached in fqCache", 1L,
|
||||
delta("lookups", filterCache.getStatistics(), filtersBefore));
|
||||
}
|
||||
|
||||
private long delta(String key, NamedList a, NamedList b) {
|
||||
return (Long) a.get(key) - (Long) b.get(key);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void nullInit() {
|
||||
new BlockJoinParentQParserPlugin().init(null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,601 @@
|
|||
package org.apache.solr.update;
|
||||
|
||||
import org.apache.commons.io.output.ByteArrayOutputStream;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TermRangeFilter;
|
||||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.search.join.ScoreMode;
|
||||
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.request.RequestWriter;
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.util.JavaBinCodec;
|
||||
import org.apache.solr.handler.loader.XMLLoader;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.DefaultSolrThreadFactory;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
import org.dom4j.Document;
|
||||
import org.dom4j.DocumentHelper;
|
||||
import org.dom4j.Element;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import javax.xml.stream.XMLInputFactory;
|
||||
import javax.xml.stream.XMLStreamException;
|
||||
import javax.xml.stream.XMLStreamReader;
|
||||
|
||||
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import javax.xml.stream.XMLStreamException;
|
||||
import javax.xml.stream.XMLStreamReader;
|
||||
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with this
|
||||
* work for additional information regarding copyright ownership. The ASF
|
||||
* licenses this file to You under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
|
||||
public class AddBlockUpdateTest extends SolrTestCaseJ4 {
|
||||
|
||||
private static final String child = "child_s";
|
||||
private static final String parent = "parent_s";
|
||||
private static final String type = "type_s";
|
||||
|
||||
private static ExecutorService exe;
|
||||
private static AtomicInteger counter = new AtomicInteger();
|
||||
private static boolean cachedMode;
|
||||
|
||||
private static XMLInputFactory inputFactory;
|
||||
|
||||
private RefCounted<SolrIndexSearcher> searcherRef;
|
||||
private SolrIndexSearcher _searcher;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
String oldCacheNamePropValue = System
|
||||
.getProperty("blockJoinParentFilterCache");
|
||||
System.setProperty("blockJoinParentFilterCache", (cachedMode = random()
|
||||
.nextBoolean()) ? "blockJoinParentFilterCache" : "don't cache");
|
||||
if (oldCacheNamePropValue != null) {
|
||||
System.setProperty("blockJoinParentFilterCache", oldCacheNamePropValue);
|
||||
}
|
||||
inputFactory = XMLInputFactory.newInstance();
|
||||
|
||||
exe = // Executors.newSingleThreadExecutor();
|
||||
rarely() ? Executors.newFixedThreadPool(atLeast(2), new DefaultSolrThreadFactory("AddBlockUpdateTest")) : Executors
|
||||
.newCachedThreadPool(new DefaultSolrThreadFactory("AddBlockUpdateTest"));
|
||||
|
||||
|
||||
initCore("solrconfig.xml", "schema15.xml");
|
||||
}
|
||||
|
||||
@Before
|
||||
public void prepare() {
|
||||
// assertU("<rollback/>");
|
||||
assertU(delQ("*:*"));
|
||||
assertU(commit("expungeDeletes", "true"));
|
||||
|
||||
}
|
||||
|
||||
private SolrIndexSearcher getSearcher() {
|
||||
if (_searcher == null) {
|
||||
searcherRef = h.getCore().getSearcher();
|
||||
_searcher = searcherRef.get();
|
||||
}
|
||||
return _searcher;
|
||||
}
|
||||
|
||||
@After
|
||||
public void cleanup() {
|
||||
if (searcherRef != null || _searcher != null) {
|
||||
searcherRef.decref();
|
||||
searcherRef = null;
|
||||
_searcher = null;
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() throws Exception {
|
||||
inputFactory = null;
|
||||
exe.shutdownNow();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBasics() throws Exception {
|
||||
List<Document> blocks = new ArrayList<Document>(Arrays.asList(
|
||||
block("abcD"),
|
||||
block("efgH"),
|
||||
merge(block("ijkL"), block("mnoP")),
|
||||
merge(block("qrsT"), block("uvwX")),
|
||||
block("Y"),
|
||||
block("Z")));
|
||||
|
||||
Collections.shuffle(blocks);
|
||||
|
||||
log.trace("{}", blocks);
|
||||
|
||||
for (Future<Void> f : exe.invokeAll(callables(blocks))) {
|
||||
f.get(); // exceptions?
|
||||
}
|
||||
|
||||
assertU(commit());
|
||||
|
||||
final SolrIndexSearcher searcher = getSearcher();
|
||||
// final String resp = h.query(req("q","*:*", "sort","_docid_ asc", "rows",
|
||||
// "10000"));
|
||||
// log.trace(resp);
|
||||
int parentsNum = "DHLPTXYZ".length();
|
||||
assertQ(req(parent + ":[* TO *]"), "//*[@numFound='" + parentsNum + "']");
|
||||
assertQ(req(child + ":[* TO *]"), "//*[@numFound='"
|
||||
+ (('z' - 'a' + 1) - parentsNum) + "']");
|
||||
assertQ(req("*:*"), "//*[@numFound='" + ('z' - 'a' + 1) + "']");
|
||||
assertSingleParentOf(searcher, one("abc"), "D");
|
||||
assertSingleParentOf(searcher, one("efg"), "H");
|
||||
assertSingleParentOf(searcher, one("ijk"), "L");
|
||||
assertSingleParentOf(searcher, one("mno"), "P");
|
||||
assertSingleParentOf(searcher, one("qrs"), "T");
|
||||
assertSingleParentOf(searcher, one("uvw"), "X");
|
||||
}
|
||||
|
||||
/***
|
||||
@Test
|
||||
public void testSmallBlockDirect() throws Exception {
|
||||
final AddBlockUpdateCommand cmd = new AddBlockUpdateCommand(req("*:*"));
|
||||
final List<SolrInputDocument> docs = Arrays.asList(new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField(child, "a");
|
||||
}
|
||||
}, new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField(parent, "B");
|
||||
}
|
||||
});
|
||||
cmd.setDocs(docs);
|
||||
assertEquals(2, h.getCore().getUpdateHandler().addBlock(cmd));
|
||||
assertU(commit());
|
||||
|
||||
final SolrIndexSearcher searcher = getSearcher();
|
||||
assertQ(req("*:*"), "//*[@numFound='2']");
|
||||
assertSingleParentOf(searcher, one("a"), "B");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyDirect() throws Exception {
|
||||
final AddBlockUpdateCommand cmd = new AddBlockUpdateCommand(req("*:*"));
|
||||
// let's add empty one
|
||||
cmd.setDocs(Collections.<SolrInputDocument> emptyList());
|
||||
assertEquals(0,
|
||||
((DirectUpdateHandler2) h.getCore().getUpdateHandler()).addBlock(cmd));
|
||||
assertU(commit());
|
||||
|
||||
assertQ(req("*:*"), "//*[@numFound='0']");
|
||||
}
|
||||
***/
|
||||
|
||||
@Test
|
||||
public void testExceptionThrown() throws Exception {
|
||||
final String abcD = block("abcD").asXML();
|
||||
log.info(abcD);
|
||||
assertBlockU(abcD);
|
||||
|
||||
Document docToFail = DocumentHelper.createDocument();
|
||||
Element root = docToFail.addElement("add");
|
||||
Element doc1 = root.addElement("doc");
|
||||
attachField(doc1, "id", id());
|
||||
attachField(doc1, parent, "Y");
|
||||
attachField(doc1, "sample_i", "notanumber");
|
||||
Element subDoc1 = doc1.addElement("doc");
|
||||
attachField(subDoc1, "id", id());
|
||||
attachField(subDoc1, child, "x");
|
||||
Element doc2 = root.addElement("doc");
|
||||
attachField(doc2, "id", id());
|
||||
attachField(doc2, parent, "W");
|
||||
|
||||
assertFailedBlockU(docToFail.asXML());
|
||||
|
||||
assertBlockU(block("efgH").asXML());
|
||||
assertBlockU(commit());
|
||||
|
||||
final SolrIndexSearcher searcher = getSearcher();
|
||||
assertQ(req("q","*:*","indent","true", "fl","id,parent_s,child_s"), "//*[@numFound='" + "abcDefgH".length() + "']");
|
||||
assertSingleParentOf(searcher, one("abc"), "D");
|
||||
assertSingleParentOf(searcher, one("efg"), "H");
|
||||
|
||||
assertQ(req(child + ":x"), "//*[@numFound='0']");
|
||||
assertQ(req(parent + ":Y"), "//*[@numFound='0']");
|
||||
assertQ(req(parent + ":W"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@SuppressWarnings("serial")
|
||||
@Test
|
||||
public void testSolrJXML() throws IOException {
|
||||
UpdateRequest req = new UpdateRequest();
|
||||
|
||||
List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
|
||||
|
||||
SolrInputDocument document1 = new SolrInputDocument() {
|
||||
{
|
||||
final String id = id();
|
||||
addField("id", id);
|
||||
addField("parent_s", "X");
|
||||
|
||||
ArrayList<SolrInputDocument> ch1 = new ArrayList<SolrInputDocument>(
|
||||
Arrays.asList(new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField("child_s", "y");
|
||||
}
|
||||
}, new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField("child_s", "z");
|
||||
}
|
||||
}));
|
||||
|
||||
Collections.shuffle(ch1, random());
|
||||
addChildDocuments(ch1);
|
||||
}
|
||||
};
|
||||
|
||||
SolrInputDocument document2 = new SolrInputDocument() {
|
||||
{
|
||||
final String id = id();
|
||||
addField("id", id);
|
||||
addField("parent_s", "A");
|
||||
addChildDocument(new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField("child_s", "b");
|
||||
}
|
||||
});
|
||||
addChildDocument(new SolrInputDocument() {
|
||||
{
|
||||
addField("id", id());
|
||||
addField("child_s", "c");
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
docs.add(document1);
|
||||
docs.add(document2);
|
||||
|
||||
Collections.shuffle(docs, random());
|
||||
req.add(docs);
|
||||
|
||||
RequestWriter requestWriter = new RequestWriter();
|
||||
OutputStream os = new ByteArrayOutputStream();
|
||||
requestWriter.write(req, os);
|
||||
assertBlockU(os.toString());
|
||||
assertU(commit());
|
||||
|
||||
final SolrIndexSearcher searcher = getSearcher();
|
||||
assertSingleParentOf(searcher, one("yz"), "X");
|
||||
assertSingleParentOf(searcher, one("bc"), "A");
|
||||
}
|
||||
//This is the same as testSolrJXML above but uses the XMLLoader
|
||||
// to illustrate the structure of the XML documents
|
||||
@Test
|
||||
public void testXML() throws IOException, XMLStreamException {
|
||||
UpdateRequest req = new UpdateRequest();
|
||||
|
||||
List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
|
||||
|
||||
|
||||
String xml_doc1 =
|
||||
"<doc >" +
|
||||
" <field name=\"id\">1</field>" +
|
||||
" <field name=\"parent_s\">X</field>" +
|
||||
"<doc> " +
|
||||
" <field name=\"id\" >2</field>" +
|
||||
" <field name=\"child_s\">y</field>" +
|
||||
"</doc>"+
|
||||
"<doc> " +
|
||||
" <field name=\"id\" >3</field>" +
|
||||
" <field name=\"child_s\">z</field>" +
|
||||
"</doc>"+
|
||||
"</doc>";
|
||||
|
||||
String xml_doc2 =
|
||||
"<doc >" +
|
||||
" <field name=\"id\">4</field>" +
|
||||
" <field name=\"parent_s\">A</field>" +
|
||||
"<doc> " +
|
||||
" <field name=\"id\" >5</field>" +
|
||||
" <field name=\"child_s\">b</field>" +
|
||||
"</doc>"+
|
||||
"<doc> " +
|
||||
" <field name=\"id\" >6</field>" +
|
||||
" <field name=\"child_s\">c</field>" +
|
||||
"</doc>"+
|
||||
"</doc>";
|
||||
|
||||
|
||||
XMLStreamReader parser =
|
||||
inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) );
|
||||
parser.next(); // read the START document...
|
||||
//null for the processor is all right here
|
||||
XMLLoader loader = new XMLLoader();
|
||||
SolrInputDocument document1 = loader.readDoc( parser );
|
||||
|
||||
XMLStreamReader parser2 =
|
||||
inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) );
|
||||
parser2.next(); // read the START document...
|
||||
//null for the processor is all right here
|
||||
//XMLLoader loader = new XMLLoader();
|
||||
SolrInputDocument document2 = loader.readDoc( parser2 );
|
||||
|
||||
|
||||
docs.add(document1);
|
||||
docs.add(document2);
|
||||
|
||||
Collections.shuffle(docs, random());
|
||||
req.add(docs);
|
||||
|
||||
RequestWriter requestWriter = new RequestWriter();
|
||||
OutputStream os = new ByteArrayOutputStream();
|
||||
requestWriter.write(req, os);
|
||||
assertBlockU(os.toString());
|
||||
assertU(commit());
|
||||
|
||||
final SolrIndexSearcher searcher = getSearcher();
|
||||
assertSingleParentOf(searcher, one("yz"), "X");
|
||||
assertSingleParentOf(searcher, one("bc"), "A");
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testJavaBinCodec() throws IOException { //actually this test must be in other test class
|
||||
SolrInputDocument topDocument = new SolrInputDocument();
|
||||
topDocument.addField("parent_f1", "v1");
|
||||
topDocument.addField("parent_f2", "v2");
|
||||
|
||||
int childsNum = atLeast(10);
|
||||
for (int index = 0; index < childsNum; ++index) {
|
||||
addChildren("child", topDocument, index, false);
|
||||
}
|
||||
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
new JavaBinCodec().marshal(topDocument, os);
|
||||
byte[] buffer = os.toByteArray();
|
||||
//now read the Object back
|
||||
InputStream is = new ByteArrayInputStream(buffer);
|
||||
SolrInputDocument result = (SolrInputDocument) new JavaBinCodec().unmarshal(is);
|
||||
assertEquals(2, result.size());
|
||||
assertEquals("v1", result.getFieldValue("parent_f1"));
|
||||
assertEquals("v2", result.getFieldValue("parent_f2"));
|
||||
|
||||
List<SolrInputDocument> resultChilds = result.getChildDocuments();
|
||||
assertEquals(childsNum, resultChilds.size());
|
||||
|
||||
for (int childIndex = 0; childIndex < childsNum; ++childIndex) {
|
||||
SolrInputDocument child = resultChilds.get(childIndex);
|
||||
for (int fieldNum = 0; fieldNum < childIndex; ++fieldNum) {
|
||||
assertEquals(childIndex + "value" + fieldNum, child.getFieldValue(childIndex + "child" + fieldNum));
|
||||
}
|
||||
|
||||
List<SolrInputDocument> grandChilds = child.getChildDocuments();
|
||||
assertEquals(childIndex * 2, grandChilds.size());
|
||||
for (int grandIndex = 0; grandIndex < childIndex * 2; ++grandIndex) {
|
||||
SolrInputDocument grandChild = grandChilds.get(grandIndex);
|
||||
assertFalse(grandChild.hasChildDocuments());
|
||||
for (int fieldNum = 0; fieldNum < grandIndex; ++fieldNum) {
|
||||
assertEquals(grandIndex + "value" + fieldNum, grandChild.getFieldValue(grandIndex + "grand" + fieldNum));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void addChildren(String prefix, SolrInputDocument topDocument, int childIndex, boolean lastLevel) {
|
||||
SolrInputDocument childDocument = new SolrInputDocument();
|
||||
for (int index = 0; index < childIndex; ++index) {
|
||||
childDocument.addField(childIndex + prefix + index, childIndex + "value"+ index);
|
||||
}
|
||||
|
||||
if (!lastLevel) {
|
||||
for (int i = 0; i < childIndex * 2; ++i) {
|
||||
addChildren("grand", childDocument, i, true);
|
||||
}
|
||||
}
|
||||
topDocument.addChildDocument(childDocument);
|
||||
}
|
||||
|
||||
/**
|
||||
* on the given abcD it generates one parent doc, taking D from the tail and
|
||||
* two subdocs relaitons ab and c uniq ids are supplied also
|
||||
*
|
||||
* <pre>
|
||||
* {@code
|
||||
* <add>
|
||||
* <doc>
|
||||
* <field name="parent_s">D</field>
|
||||
* <doc>
|
||||
* <field name="child_s">a</field>
|
||||
* <field name="type_s">1</field>
|
||||
* </doc>
|
||||
* <doc>
|
||||
* <field name="child_s">b</field>
|
||||
* <field name="type_s">1</field>
|
||||
* </doc>
|
||||
* <doc>
|
||||
* <field name="child_s">c</field>
|
||||
* <field name="type_s">2</field>
|
||||
* </doc>
|
||||
* </doc>
|
||||
* </add>
|
||||
* }
|
||||
* </pre>
|
||||
* */
|
||||
private Document block(String string) {
|
||||
Document document = DocumentHelper.createDocument();
|
||||
Element root = document.addElement("add");
|
||||
Element doc = root.addElement("doc");
|
||||
|
||||
if (string.length() > 0) {
|
||||
// last character is a top parent
|
||||
attachField(doc, parent,
|
||||
String.valueOf(string.charAt(string.length() - 1)));
|
||||
attachField(doc, "id", id());
|
||||
|
||||
// add subdocs
|
||||
int type = 1;
|
||||
for (int i = 0; i < string.length() - 1; i += 2) {
|
||||
String relation = string.substring(i,
|
||||
Math.min(i + 2, string.length() - 1));
|
||||
attachSubDocs(doc, relation, type);
|
||||
type++;
|
||||
}
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
private void attachSubDocs(Element parent, String relation, int typeValue) {
|
||||
for (int j = 0; j < relation.length(); j++) {
|
||||
Element document = parent.addElement("doc");
|
||||
attachField(document, child, String.valueOf(relation.charAt(j)));
|
||||
attachField(document, "id", id());
|
||||
attachField(document, type, String.valueOf(typeValue));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges two documents like
|
||||
*
|
||||
* <pre>
|
||||
* {@code <add>...</add> + <add>...</add> = <add>... + ...</add>}
|
||||
* </pre>
|
||||
*
|
||||
* @param doc1
|
||||
* first document
|
||||
* @param doc2
|
||||
* second document
|
||||
* @return merged document
|
||||
*/
|
||||
private Document merge(Document doc1, Document doc2) {
|
||||
List<Element> list = doc2.getRootElement().elements();
|
||||
for (Element element : list) {
|
||||
doc1.getRootElement().add(element.detach());
|
||||
}
|
||||
|
||||
return doc1;
|
||||
}
|
||||
|
||||
private void attachField(Element root, String fieldName, String value) {
|
||||
Element field = root.addElement("field");
|
||||
field.addAttribute("name", fieldName);
|
||||
field.addText(value);
|
||||
}
|
||||
|
||||
private static String id() {
|
||||
return "" + counter.incrementAndGet();
|
||||
}
|
||||
|
||||
private String one(String string) {
|
||||
return "" + string.charAt(random().nextInt(string.length()));
|
||||
}
|
||||
|
||||
protected void assertSingleParentOf(final SolrIndexSearcher searcher,
|
||||
final String childTerm, String parentExp) throws IOException {
|
||||
final TopDocs docs = searcher.search(join(childTerm), 10);
|
||||
assertEquals(1, docs.totalHits);
|
||||
final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent);
|
||||
assertEquals(parentExp, pAct);
|
||||
}
|
||||
|
||||
protected ToParentBlockJoinQuery join(final String childTerm) {
|
||||
return new ToParentBlockJoinQuery(
|
||||
new TermQuery(new Term(child, childTerm)), new TermRangeFilter(parent,
|
||||
null, null, false, false), ScoreMode.None);
|
||||
}
|
||||
|
||||
private Collection<? extends Callable<Void>> callables(List<Document> blocks) {
|
||||
final List<Callable<Void>> rez = new ArrayList<Callable<Void>>();
|
||||
for (Document block : blocks) {
|
||||
final String msg = block.asXML();
|
||||
if (msg.length() > 0) {
|
||||
rez.add(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() {
|
||||
assertBlockU(msg);
|
||||
return null;
|
||||
}
|
||||
|
||||
});
|
||||
if (rarely()) {
|
||||
rez.add(new Callable<Void>() {
|
||||
@Override
|
||||
public Void call() {
|
||||
assertBlockU(commit());
|
||||
return null;
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return rez;
|
||||
}
|
||||
|
||||
private void assertBlockU(final String msg) {
|
||||
assertBlockU(msg, "0");
|
||||
}
|
||||
|
||||
private void assertFailedBlockU(final String msg) {
|
||||
try {
|
||||
assertBlockU(msg, "1");
|
||||
fail("expecting fail");
|
||||
} catch (Exception e) {
|
||||
// gotcha
|
||||
}
|
||||
}
|
||||
|
||||
private void assertBlockU(final String msg, String expected) {
|
||||
try {
|
||||
String res = h.checkUpdateStatus(msg, expected);
|
||||
if (res != null) {
|
||||
fail("update was not successful: " + res + " expected: " + expected);
|
||||
}
|
||||
} catch (SAXException e) {
|
||||
throw new RuntimeException("Invalid XML", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -133,6 +133,11 @@ public class ClientUtils
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (SolrInputDocument childDocument : doc.getChildDocuments()) {
|
||||
writeXML(childDocument, writer);
|
||||
}
|
||||
|
||||
writer.write("</doc>");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,9 +18,12 @@
|
|||
package org.apache.solr.common;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
|
|
@ -36,13 +39,16 @@ public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<S
|
|||
{
|
||||
private final Map<String,SolrInputField> _fields;
|
||||
private float _documentBoost = 1.0f;
|
||||
|
||||
private List<SolrInputDocument> _childDocuments;
|
||||
|
||||
public SolrInputDocument() {
|
||||
_fields = new LinkedHashMap<String,SolrInputField>();
|
||||
_childDocuments = new ArrayList<SolrInputDocument>();
|
||||
}
|
||||
|
||||
public SolrInputDocument(Map<String,SolrInputField> fields) {
|
||||
_fields = fields;
|
||||
_childDocuments = new ArrayList<SolrInputDocument>();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -52,7 +58,10 @@ public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<S
|
|||
public void clear()
|
||||
{
|
||||
if( _fields != null ) {
|
||||
_fields.clear();
|
||||
_fields.clear();
|
||||
}
|
||||
if (_childDocuments != null) {
|
||||
_childDocuments.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -189,7 +198,7 @@ public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<S
|
|||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "SolrInputDocument" + _fields.values();
|
||||
return "SolrInputDocument(fields: " + _fields.values() + ", childs: " + _childDocuments + ")";
|
||||
}
|
||||
|
||||
public SolrInputDocument deepCopy() {
|
||||
|
|
@ -199,6 +208,12 @@ public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<S
|
|||
clone._fields.put(fieldEntry.getKey(), fieldEntry.getValue().deepCopy());
|
||||
}
|
||||
clone._documentBoost = _documentBoost;
|
||||
|
||||
clone._childDocuments = new ArrayList<SolrInputDocument>(_childDocuments.size());
|
||||
for (SolrInputDocument child : _childDocuments) {
|
||||
clone._childDocuments.add(child.deepCopy());
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
|
|
@ -260,4 +275,23 @@ public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<S
|
|||
public Collection<SolrInputField> values() {
|
||||
return _fields.values();
|
||||
}
|
||||
|
||||
public void addChildDocument(SolrInputDocument child) {
|
||||
_childDocuments.add(child);
|
||||
}
|
||||
|
||||
public void addChildDocuments(Collection<SolrInputDocument> childs) {
|
||||
for (SolrInputDocument child : childs) {
|
||||
addChildDocument(child);
|
||||
}
|
||||
}
|
||||
|
||||
public List<SolrInputDocument> getChildDocuments() {
|
||||
return _childDocuments;
|
||||
}
|
||||
|
||||
public boolean hasChildDocuments() {
|
||||
boolean isEmpty = (_childDocuments == null || _childDocuments.isEmpty());
|
||||
return !isEmpty;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ public class JavaBinCodec {
|
|||
END = 15,
|
||||
|
||||
SOLRINPUTDOC = 16,
|
||||
SOLRINPUTDOC_CHILDS = 17,
|
||||
|
||||
// types that combine tag + length (or other info) in a single byte
|
||||
TAG_AND_LEN = (byte) (1 << 5),
|
||||
|
|
@ -358,6 +359,8 @@ public class JavaBinCodec {
|
|||
|
||||
public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException {
|
||||
int sz = readVInt(dis);
|
||||
dis.readByte(); // skip childDocuments tag
|
||||
int childsSize = readVInt(dis);
|
||||
float docBoost = (Float)readVal(dis);
|
||||
SolrInputDocument sdoc = new SolrInputDocument();
|
||||
sdoc.setDocumentBoost(docBoost);
|
||||
|
|
@ -374,11 +377,17 @@ public class JavaBinCodec {
|
|||
Object fieldVal = readVal(dis);
|
||||
sdoc.setField(fieldName, fieldVal, boost);
|
||||
}
|
||||
for (int i = 0; i < childsSize; i++) {
|
||||
dis.readByte(); // skip solrinputdoc tag
|
||||
SolrInputDocument child = readSolrInputDocument(dis);
|
||||
sdoc.addChildDocument(child);
|
||||
}
|
||||
return sdoc;
|
||||
}
|
||||
|
||||
public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException {
|
||||
writeTag(SOLRINPUTDOC, sdoc.size());
|
||||
writeTag(SOLRINPUTDOC_CHILDS, sdoc.getChildDocuments().size());
|
||||
writeFloat(sdoc.getDocumentBoost());
|
||||
for (SolrInputField inputField : sdoc.values()) {
|
||||
if (inputField.getBoost() != 1.0f) {
|
||||
|
|
@ -387,6 +396,9 @@ public class JavaBinCodec {
|
|||
writeExternString(inputField.getName());
|
||||
writeVal(inputField.getValue());
|
||||
}
|
||||
for (SolrInputDocument child : sdoc.getChildDocuments()) {
|
||||
writeSolrInputDocument(child);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ import java.io.File;
|
|||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
|
|
@ -969,6 +970,10 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
return sd;
|
||||
}
|
||||
|
||||
public static List<SolrInputDocument> sdocs(SolrInputDocument... docs) {
|
||||
return Arrays.asList(docs);
|
||||
}
|
||||
|
||||
/** Converts "test JSON" and returns standard JSON.
|
||||
* Currently this only consists of changing unescaped single quotes to double quotes,
|
||||
* and escaped single quotes to single quotes.
|
||||
|
|
@ -1527,7 +1532,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
}
|
||||
|
||||
/** Return a Map from field value to a list of document ids */
|
||||
Map<Comparable, List<Comparable>> invertField(Map<Comparable, Doc> model, String field) {
|
||||
public Map<Comparable, List<Comparable>> invertField(Map<Comparable, Doc> model, String field) {
|
||||
Map<Comparable, List<Comparable>> value_to_id = new HashMap<Comparable, List<Comparable>>();
|
||||
|
||||
// invert field
|
||||
|
|
|
|||
Loading…
Reference in New Issue