mirror of https://github.com/apache/lucene.git
SOLR-11598: Support more than 4 sort fields in the export writer
This commit is contained in:
Varun Thacker
parent
509561bf2a
commit
9d9c3a0cd8
|
|
@ -169,6 +169,10 @@ Optimizations
|
|||
* SOLR-11654: Time Routed Alias will now route documents to the ideal shard of a collection, thus avoiding a hop.
|
||||
Usually documents were already routed well but not always. (Gus Heck, David Smiley)
|
||||
|
||||
* SOLR-11598: The export handler does not limit users to 4 sort fields and is now unlimited. However the speed at
|
||||
which we can export is directly proportional to the number of sort fields specified. This change also allows streaming
|
||||
expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
|
||||
|
||||
Other Changes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.solr.common.params.CommonParams;
|
|||
import org.apache.solr.common.params.MapSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.handler.component.SearchHandler;
|
||||
import org.apache.solr.handler.export.ExportWriter;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
|
||||
class BoolFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
private FieldType fieldType;
|
||||
private CharsRefBuilder cref = new CharsRefBuilder();
|
||||
|
||||
public BoolFieldWriter(String field, FieldType fieldType) {
|
||||
this.field = field;
|
||||
this.fieldType = fieldType;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
SortedDocValues vals = DocValues.getSorted(reader, this.field);
|
||||
if (vals.advance(docId) != docId) {
|
||||
return false;
|
||||
}
|
||||
int ord = vals.ordValue();
|
||||
|
||||
BytesRef ref = vals.lookupOrd(ord);
|
||||
fieldType.indexedToReadable(ref, cref);
|
||||
ew.put(this.field, "true".equals(cref.toString()));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
class DateFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
||||
public DateFieldWriter(String field) {
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
|
||||
long val;
|
||||
if (vals.advance(docId) == docId) {
|
||||
val = vals.longValue();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
ew.put(this.field, new Date(val));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
|
||||
class DoubleValue implements SortValue {
|
||||
|
||||
protected NumericDocValues vals;
|
||||
protected String field;
|
||||
protected double currentValue;
|
||||
protected DoubleComp comp;
|
||||
private int lastDocID;
|
||||
private LeafReader reader;
|
||||
|
||||
public DoubleValue(String field, DoubleComp comp) {
|
||||
this.field = field;
|
||||
this.comp = comp;
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public DoubleValue copy() {
|
||||
return new DoubleValue(field, comp);
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
this.reader = context.reader();
|
||||
this.vals = DocValues.getNumeric(this.reader, this.field);
|
||||
lastDocID = 0;
|
||||
}
|
||||
|
||||
public void setCurrentValue(int docId) throws IOException {
|
||||
if (docId < lastDocID) {
|
||||
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
|
||||
}
|
||||
lastDocID = docId;
|
||||
int curDocID = vals.docID();
|
||||
if (docId > curDocID) {
|
||||
curDocID = vals.advance(docId);
|
||||
}
|
||||
if (docId == curDocID) {
|
||||
currentValue = Double.longBitsToDouble(vals.longValue());
|
||||
} else {
|
||||
currentValue = 0f;
|
||||
}
|
||||
}
|
||||
|
||||
public void setCurrentValue(SortValue sv) {
|
||||
DoubleValue dv = (DoubleValue)sv;
|
||||
this.currentValue = dv.currentValue;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public int compareTo(SortValue o) {
|
||||
DoubleValue dv = (DoubleValue)o;
|
||||
return comp.compare(currentValue, dv.currentValue);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
interface DoubleComp {
|
||||
int compare(double a, double b);
|
||||
double resetValue();
|
||||
}
|
||||
|
||||
class DoubleAsc implements DoubleComp {
|
||||
public double resetValue() {
|
||||
return Double.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(double a, double b) {
|
||||
return Double.compare(b, a);
|
||||
}
|
||||
}
|
||||
|
||||
class DoubleDesc implements DoubleComp {
|
||||
public double resetValue() {
|
||||
return -Double.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(double a, double b) {
|
||||
return Double.compare(a, b);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
class DoubleFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
||||
public DoubleFieldWriter(String field) {
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
|
||||
long val;
|
||||
if (vals.advance(docId) == docId) {
|
||||
val = vals.longValue();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
ew.put(this.field, Double.longBitsToDouble(val));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,448 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.PrintWriter;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Sort;
|
||||
import org.apache.lucene.search.SortField;
|
||||
import org.apache.lucene.util.BitSetIterator;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
|
||||
import org.apache.solr.common.IteratorWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.common.MapWriter.EntryWriter;
|
||||
import org.apache.solr.common.PushWriter;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.JavaBinCodec;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
import org.apache.solr.response.BinaryResponseWriter;
|
||||
import org.apache.solr.response.JSONResponseWriter;
|
||||
import org.apache.solr.response.QueryResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.BoolField;
|
||||
import org.apache.solr.schema.DateValueFieldType;
|
||||
import org.apache.solr.schema.DoubleValueFieldType;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.FloatValueFieldType;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.IntValueFieldType;
|
||||
import org.apache.solr.schema.LongValueFieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.apache.solr.common.util.Utils.makeMap;
|
||||
|
||||
public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
||||
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private OutputStreamWriter respWriter;
|
||||
final SolrQueryRequest req;
|
||||
final SolrQueryResponse res;
|
||||
FieldWriter[] fieldWriters;
|
||||
int totalHits = 0;
|
||||
FixedBitSet[] sets = null;
|
||||
PushWriter writer;
|
||||
private String wt;
|
||||
|
||||
|
||||
public ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
|
||||
this.req = req;
|
||||
this.res = res;
|
||||
this.wt = wt;
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
if ("javabin".equals(wt)) {
|
||||
return BinaryResponseParser.BINARY_CONTENT_TYPE;
|
||||
} else return "json";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
if (writer != null) writer.close();
|
||||
if (respWriter != null) {
|
||||
respWriter.flush();
|
||||
respWriter.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
|
||||
w.writeMap(mw -> {
|
||||
mw.put("responseHeader", singletonMap("status", 400))
|
||||
.put("response", makeMap(
|
||||
"numFound", 0,
|
||||
"docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
|
||||
});
|
||||
if (log) {
|
||||
SolrException.log(logger, e);
|
||||
}
|
||||
}
|
||||
|
||||
public void write(OutputStream os) throws IOException {
|
||||
QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
|
||||
if (rw instanceof BinaryResponseWriter) {
|
||||
//todo add support for other writers after testing
|
||||
writer = new JavaBinCodec(os, null);
|
||||
} else {
|
||||
respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
|
||||
writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
|
||||
}
|
||||
Exception exception = res.getException();
|
||||
if (exception != null) {
|
||||
if (!(exception instanceof IgnoreException)) {
|
||||
writeException(exception, writer, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
|
||||
SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
|
||||
|
||||
if (sortSpec == null) {
|
||||
writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
|
||||
return;
|
||||
}
|
||||
|
||||
SolrIndexSearcher searcher = req.getSearcher();
|
||||
Sort sort = searcher.weightSort(sortSpec.getSort());
|
||||
|
||||
if (sort == null) {
|
||||
writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
|
||||
return;
|
||||
}
|
||||
|
||||
if (sort != null && sort.needsScores()) {
|
||||
writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
|
||||
return;
|
||||
}
|
||||
|
||||
// There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
|
||||
// if (lastDocRequested <= 0) {
|
||||
// That causes the totalHits and export entries in the context to _not_ get set.
|
||||
// The only time that really matters is when we search against an _empty_ set. That's too obscure
|
||||
// a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
|
||||
// ant test -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
|
||||
// You'll have to uncomment the if below to hit the null pointer exception.
|
||||
// This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
|
||||
// This came to light in the very artifical case of indexing a single doc to Cloud.
|
||||
if (req.getContext().get("totalHits") != null) {
|
||||
totalHits = ((Integer) req.getContext().get("totalHits")).intValue();
|
||||
sets = (FixedBitSet[]) req.getContext().get("export");
|
||||
if (sets == null) {
|
||||
writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
SolrParams params = req.getParams();
|
||||
String fl = params.get("fl");
|
||||
|
||||
String[] fields = null;
|
||||
|
||||
if (fl == null) {
|
||||
writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
|
||||
return;
|
||||
} else {
|
||||
fields = fl.split(",");
|
||||
|
||||
for (int i = 0; i < fields.length; i++) {
|
||||
|
||||
fields[i] = fields[i].trim();
|
||||
|
||||
if (fields[i].equals("score")) {
|
||||
writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
fieldWriters = getFieldWriters(fields, req.getSearcher());
|
||||
} catch (Exception e) {
|
||||
writeException(e, writer, true);
|
||||
return;
|
||||
}
|
||||
|
||||
writer.writeMap(m -> {
|
||||
m.put("responseHeader", singletonMap("status", 0));
|
||||
m.put("response", (MapWriter) mw -> {
|
||||
mw.put("numFound", totalHits);
|
||||
mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
|
||||
});
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
|
||||
//Write the data.
|
||||
List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
|
||||
SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
|
||||
int count = 0;
|
||||
int queueSize = 30000;
|
||||
if (totalHits < 30000) {
|
||||
queueSize = totalHits;
|
||||
}
|
||||
SortQueue queue = new SortQueue(queueSize, sortDoc);
|
||||
SortDoc[] outDocs = new SortDoc[queueSize];
|
||||
|
||||
while (count < totalHits) {
|
||||
//long begin = System.nanoTime();
|
||||
queue.reset();
|
||||
SortDoc top = queue.top();
|
||||
for (int i = 0; i < leaves.size(); i++) {
|
||||
sortDoc.setNextReader(leaves.get(i));
|
||||
DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
|
||||
int docId;
|
||||
while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
|
||||
sortDoc.setValues(docId);
|
||||
if (top.lessThan(sortDoc)) {
|
||||
top.setValues(sortDoc);
|
||||
top = queue.updateTop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int outDocsIndex = -1;
|
||||
|
||||
for (int i = 0; i < queueSize; i++) {
|
||||
SortDoc s = queue.pop();
|
||||
if (s.docId > -1) {
|
||||
outDocs[++outDocsIndex] = s;
|
||||
}
|
||||
}
|
||||
|
||||
//long end = System.nanoTime();
|
||||
|
||||
count += (outDocsIndex + 1);
|
||||
|
||||
try {
|
||||
for (int i = outDocsIndex; i >= 0; --i) {
|
||||
SortDoc s = outDocs[i];
|
||||
writer.add((MapWriter) ew -> {
|
||||
writeDoc(s, leaves, ew);
|
||||
s.reset();
|
||||
});
|
||||
}
|
||||
} catch (Throwable e) {
|
||||
Throwable ex = e;
|
||||
while (ex != null) {
|
||||
String m = ex.getMessage();
|
||||
if (m != null && m.contains("Broken pipe")) {
|
||||
throw new IgnoreException();
|
||||
}
|
||||
ex = ex.getCause();
|
||||
}
|
||||
|
||||
if (e instanceof IOException) {
|
||||
throw ((IOException) e);
|
||||
} else {
|
||||
throw new IOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void writeDoc(SortDoc sortDoc,
|
||||
List<LeafReaderContext> leaves,
|
||||
EntryWriter ew) throws IOException {
|
||||
|
||||
int ord = sortDoc.ord;
|
||||
FixedBitSet set = sets[ord];
|
||||
set.clear(sortDoc.docId);
|
||||
LeafReaderContext context = leaves.get(ord);
|
||||
int fieldIndex = 0;
|
||||
for (FieldWriter fieldWriter : fieldWriters) {
|
||||
if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
|
||||
++fieldIndex;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
|
||||
IndexSchema schema = searcher.getSchema();
|
||||
FieldWriter[] writers = new FieldWriter[fields.length];
|
||||
for (int i = 0; i < fields.length; i++) {
|
||||
String field = fields[i];
|
||||
SchemaField schemaField = null;
|
||||
|
||||
try {
|
||||
schemaField = schema.getField(field);
|
||||
} catch (Exception e) {
|
||||
throw new IOException(e);
|
||||
}
|
||||
|
||||
if (!schemaField.hasDocValues()) {
|
||||
throw new IOException(field + " must have DocValues to use this feature.");
|
||||
}
|
||||
|
||||
boolean multiValued = schemaField.multiValued();
|
||||
FieldType fieldType = schemaField.getType();
|
||||
if (fieldType instanceof IntValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new IntFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof LongValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new LongFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof FloatValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new FloatFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof DoubleValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new DoubleFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof StrField) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
|
||||
} else {
|
||||
writers[i] = new StringFieldWriter(field, fieldType);
|
||||
}
|
||||
} else if (fieldType instanceof DateValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
|
||||
} else {
|
||||
writers[i] = new DateFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof BoolField) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new BoolFieldWriter(field, fieldType);
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
|
||||
}
|
||||
}
|
||||
return writers;
|
||||
}
|
||||
|
||||
private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
|
||||
SortValue[] sortValues = new SortValue[sortFields.length];
|
||||
IndexSchema schema = searcher.getSchema();
|
||||
for (int i = 0; i < sortFields.length; ++i) {
|
||||
SortField sf = sortFields[i];
|
||||
String field = sf.getField();
|
||||
boolean reverse = sf.getReverse();
|
||||
SchemaField schemaField = schema.getField(field);
|
||||
FieldType ft = schemaField.getType();
|
||||
|
||||
if (!schemaField.hasDocValues()) {
|
||||
throw new IOException(field + " must have DocValues to use this feature.");
|
||||
}
|
||||
|
||||
if (ft instanceof IntValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new IntValue(field, new IntDesc());
|
||||
} else {
|
||||
sortValues[i] = new IntValue(field, new IntAsc());
|
||||
}
|
||||
} else if (ft instanceof FloatValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new FloatValue(field, new FloatDesc());
|
||||
} else {
|
||||
sortValues[i] = new FloatValue(field, new FloatAsc());
|
||||
}
|
||||
} else if (ft instanceof DoubleValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new DoubleValue(field, new DoubleDesc());
|
||||
} else {
|
||||
sortValues[i] = new DoubleValue(field, new DoubleAsc());
|
||||
}
|
||||
} else if (ft instanceof LongValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new LongValue(field, new LongDesc());
|
||||
} else {
|
||||
sortValues[i] = new LongValue(field, new LongAsc());
|
||||
}
|
||||
} else if (ft instanceof StrField) {
|
||||
LeafReader reader = searcher.getSlowAtomicReader();
|
||||
SortedDocValues vals = reader.getSortedDocValues(field);
|
||||
if (reverse) {
|
||||
sortValues[i] = new StringValue(vals, field, new IntDesc());
|
||||
} else {
|
||||
sortValues[i] = new StringValue(vals, field, new IntAsc());
|
||||
}
|
||||
} else if (ft instanceof DateValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new LongValue(field, new LongDesc());
|
||||
} else {
|
||||
sortValues[i] = new LongValue(field, new LongAsc());
|
||||
}
|
||||
} else if (ft instanceof BoolField) {
|
||||
// This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
|
||||
// _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
|
||||
// we can just use the existing StringValue here.
|
||||
LeafReader reader = searcher.getSlowAtomicReader();
|
||||
SortedDocValues vals = reader.getSortedDocValues(field);
|
||||
if (reverse) {
|
||||
sortValues[i] = new StringValue(vals, field, new IntDesc());
|
||||
} else {
|
||||
sortValues[i] = new StringValue(vals, field, new IntAsc());
|
||||
}
|
||||
} else {
|
||||
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
|
||||
}
|
||||
}
|
||||
|
||||
return new SortDoc(sortValues);
|
||||
}
|
||||
|
||||
public static class IgnoreException extends IOException {
|
||||
public void printStackTrace(PrintWriter pw) {
|
||||
pw.print("Early Client Disconnect");
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return "Early Client Disconnect";
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
abstract class FieldWriter {
|
||||
public abstract boolean write(int docId, LeafReader reader, MapWriter.EntryWriter out, int fieldIndex) throws IOException;
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
interface FloatComp {
|
||||
int compare(float a, float b);
|
||||
float resetValue();
|
||||
}
|
||||
|
||||
class FloatAsc implements FloatComp {
|
||||
public float resetValue() {
|
||||
return Float.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(float a, float b) {
|
||||
return Float.compare(b, a);
|
||||
}
|
||||
}
|
||||
|
||||
class FloatDesc implements FloatComp {
|
||||
public float resetValue() {
|
||||
return -Float.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(float a, float b) {
|
||||
return Float.compare(a, b);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
class FloatFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
||||
public FloatFieldWriter(String field) {
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
|
||||
int val;
|
||||
if (vals.advance(docId) == docId) {
|
||||
val = (int)vals.longValue();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
ew.put(this.field, Float.intBitsToFloat(val));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
|
||||
class FloatValue implements SortValue {
|
||||
|
||||
protected NumericDocValues vals;
|
||||
protected String field;
|
||||
protected float currentValue;
|
||||
protected FloatComp comp;
|
||||
private int lastDocID;
|
||||
|
||||
public FloatValue(String field, FloatComp comp) {
|
||||
this.field = field;
|
||||
this.comp = comp;
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public FloatValue copy() {
|
||||
return new FloatValue(field, comp);
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
this.vals = DocValues.getNumeric(context.reader(), field);
|
||||
lastDocID = 0;
|
||||
}
|
||||
|
||||
public void setCurrentValue(int docId) throws IOException {
|
||||
if (docId < lastDocID) {
|
||||
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
|
||||
}
|
||||
lastDocID = docId;
|
||||
int curDocID = vals.docID();
|
||||
if (docId > curDocID) {
|
||||
curDocID = vals.advance(docId);
|
||||
}
|
||||
if (docId == curDocID) {
|
||||
currentValue = Float.intBitsToFloat((int)vals.longValue());
|
||||
} else {
|
||||
currentValue = 0f;
|
||||
}
|
||||
}
|
||||
|
||||
public void setCurrentValue(SortValue sv) {
|
||||
FloatValue fv = (FloatValue)sv;
|
||||
this.currentValue = fv.currentValue;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public int compareTo(SortValue o) {
|
||||
FloatValue fv = (FloatValue)o;
|
||||
return comp.compare(currentValue, fv.currentValue);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
public interface IntComp {
|
||||
int compare(int a, int b);
|
||||
int resetValue();
|
||||
}
|
||||
|
||||
class IntAsc implements IntComp {
|
||||
|
||||
public int resetValue() {
|
||||
return Integer.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(int a, int b) {
|
||||
return Integer.compare(b, a);
|
||||
}
|
||||
}
|
||||
|
||||
class IntDesc implements IntComp {
|
||||
|
||||
public int resetValue() {
|
||||
return Integer.MIN_VALUE;
|
||||
}
|
||||
|
||||
public int compare(int a, int b) {
|
||||
return Integer.compare(a, b);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
class IntFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
||||
public IntFieldWriter(String field) {
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
|
||||
int val;
|
||||
if (vals.advance(docId) == docId) {
|
||||
val = (int) vals.longValue();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
ew.put(this.field, val);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
|
||||
public class IntValue implements SortValue {
|
||||
|
||||
protected NumericDocValues vals;
|
||||
protected String field;
|
||||
protected int currentValue;
|
||||
protected IntComp comp;
|
||||
private int lastDocID;
|
||||
|
||||
public IntValue copy() {
|
||||
return new IntValue(field, comp);
|
||||
}
|
||||
|
||||
public IntValue(String field, IntComp comp) {
|
||||
this.field = field;
|
||||
this.comp = comp;
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
this.vals = DocValues.getNumeric(context.reader(), field);
|
||||
lastDocID = 0;
|
||||
}
|
||||
|
||||
public void setCurrentValue(int docId) throws IOException {
|
||||
if (docId < lastDocID) {
|
||||
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
|
||||
}
|
||||
lastDocID = docId;
|
||||
int curDocID = vals.docID();
|
||||
if (docId > curDocID) {
|
||||
curDocID = vals.advance(docId);
|
||||
}
|
||||
if (docId == curDocID) {
|
||||
currentValue = (int) vals.longValue();
|
||||
} else {
|
||||
currentValue = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public int compareTo(SortValue o) {
|
||||
IntValue iv = (IntValue)o;
|
||||
return comp.compare(currentValue, iv.currentValue);
|
||||
}
|
||||
|
||||
public void setCurrentValue (SortValue value) {
|
||||
currentValue = ((IntValue)value).currentValue;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
currentValue = comp.resetValue();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
interface LongComp {
|
||||
int compare(long a, long b);
|
||||
long resetValue();
|
||||
}
|
||||
|
||||
class LongAsc implements LongComp {
|
||||
|
||||
public long resetValue() {
|
||||
return Long.MAX_VALUE;
|
||||
}
|
||||
|
||||
public int compare(long a, long b) {
|
||||
return Long.compare(b, a);
|
||||
}
|
||||
}
|
||||
|
||||
class LongDesc implements LongComp {
|
||||
|
||||
public long resetValue() {
|
||||
return Long.MIN_VALUE;
|
||||
}
|
||||
|
||||
public int compare(long a, long b) {
|
||||
return Long.compare(a, b);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
||||
class LongFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
||||
public LongFieldWriter(String field) {
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
|
||||
long val;
|
||||
if (vals.advance(docId) == docId) {
|
||||
val = vals.longValue();
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
ew.put(field, val);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
|
||||
public class LongValue implements SortValue {
|
||||
|
||||
protected NumericDocValues vals;
|
||||
protected String field;
|
||||
protected long currentValue;
|
||||
protected LongComp comp;
|
||||
private int lastDocID;
|
||||
|
||||
public LongValue(String field, LongComp comp) {
|
||||
this.field = field;
|
||||
this.comp = comp;
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
|
||||
public LongValue copy() {
|
||||
return new LongValue(field, comp);
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
this.vals = DocValues.getNumeric(context.reader(), field);
|
||||
lastDocID = 0;
|
||||
}
|
||||
|
||||
public void setCurrentValue(int docId) throws IOException {
|
||||
if (docId < lastDocID) {
|
||||
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
|
||||
}
|
||||
lastDocID = docId;
|
||||
int curDocID = vals.docID();
|
||||
if (docId > curDocID) {
|
||||
curDocID = vals.advance(docId);
|
||||
}
|
||||
if (docId == curDocID) {
|
||||
currentValue = vals.longValue();
|
||||
} else {
|
||||
currentValue = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public void setCurrentValue(SortValue sv) {
|
||||
LongValue lv = (LongValue)sv;
|
||||
this.currentValue = lv.currentValue;
|
||||
}
|
||||
|
||||
public int compareTo(SortValue o) {
|
||||
LongValue l = (LongValue)o;
|
||||
return comp.compare(currentValue, l.currentValue);
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
this.currentValue = comp.resetValue();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Date;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.SortedNumericDocValues;
|
||||
import org.apache.lucene.index.SortedSetDocValues;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.apache.solr.common.IteratorWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
|
||||
class MultiFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
private FieldType fieldType;
|
||||
private SchemaField schemaField;
|
||||
private boolean numeric;
|
||||
private CharsRefBuilder cref = new CharsRefBuilder();
|
||||
private final LongFunction<Object> bitsToValue;
|
||||
|
||||
public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
|
||||
this.field = field;
|
||||
this.fieldType = fieldType;
|
||||
this.schemaField = schemaField;
|
||||
this.numeric = numeric;
|
||||
if (this.fieldType.isPointField()) {
|
||||
bitsToValue = bitsToValue(fieldType);
|
||||
} else {
|
||||
bitsToValue = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter out, int fieldIndex) throws IOException {
|
||||
if (this.fieldType.isPointField()) {
|
||||
SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
|
||||
if (!vals.advanceExact(docId)) return false;
|
||||
out.put(this.field,
|
||||
(IteratorWriter) w -> {
|
||||
for (int i = 0; i < vals.docValueCount(); i++) {
|
||||
w.add(bitsToValue.apply(vals.nextValue()));
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} else {
|
||||
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
|
||||
if (vals.advance(docId) != docId) return false;
|
||||
out.put(this.field,
|
||||
(IteratorWriter) w -> {
|
||||
long o;
|
||||
while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
|
||||
BytesRef ref = vals.lookupOrd(o);
|
||||
fieldType.indexedToReadable(ref, cref);
|
||||
IndexableField f = fieldType.createField(schemaField, cref.toString());
|
||||
if (f == null) w.add(cref.toString());
|
||||
else w.add(fieldType.toObject(f));
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
static LongFunction<Object> bitsToValue(FieldType fieldType) {
|
||||
switch (fieldType.getNumberType()) {
|
||||
case LONG:
|
||||
return (bits)-> bits;
|
||||
case DATE:
|
||||
return (bits)-> new Date(bits);
|
||||
case INTEGER:
|
||||
return (bits)-> (int)bits;
|
||||
case FLOAT:
|
||||
return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
|
||||
case DOUBLE:
|
||||
return (bits)-> NumericUtils.sortableLongToDouble(bits);
|
||||
default:
|
||||
throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
|
||||
public abstract class PriorityQueue<T> {
|
||||
protected int size = 0;
|
||||
protected final int maxSize;
|
||||
private final T[] heap;
|
||||
|
||||
public PriorityQueue(int maxSize) {
|
||||
this(maxSize, true);
|
||||
}
|
||||
|
||||
public PriorityQueue(int maxSize, boolean prepopulate) {
|
||||
final int heapSize;
|
||||
if (0 == maxSize) {
|
||||
// We allocate 1 extra to avoid if statement in top()
|
||||
heapSize = 2;
|
||||
} else {
|
||||
if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
|
||||
// Don't wrap heapSize to -1, in this case, which
|
||||
// causes a confusing NegativeArraySizeException.
|
||||
// Note that very likely this will simply then hit
|
||||
// an OOME, but at least that's more indicative to
|
||||
// caller that this values is too big. We don't +1
|
||||
// in this case, but it's very unlikely in practice
|
||||
// one will actually insert this many objects into
|
||||
// the PQ:
|
||||
// Throw exception to prevent confusing OOME:
|
||||
throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
|
||||
} else {
|
||||
// NOTE: we add +1 because all access to heap is
|
||||
// 1-based not 0-based. heap[0] is unused.
|
||||
heapSize = maxSize + 1;
|
||||
}
|
||||
}
|
||||
// T is unbounded type, so this unchecked cast works always:
|
||||
@SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
|
||||
this.heap = h;
|
||||
this.maxSize = maxSize;
|
||||
|
||||
if (prepopulate) {
|
||||
// If sentinel objects are supported, populate the queue with them
|
||||
T sentinel = getSentinelObject();
|
||||
if (sentinel != null) {
|
||||
heap[1] = sentinel;
|
||||
for (int i = 2; i < heap.length; i++) {
|
||||
heap[i] = getSentinelObject();
|
||||
}
|
||||
size = maxSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Determines the ordering of objects in this priority queue. Subclasses
|
||||
* must define this one method.
|
||||
* @return <code>true</code> iff parameter <tt>a</tt> is less than parameter <tt>b</tt>.
|
||||
*/
|
||||
protected abstract boolean lessThan(T a, T b);
|
||||
|
||||
|
||||
protected T getSentinelObject() {
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an Object to a PriorityQueue in log(size) time. If one tries to add
|
||||
* more objects than maxSize from initialize an
|
||||
*
|
||||
* @return the new 'top' element in the queue.
|
||||
*/
|
||||
public final T add(T element) {
|
||||
size++;
|
||||
heap[size] = element;
|
||||
upHeap();
|
||||
return heap[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds an Object to a PriorityQueue in log(size) time.
|
||||
* It returns the object (if any) that was
|
||||
* dropped off the heap because it was full. This can be
|
||||
* the given parameter (in case it is smaller than the
|
||||
* full heap's minimum, and couldn't be added), or another
|
||||
* object that was previously the smallest value in the
|
||||
* heap and now has been replaced by a larger one, or null
|
||||
* if the queue wasn't yet full with maxSize elements.
|
||||
*/
|
||||
public T insertWithOverflow(T element) {
|
||||
if (size < maxSize) {
|
||||
add(element);
|
||||
return null;
|
||||
} else if (size > 0 && !lessThan(element, heap[1])) {
|
||||
T ret = heap[1];
|
||||
heap[1] = element;
|
||||
updateTop();
|
||||
return ret;
|
||||
} else {
|
||||
return element;
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns the least element of the PriorityQueue in constant time. */
|
||||
public final T top() {
|
||||
// We don't need to check size here: if maxSize is 0,
|
||||
// then heap is length 2 array with both entries null.
|
||||
// If size is 0 then heap[1] is already null.
|
||||
return heap[1];
|
||||
}
|
||||
|
||||
/** Removes and returns the least element of the PriorityQueue in log(size)
|
||||
time. */
|
||||
public final T pop() {
|
||||
if (size > 0) {
|
||||
T result = heap[1]; // save first value
|
||||
heap[1] = heap[size]; // move last to first
|
||||
heap[size] = null; // permit GC of objects
|
||||
size--;
|
||||
downHeap(); // adjust heap
|
||||
return result;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Should be called when the Object at top changes values. Still log(n) worst
|
||||
* case, but it's at least twice as fast to
|
||||
*
|
||||
* <pre class="prettyprint">
|
||||
* pq.top().change();
|
||||
* pq.updateTop();
|
||||
* </pre>
|
||||
*
|
||||
* instead of
|
||||
*
|
||||
* <pre class="prettyprint">
|
||||
* o = pq.pop();
|
||||
* o.change();
|
||||
* pq.push(o);
|
||||
* </pre>
|
||||
*
|
||||
* @return the new 'top' element.
|
||||
*/
|
||||
public final T updateTop() {
|
||||
downHeap();
|
||||
return heap[1];
|
||||
}
|
||||
|
||||
/** Returns the number of elements currently stored in the PriorityQueue. */
|
||||
public final int size() {
|
||||
return size;
|
||||
}
|
||||
|
||||
/** Removes all entries from the PriorityQueue. */
|
||||
public final void clear() {
|
||||
for (int i = 0; i <= size; i++) {
|
||||
heap[i] = null;
|
||||
}
|
||||
size = 0;
|
||||
}
|
||||
|
||||
private final void upHeap() {
|
||||
int i = size;
|
||||
T node = heap[i]; // save bottom node
|
||||
int j = i >>> 1;
|
||||
while (j > 0 && lessThan(node, heap[j])) {
|
||||
heap[i] = heap[j]; // shift parents down
|
||||
i = j;
|
||||
j = j >>> 1;
|
||||
}
|
||||
heap[i] = node; // install saved node
|
||||
}
|
||||
|
||||
private final void downHeap() {
|
||||
int i = 1;
|
||||
T node = heap[i]; // save top node
|
||||
int j = i << 1; // find smaller child
|
||||
int k = j + 1;
|
||||
if (k <= size && lessThan(heap[k], heap[j])) {
|
||||
j = k;
|
||||
}
|
||||
while (j <= size && lessThan(heap[j], node)) {
|
||||
heap[i] = heap[j]; // shift up child
|
||||
i = j;
|
||||
j = i << 1;
|
||||
k = j + 1;
|
||||
if (k <= size && lessThan(heap[k], heap[j])) {
|
||||
j = k;
|
||||
}
|
||||
}
|
||||
heap[i] = node; // install saved node
|
||||
}
|
||||
|
||||
/** This method returns the internal heap array as Object[].
|
||||
* @lucene.internal
|
||||
*/
|
||||
public final Object[] getHeapArray() {
|
||||
return (Object[]) heap;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
|
||||
class SortDoc {
|
||||
|
||||
protected int docId = -1;
|
||||
protected int ord = -1;
|
||||
protected int docBase = -1;
|
||||
|
||||
private SortValue[] sortValues;
|
||||
|
||||
public SortDoc(SortValue[] sortValues) {
|
||||
this.sortValues = sortValues;
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
this.ord = context.ord;
|
||||
this.docBase = context.docBase;
|
||||
for (SortValue value : sortValues) {
|
||||
value.setNextReader(context);
|
||||
}
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
this.docId = -1;
|
||||
for (SortValue value : sortValues) {
|
||||
value.reset();
|
||||
}
|
||||
}
|
||||
|
||||
public void setValues(int docId) throws IOException {
|
||||
this.docId = docId;
|
||||
for(SortValue sortValue : sortValues) {
|
||||
sortValue.setCurrentValue(docId);
|
||||
}
|
||||
}
|
||||
|
||||
public void setValues(SortDoc sortDoc) {
|
||||
this.docId = sortDoc.docId;
|
||||
this.ord = sortDoc.ord;
|
||||
this.docBase = sortDoc.docBase;
|
||||
SortValue[] vals = sortDoc.sortValues;
|
||||
for(int i=0; i<vals.length; i++) {
|
||||
sortValues[i].setCurrentValue(vals[i]);
|
||||
}
|
||||
}
|
||||
|
||||
public SortDoc copy() {
|
||||
SortValue[] svs = new SortValue[sortValues.length];
|
||||
for(int i=0; i<sortValues.length; i++) {
|
||||
svs[i] = sortValues[i].copy();
|
||||
}
|
||||
|
||||
return new SortDoc(svs);
|
||||
}
|
||||
|
||||
public boolean lessThan(Object o) {
|
||||
if(docId == -1) {
|
||||
return true;
|
||||
}
|
||||
SortDoc sd = (SortDoc)o;
|
||||
SortValue[] sortValues1 = sd.sortValues;
|
||||
for(int i=0; i<sortValues.length; i++) {
|
||||
int comp = sortValues[i].compareTo(sortValues1[i]);
|
||||
if(comp < 0) {
|
||||
return true;
|
||||
} if(comp > 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return docId+docBase > sd.docId+sd.docBase; //index order
|
||||
}
|
||||
|
||||
public int compareTo(Object o) {
|
||||
SortDoc sd = (SortDoc)o;
|
||||
for (int i=0; i<sortValues.length; i++) {
|
||||
int comp = sortValues[i].compareTo(sd.sortValues[i]);
|
||||
if (comp != 0) {
|
||||
return comp;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
public String toString() {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append("docId: " + docId + "; ");
|
||||
for (int i=0; i < sortValues.length; i++) {
|
||||
builder.append("value" + i + ": " + sortValues[i] + ", ");
|
||||
}
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
class SortQueue extends PriorityQueue<SortDoc> {
|
||||
|
||||
private SortDoc proto;
|
||||
private Object[] cache;
|
||||
|
||||
public SortQueue(int len, SortDoc proto) {
|
||||
super(len);
|
||||
this.proto = proto;
|
||||
}
|
||||
|
||||
protected boolean lessThan(SortDoc t1, SortDoc t2) {
|
||||
return t1.lessThan(t2);
|
||||
}
|
||||
|
||||
protected void populate() {
|
||||
Object[] heap = getHeapArray();
|
||||
cache = new SortDoc[heap.length];
|
||||
for (int i = 1; i < heap.length; i++) {
|
||||
cache[i] = heap[i] = proto.copy();
|
||||
}
|
||||
size = maxSize;
|
||||
}
|
||||
|
||||
protected void reset() {
|
||||
Object[] heap = getHeapArray();
|
||||
if(cache != null) {
|
||||
System.arraycopy(cache, 1, heap, 1, heap.length-1);
|
||||
size = maxSize;
|
||||
} else {
|
||||
populate();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
|
||||
public interface SortValue extends Comparable<SortValue> {
|
||||
public void setCurrentValue(int docId) throws IOException;
|
||||
public void setNextReader(LeafReaderContext context) throws IOException;
|
||||
public void setCurrentValue(SortValue value);
|
||||
public void reset();
|
||||
public SortValue copy();
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
|
||||
class StringFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
private FieldType fieldType;
|
||||
private CharsRefBuilder cref = new CharsRefBuilder();
|
||||
|
||||
public StringFieldWriter(String field, FieldType fieldType) {
|
||||
this.field = field;
|
||||
this.fieldType = fieldType;
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
|
||||
SortedDocValues vals = DocValues.getSorted(reader, this.field);
|
||||
if (vals.advance(docId) != docId) {
|
||||
return false;
|
||||
}
|
||||
int ord = vals.ordValue();
|
||||
|
||||
BytesRef ref = vals.lookupOrd(ord);
|
||||
fieldType.indexedToReadable(ref, cref);
|
||||
ew.put(this.field, cref.toString());
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.MultiDocValues;
|
||||
import org.apache.lucene.index.OrdinalMap;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.util.LongValues;
|
||||
|
||||
class StringValue implements SortValue {
|
||||
|
||||
protected SortedDocValues globalDocValues;
|
||||
|
||||
protected OrdinalMap ordinalMap;
|
||||
protected LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN;
|
||||
protected SortedDocValues docValues;
|
||||
|
||||
protected String field;
|
||||
protected int currentOrd;
|
||||
protected IntComp comp;
|
||||
protected int lastDocID;
|
||||
|
||||
public StringValue(SortedDocValues globalDocValues, String field, IntComp comp) {
|
||||
this.globalDocValues = globalDocValues;
|
||||
this.docValues = globalDocValues;
|
||||
if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
|
||||
this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) globalDocValues).mapping;
|
||||
}
|
||||
this.field = field;
|
||||
this.comp = comp;
|
||||
this.currentOrd = comp.resetValue();
|
||||
}
|
||||
|
||||
public StringValue copy() {
|
||||
return new StringValue(globalDocValues, field, comp);
|
||||
}
|
||||
|
||||
public void setCurrentValue(int docId) throws IOException {
|
||||
if (docId < lastDocID) {
|
||||
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
|
||||
}
|
||||
|
||||
lastDocID = docId;
|
||||
|
||||
if (docId > docValues.docID()) {
|
||||
docValues.advance(docId);
|
||||
}
|
||||
if (docId == docValues.docID()) {
|
||||
currentOrd = (int) toGlobal.get(docValues.ordValue());
|
||||
} else {
|
||||
currentOrd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
public void setCurrentValue(SortValue sv) {
|
||||
StringValue v = (StringValue)sv;
|
||||
this.currentOrd = v.currentOrd;
|
||||
}
|
||||
|
||||
public void setNextReader(LeafReaderContext context) throws IOException {
|
||||
if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
|
||||
toGlobal = ordinalMap.getGlobalOrds(context.ord);
|
||||
docValues = DocValues.getSorted(context.reader(), field);
|
||||
}
|
||||
lastDocID = 0;
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
this.currentOrd = comp.resetValue();
|
||||
}
|
||||
|
||||
public int compareTo(SortValue o) {
|
||||
StringValue sv = (StringValue)o;
|
||||
return comp.compare(currentOrd, sv.currentOrd);
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return Integer.toString(this.currentOrd);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Solr's Export Handler Functionality
|
||||
*/
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
|
||||
|
|
@ -40,7 +40,7 @@
|
|||
|
||||
<fieldType name="uuid" class="solr.UUIDField"/>
|
||||
|
||||
<field name="id" type="string" required="true" indexed="true"/>
|
||||
<field name="id" type="string" required="true" indexed="true" docValues="true"/>
|
||||
<field name="floatdv_m" type="float" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="intdv_m" type="int" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="doubledv_m" type="double" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
|
|
@ -54,6 +54,9 @@
|
|||
<field name="longdv" type="long" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="datedv" type="date" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="stringdv" type="string" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="booleandv" type="boolean" indexed="false" stored="false" docValues="true" />
|
||||
|
||||
<dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
|
||||
<!-- Point fields explicitly -->
|
||||
<dynamicField name="*_i_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
|
|
|
|||
|
|
@ -14,20 +14,30 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.response;
|
||||
package org.apache.solr.handler.export;
|
||||
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.util.SuppressForbidden;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.ResultContext;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocList;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
|
@ -139,6 +149,189 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSmallChains() throws Exception {
|
||||
clearIndex();
|
||||
|
||||
assertU(adoc("id","1",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p","1"));
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","2",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p",Integer.toString(Integer.MIN_VALUE + 1)));
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","3",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p",Integer.toString(Integer.MIN_VALUE)));
|
||||
assertU(commit());
|
||||
|
||||
//Test single value DocValue output
|
||||
//Expected for asc sort doc3 -> doc2 -> doc1
|
||||
String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "field1_i_p asc,field2_i_p asc"));
|
||||
assertJsonEquals(s, "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":3,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"3\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"2\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"1\"}]}}");
|
||||
|
||||
clearIndex();
|
||||
//Adding 3 docs of integers with the following values
|
||||
// doc1: Integer.MIN_VALUE,1,2,Integer.MAX_VALUE,3,4,5,6
|
||||
// doc2: Integer.MIN_VALUE,Integer.MIN_VALUE,2,Integer.MAX_VALUE,4,4,5,6
|
||||
// doc3: Integer.MIN_VALUE,Integer.MIN_VALUE,2,Integer.MAX_VALUE,3,4,5,6
|
||||
|
||||
assertU(adoc("id","1",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p","1",
|
||||
"field3_i_p","2",
|
||||
"field4_i_p",Integer.toString(Integer.MAX_VALUE),
|
||||
"field5_i_p","3",
|
||||
"field6_i_p","4",
|
||||
"field7_i_p","5",
|
||||
"field8_i_p","6"));
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","2",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field3_i_p","2",
|
||||
"field4_i_p",Integer.toString(Integer.MAX_VALUE),
|
||||
"field5_i_p","4",
|
||||
"field6_i_p","4",
|
||||
"field7_i_p","5",
|
||||
"field8_i_p","6"));
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","3",
|
||||
"field1_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field2_i_p",Integer.toString(Integer.MIN_VALUE),
|
||||
"field3_i_p","2",
|
||||
"field4_i_p",Integer.toString(Integer.MAX_VALUE),
|
||||
"field5_i_p","3",
|
||||
"field6_i_p","4",
|
||||
"field7_i_p","5",
|
||||
"field8_i_p","6"));
|
||||
assertU(commit());
|
||||
|
||||
s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "field1_i_p asc,field2_i_p asc,field3_i_p asc,field4_i_p asc,field5_i_p desc,field6_i_p desc,field7_i_p desc,field8_i_p asc"));
|
||||
assertJsonEquals(s, "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":3,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"2\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"3\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"1\"}]}}");
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIndexOrder() throws Exception {
|
||||
clearIndex();
|
||||
|
||||
assertU(adoc("id","1", "stringdv","a"));
|
||||
assertU(adoc("id","2", "stringdv","a"));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","3", "stringdv","a"));
|
||||
assertU(adoc("id","4", "stringdv","a"));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
String expectedResult = "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":4,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"1\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"2\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"3\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"4\"}]}}";
|
||||
|
||||
String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv asc"));
|
||||
assertJsonEquals(s, expectedResult);
|
||||
|
||||
s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv desc"));
|
||||
assertJsonEquals(s, expectedResult);
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStringWithCase() throws Exception {
|
||||
clearIndex();
|
||||
|
||||
assertU(adoc("id","1", "stringdv","a"));
|
||||
assertU(adoc("id","2", "stringdv","ABC"));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","3", "stringdv","xyz"));
|
||||
assertU(adoc("id","4", "stringdv","a"));
|
||||
|
||||
assertU(commit());
|
||||
|
||||
String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv desc"));
|
||||
assertJsonEquals(s, "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":4,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"3\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"1\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"4\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"2\"}]}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBooleanField() throws Exception {
|
||||
clearIndex();
|
||||
|
||||
assertU(adoc("id","1",
|
||||
"booleandv","true"));
|
||||
assertU(commit());
|
||||
|
||||
assertU(adoc("id","2",
|
||||
"booleandv","false"));
|
||||
assertU(commit());
|
||||
|
||||
String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "booleandv asc"));
|
||||
assertJsonEquals(s, "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":2,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"2\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"1\"}]}}");
|
||||
|
||||
s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "booleandv desc"));
|
||||
assertJsonEquals(s, "{\n" +
|
||||
" \"responseHeader\":{\"status\":0},\n" +
|
||||
" \"response\":{\n" +
|
||||
" \"numFound\":2,\n" +
|
||||
" \"docs\":[{\n" +
|
||||
" \"id\":\"1\"}\n" +
|
||||
" ,{\n" +
|
||||
" \"id\":\"2\"}]}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSortingOutput() throws Exception {
|
||||
|
||||
|
|
@ -195,6 +388,38 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,floatdv desc,floatdv asc,intdv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
|
||||
//Test five sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc,floatdv desc,floatdv desc,intdv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
|
||||
|
||||
//Test six sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,intdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
|
||||
//Test seven sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
|
||||
|
||||
//Test eight sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
|
||||
|
||||
//Test nine sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv asc,intdv desc,floatdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv desc,intdv asc,floatdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
|
||||
//Test ten sort fields
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv asc,intdv desc,floatdv desc,floatdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv desc,intdv asc,floatdv desc,floatdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
|
||||
|
||||
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}}");
|
||||
|
||||
|
|
@ -331,6 +556,160 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultipleSorts() throws Exception {
|
||||
assertU(delQ("*:*"));
|
||||
assertU(commit());
|
||||
|
||||
int numDocs = 1000;
|
||||
|
||||
//10 unique values
|
||||
String[] str_vals = new String[10];
|
||||
for (int i=0; i<str_vals.length; i++) {
|
||||
str_vals[i] = TestUtil.randomSimpleString(random(), 10);
|
||||
}
|
||||
|
||||
float[] float_vals = new float[10];
|
||||
float_vals[0] = 0.0f;
|
||||
float_vals[1] = +0.0f;
|
||||
float_vals[2] = -0.0f;
|
||||
float_vals[3] = +0.00001f;
|
||||
float_vals[4] = +0.000011f;
|
||||
float_vals[5] = Float.MAX_VALUE;
|
||||
float_vals[6] = Float.MIN_VALUE;
|
||||
float_vals[7] = 1/3f; //0.33333334
|
||||
float_vals[8] = 0.33333333f;
|
||||
float_vals[9] = random().nextFloat();
|
||||
|
||||
for (int i = 0; i < numDocs; i++) {
|
||||
int number = TestUtil.nextInt(random(), 0, 9);
|
||||
assertU(adoc("id", String.valueOf(i),
|
||||
"floatdv", String.valueOf(number),
|
||||
"intdv", String.valueOf(number),
|
||||
"stringdv", String.valueOf(str_vals[number]),
|
||||
"longdv", String.valueOf(number),
|
||||
"doubledv", String.valueOf(number),
|
||||
"datedv", randomSkewedDate(),
|
||||
"booleandv", String.valueOf(random().nextBoolean()),
|
||||
"field1_s_dv", String.valueOf(str_vals[number]),
|
||||
"field2_i_p", String.valueOf(number),
|
||||
"field3_l_p", String.valueOf(number)));
|
||||
if (numDocs % 3000 ==0) {
|
||||
assertU(commit());
|
||||
}
|
||||
}
|
||||
assertU(commit());
|
||||
|
||||
validateSort(numDocs);
|
||||
}
|
||||
|
||||
private void validateSort(int numDocs) throws Exception {
|
||||
// 10 fields
|
||||
List<String> fieldNames = new ArrayList<>(Arrays.asList("floatdv", "intdv", "stringdv", "longdv", "doubledv",
|
||||
"datedv", "booleandv", "field1_s_dv", "field2_i_p", "field3_l_p"));
|
||||
|
||||
SortFields[] fieldSorts = new SortFields[TestUtil.nextInt(random(), 1, fieldNames.size())];
|
||||
for (int i = 0; i < fieldSorts.length; i++) {
|
||||
fieldSorts[i] = new SortFields(fieldNames.get(TestUtil.nextInt(random(), 0, fieldNames.size() - 1)));
|
||||
fieldNames.remove(fieldSorts[i].getField());
|
||||
}
|
||||
String[] fieldWithOrderStrs = new String[fieldSorts.length];
|
||||
String[] fieldStrs = new String[fieldSorts.length];
|
||||
for (int i = 0; i < fieldSorts.length; i++) {
|
||||
fieldWithOrderStrs[i] = fieldSorts[i].getFieldWithOrder();
|
||||
fieldStrs[i] = fieldSorts[i].getField();
|
||||
}
|
||||
|
||||
String sortStr = String.join(",", fieldWithOrderStrs); // sort : field1 asc, field2 desc
|
||||
String fieldsStr = String.join(",", fieldStrs); // fl : field1, field2
|
||||
|
||||
String resp = h.query(req("q", "*:*", "qt", "/export", "fl", "id," + fieldsStr, "sort", sortStr));
|
||||
|
||||
//We cannot compare /select vs /export as for docs with the same values ( ties ) the ordering is different
|
||||
SolrQueryRequest req = null;
|
||||
try {
|
||||
req = req("q", "*:*", "qt", "/select", "fl", "id," + fieldsStr, "sort", sortStr, "rows", Integer.toString(numDocs));
|
||||
|
||||
SolrQueryResponse selectRsp = h.queryAndResponse("", req);
|
||||
DocList selectDocList = ((ResultContext)selectRsp.getResponse()).getDocList();
|
||||
assert selectDocList.size() == numDocs;
|
||||
DocIterator selectDocListIter = selectDocList.iterator();
|
||||
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
HashMap respMap = mapper.readValue(resp, HashMap.class);
|
||||
List docs = (ArrayList) ((HashMap) respMap.get("response")).get("docs");
|
||||
assert docs.size() == numDocs;
|
||||
|
||||
for (int i = 0; i < docs.size() - 1; i++) { // docs..
|
||||
assertEquals("Position:" + i + " has different id value" , String.valueOf(selectDocListIter.nextDoc()), String.valueOf(((HashMap) docs.get(i)).get("id")));
|
||||
|
||||
for (int j = 0; j < fieldSorts.length; j++) { // fields ..
|
||||
String field = fieldSorts[j].getField();
|
||||
String sort = fieldSorts[j].getSort();
|
||||
String fieldVal1 = String.valueOf(((HashMap) docs.get(i)).get(field)); // 1st doc
|
||||
String fieldVal2 = String.valueOf(((HashMap) docs.get(i + 1)).get(field)); // 2nd obj
|
||||
if (fieldVal1.equals(fieldVal2)) {
|
||||
continue;
|
||||
} else {
|
||||
if (sort.equals("asc")) {
|
||||
if (field.equals("stringdv") || field.equals("field1_s_dv")|| field.equals("datedv") || field.equals("booleandv")) { // use string comparator
|
||||
assertTrue(fieldVal1.compareTo(fieldVal2) < 0);
|
||||
} else if (field.equals("doubledv")){
|
||||
assertTrue(Double.compare(Double.valueOf(fieldVal1), Double.valueOf(fieldVal2)) <= 0);
|
||||
} else if(field.equals("floatdv")) {
|
||||
assertTrue(Float.compare(Float.valueOf(fieldVal1), Float.valueOf(fieldVal2)) <= 0);
|
||||
} else if(field.equals("intdv") || "field2_i_p".equals(field)) {
|
||||
assertTrue(Integer.compare(Integer.valueOf(fieldVal1), Integer.valueOf(fieldVal2)) <= 0);
|
||||
} else if(field.equals("longdv") || field.equals("field3_l_p")) {
|
||||
assertTrue(Long.compare(Integer.valueOf(fieldVal1), Long.valueOf(fieldVal2)) <= 0);
|
||||
}
|
||||
} else {
|
||||
if (field.equals("stringdv") || field.equals("field1_s_dv")|| field.equals("datedv") || field.equals("booleandv")) { // use string comparator
|
||||
assertTrue(fieldVal1.compareTo(fieldVal2) > 0);
|
||||
} else if (field.equals("doubledv")){
|
||||
assertTrue(Double.compare(Double.valueOf(fieldVal1), Double.valueOf(fieldVal2)) >= 0);
|
||||
} else if(field.equals("floatdv")) {
|
||||
assertTrue(Float.compare(Float.valueOf(fieldVal1), Float.valueOf(fieldVal2)) >= 0);
|
||||
} else if(field.equals("intdv") || "field2_i_p".equals(field)) {
|
||||
assertTrue(Integer.compare(Integer.valueOf(fieldVal1), Integer.valueOf(fieldVal2)) >= 0);
|
||||
} else if(field.equals("longdv") || field.equals("field3_l_p")) {
|
||||
assertTrue(Long.compare(Integer.valueOf(fieldVal1), Long.valueOf(fieldVal2)) >= 0);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (req != null) {
|
||||
req.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class SortFields {
|
||||
String fieldName;
|
||||
String sortOrder;
|
||||
String[] orders = {"asc", "desc"};
|
||||
|
||||
SortFields(String fn) {
|
||||
this.fieldName = fn;
|
||||
this.sortOrder = orders[random().nextInt(2)];
|
||||
}
|
||||
|
||||
public String getFieldWithOrder() {
|
||||
return this.fieldName + " " + this.sortOrder;
|
||||
}
|
||||
|
||||
public String getField() {
|
||||
return this.fieldName;
|
||||
}
|
||||
|
||||
public String getSort() {
|
||||
return this.sortOrder;
|
||||
}
|
||||
}
|
||||
|
||||
private void doTestQuery(String query, List<String> trieFields, List<String> pointFields) throws Exception {
|
||||
String trieFieldsFl = String.join(",", trieFields);
|
||||
String pointFieldsFl = String.join(",", pointFields);
|
||||
|
|
@ -52,7 +52,8 @@ http://localhost:8983/solr/core_name/export?q=my-query&sort=severity+desc,timest
|
|||
|
||||
The `sort` property defines how documents will be sorted in the exported result set. Results can be sorted by any field that has a field type of int,long, float, double, string. The sort fields must be single valued fields.
|
||||
|
||||
Up to four sort fields can be specified per request, with the 'asc' or 'desc' properties.
|
||||
The export performance will get slower as you add more sort fields. If there is enough physical memory available outside of the JVM to load up the sort fields then the performance will be linearly slower with addition of sort fields.
|
||||
It can get worse otherwise.
|
||||
|
||||
=== Specifying the Field List
|
||||
|
||||
|
|
|
|||
|
|
@ -321,10 +321,6 @@ The Column Identifiers can contain both fields in the Solr index and aggregate f
|
|||
|
||||
The non-function fields in the field list determine the fields to calculate the aggregations over.
|
||||
|
||||
*`GROUP BY` Clause*
|
||||
|
||||
The `GROUP BY` clause can contain up to 4 fields in the Solr index. These fields should correspond with the non-function fields in the field list.
|
||||
|
||||
=== HAVING Clause
|
||||
|
||||
The `HAVING` clause may contain any function listed in the field list. Complex `HAVING` clauses such as this are supported:
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ public class SolrException extends RuntimeException {
|
|||
return null;
|
||||
}
|
||||
|
||||
public static Throwable getRootCause(Throwable t) {
|
||||
public static Throwable getRootCause(Throwable t) {
|
||||
while (true) {
|
||||
Throwable cause = t.getCause();
|
||||
if (cause!=null) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue