mirror of https://github.com/apache/lucene.git
SOLR-10835: Add support for point fields in Export Handler
This commit is contained in:
Tomas Fernandez Lobbe
parent
0411504dd8
commit
c51f6fae75
|
|
@ -360,6 +360,8 @@ Bug Fixes
|
|||
|
||||
* SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat)
|
||||
|
||||
* SOLR-10835: Add support for point fields in Export Handler (Tomás Fernández Löbbe)
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
* SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@
|
|||
|
||||
package org.apache.solr.handler;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.apache.solr.common.util.Utils.makeMap;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
|
|
@ -26,6 +30,7 @@ import java.lang.invoke.MethodHandles;
|
|||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.function.LongFunction;
|
||||
|
||||
import org.apache.lucene.index.DocValues;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
|
|
@ -34,6 +39,7 @@ import org.apache.lucene.index.LeafReaderContext;
|
|||
import org.apache.lucene.index.MultiDocValues;
|
||||
import org.apache.lucene.index.NumericDocValues;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.index.SortedNumericDocValues;
|
||||
import org.apache.lucene.index.SortedSetDocValues;
|
||||
import org.apache.lucene.search.DocIdSetIterator;
|
||||
import org.apache.lucene.search.Sort;
|
||||
|
|
@ -44,6 +50,7 @@ import org.apache.lucene.util.BytesRef;
|
|||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.LongValues;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
|
||||
import org.apache.solr.common.IteratorWriter;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
|
|
@ -60,25 +67,21 @@ import org.apache.solr.response.JSONResponseWriter;
|
|||
import org.apache.solr.response.QueryResponseWriter;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.BoolField;
|
||||
import org.apache.solr.schema.DateValueFieldType;
|
||||
import org.apache.solr.schema.DoubleValueFieldType;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.FloatValueFieldType;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.IntValueFieldType;
|
||||
import org.apache.solr.schema.LongValueFieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.StrField;
|
||||
import org.apache.solr.schema.TrieDateField;
|
||||
import org.apache.solr.schema.TrieDoubleField;
|
||||
import org.apache.solr.schema.TrieFloatField;
|
||||
import org.apache.solr.schema.TrieIntField;
|
||||
import org.apache.solr.schema.TrieLongField;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.search.SortSpec;
|
||||
import org.apache.solr.search.SyntaxError;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static org.apache.solr.common.util.Utils.makeMap;
|
||||
|
||||
public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
||||
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private OutputStreamWriter respWriter;
|
||||
|
|
@ -322,25 +325,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
|
||||
boolean multiValued = schemaField.multiValued();
|
||||
FieldType fieldType = schemaField.getType();
|
||||
if (fieldType instanceof TrieIntField) {
|
||||
if (fieldType instanceof IntValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new IntFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof TrieLongField) {
|
||||
} else if (fieldType instanceof LongValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new LongFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof TrieFloatField) {
|
||||
} else if (fieldType instanceof FloatValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
writers[i] = new FloatFieldWriter(field);
|
||||
}
|
||||
} else if (fieldType instanceof TrieDoubleField) {
|
||||
} else if (fieldType instanceof DoubleValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
|
||||
} else {
|
||||
|
|
@ -352,7 +355,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
} else {
|
||||
writers[i] = new StringFieldWriter(field, fieldType);
|
||||
}
|
||||
} else if (fieldType instanceof TrieDateField) {
|
||||
} else if (fieldType instanceof DateValueFieldType) {
|
||||
if (multiValued) {
|
||||
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
|
||||
} else {
|
||||
|
|
@ -385,25 +388,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
throw new IOException(field+" must have DocValues to use this feature.");
|
||||
}
|
||||
|
||||
if(ft instanceof TrieIntField) {
|
||||
if(ft instanceof IntValueFieldType) {
|
||||
if(reverse) {
|
||||
sortValues[i] = new IntValue(field, new IntDesc());
|
||||
} else {
|
||||
sortValues[i] = new IntValue(field, new IntAsc());
|
||||
}
|
||||
} else if(ft instanceof TrieFloatField) {
|
||||
} else if(ft instanceof FloatValueFieldType) {
|
||||
if(reverse) {
|
||||
sortValues[i] = new FloatValue(field, new FloatDesc());
|
||||
} else {
|
||||
sortValues[i] = new FloatValue(field, new FloatAsc());
|
||||
}
|
||||
} else if(ft instanceof TrieDoubleField) {
|
||||
} else if(ft instanceof DoubleValueFieldType) {
|
||||
if(reverse) {
|
||||
sortValues[i] = new DoubleValue(field, new DoubleDesc());
|
||||
} else {
|
||||
sortValues[i] = new DoubleValue(field, new DoubleAsc());
|
||||
}
|
||||
} else if(ft instanceof TrieLongField) {
|
||||
} else if(ft instanceof LongValueFieldType) {
|
||||
if(reverse) {
|
||||
sortValues[i] = new LongValue(field, new LongDesc());
|
||||
} else {
|
||||
|
|
@ -417,7 +420,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
} else {
|
||||
sortValues[i] = new StringValue(vals, field, new IntAsc());
|
||||
}
|
||||
} else if (ft instanceof TrieDateField) {
|
||||
} else if (ft instanceof DateValueFieldType) {
|
||||
if (reverse) {
|
||||
sortValues[i] = new LongValue(field, new LongDesc());
|
||||
} else {
|
||||
|
|
@ -1352,6 +1355,23 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static LongFunction<Object> bitsToValue(FieldType fieldType) {
|
||||
switch (fieldType.getNumberType()) {
|
||||
case LONG:
|
||||
return (bits)-> bits;
|
||||
case DATE:
|
||||
return (bits)-> new Date(bits);
|
||||
case INTEGER:
|
||||
return (bits)-> (int)bits;
|
||||
case FLOAT:
|
||||
return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
|
||||
case DOUBLE:
|
||||
return (bits)-> NumericUtils.sortableLongToDouble(bits);
|
||||
default:
|
||||
throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
|
||||
}
|
||||
}
|
||||
|
||||
class MultiFieldWriter extends FieldWriter {
|
||||
private String field;
|
||||
|
|
@ -1359,29 +1379,48 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
|
|||
private SchemaField schemaField;
|
||||
private boolean numeric;
|
||||
private CharsRefBuilder cref = new CharsRefBuilder();
|
||||
private final LongFunction<Object> bitsToValue;
|
||||
|
||||
public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
|
||||
this.field = field;
|
||||
this.fieldType = fieldType;
|
||||
this.schemaField = schemaField;
|
||||
this.numeric = numeric;
|
||||
if (this.fieldType.isPointField()) {
|
||||
bitsToValue = bitsToValue(fieldType);
|
||||
} else {
|
||||
bitsToValue = null;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
|
||||
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
|
||||
if (vals.advance(docId) != docId) return false;
|
||||
out.put(this.field,
|
||||
(IteratorWriter) w -> {
|
||||
long o;
|
||||
while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
|
||||
BytesRef ref = vals.lookupOrd(o);
|
||||
fieldType.indexedToReadable(ref, cref);
|
||||
IndexableField f = fieldType.createField(schemaField, cref.toString());
|
||||
if (f == null) w.add(cref.toString());
|
||||
else w.add(fieldType.toObject(f));
|
||||
}
|
||||
});
|
||||
return true;
|
||||
if (this.fieldType.isPointField()) {
|
||||
SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
|
||||
if (!vals.advanceExact(docId)) return false;
|
||||
out.put(this.field,
|
||||
(IteratorWriter) w -> {
|
||||
for (int i = 0; i < vals.docValueCount(); i++) {
|
||||
w.add(bitsToValue.apply(vals.nextValue()));
|
||||
}
|
||||
});
|
||||
return true;
|
||||
} else {
|
||||
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
|
||||
if (vals.advance(docId) != docId) return false;
|
||||
out.put(this.field,
|
||||
(IteratorWriter) w -> {
|
||||
long o;
|
||||
while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
|
||||
BytesRef ref = vals.lookupOrd(o);
|
||||
fieldType.indexedToReadable(ref, cref);
|
||||
IndexableField f = fieldType.createField(schemaField, cref.toString());
|
||||
if (f == null) w.add(cref.toString());
|
||||
else w.add(fieldType.toObject(f));
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,14 @@
|
|||
seconds part (.999) is optional.
|
||||
-->
|
||||
<fieldType name="date" class="solr.TrieDateField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
|
||||
|
||||
<!-- Point Fields -->
|
||||
<fieldType name="pint" class="solr.IntPointField" docValues="true"/>
|
||||
<fieldType name="plong" class="solr.LongPointField" docValues="true"/>
|
||||
<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
|
||||
<fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
|
||||
<fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
|
||||
|
||||
|
||||
<fieldType name="boolean" class="solr.BoolField"/>
|
||||
<fieldType name="string" class="solr.StrField"/>
|
||||
|
|
@ -33,20 +41,63 @@
|
|||
<fieldType name="uuid" class="solr.UUIDField"/>
|
||||
|
||||
<field name="id" type="string" required="true" indexed="true"/>
|
||||
<field name="floatdv_m" type="float" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="intdv_m" type="int" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="doubledv_m" type="double" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="longdv_m" type="long" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="datedv_m" type="date" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="floatdv_m" type="${solr.tests.floatClass:pfloat}" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="intdv_m" type="${solr.tests.intClass:pint}" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="doubledv_m" type="${solr.tests.doubleClass:pdouble}" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="longdv_m" type="${solr.tests.longClass:plong}" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="datedv_m" type="${solr.tests.dateClass:pdate}" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
<field name="stringdv_m" type="string" indexed="false" stored="false" docValues="true" multiValued="true"/>
|
||||
|
||||
<field name="floatdv" type="float" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="intdv" type="int" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="doubledv" type="double" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="longdv" type="long" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="datedv" type="date" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="floatdv" type="${solr.tests.floatClass:pfloat}" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="intdv" type="${solr.tests.intClass:pint}" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="doubledv" type="${solr.tests.doubleClass:pdouble}" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="longdv" type="${solr.tests.longClass:plong}" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="datedv" type="${solr.tests.dateClass:pdate}" indexed="false" stored="false" docValues="true"/>
|
||||
<field name="stringdv" type="string" indexed="false" stored="false" docValues="true"/>
|
||||
|
||||
<!-- Point fields explicitly -->
|
||||
<dynamicField name="*_i_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_is_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_i_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_is_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_l_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ls_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_l_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ls_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_f_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_fs_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_f_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_fs_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_d_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ds_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_d_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ds_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_dt_p" type="pdate" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_dts_p" type="pdate" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_dt_ni_p" type="pdate" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_dts_ni_p" type="pdate" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
|
||||
<!-- Trie fields explicitly -->
|
||||
<dynamicField name="*_i_t" type="int" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_is_t" type="int" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_i_ni_t" type="int" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_is_ni_t" type="int" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_l_t" type="long" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ls_t" type="long" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_l_ni_t" type="long" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ls_ni_t" type="long" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_f_t" type="float" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_fs_t" type="float" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_f_ni_t" type="float" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_fs_ni_t" type="float" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_d_t" type="double" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ds_t" type="double" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_d_ni_t" type="double" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_ds_ni_t" type="double" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_dt_t" type="date" indexed="true" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_dts_t" type="date" indexed="true" stored="true" docValues="true" multiValued="true"/>
|
||||
<dynamicField name="*_dt_ni_t" type="date" indexed="false" stored="true" docValues="true" multiValued="false"/>
|
||||
<dynamicField name="*_dts_ni_t" type="date" indexed="false" stored="true" docValues="true" multiValued="true"/>
|
||||
|
||||
<uniqueKey>id</uniqueKey>
|
||||
|
||||
|
|
|
|||
|
|
@ -16,17 +16,40 @@
|
|||
*/
|
||||
package org.apache.solr.response;
|
||||
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.junit.*;
|
||||
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.common.SolrInputDocument;
|
||||
import org.apache.solr.common.util.SuppressForbidden;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.RefCounted;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
@SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
|
||||
public class TestExportWriter extends SolrTestCaseJ4 {
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
System.setProperty("export.test", "true");
|
||||
initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml");
|
||||
}
|
||||
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
assertU(delQ("*:*"));
|
||||
assertU(commit());
|
||||
createIndex();
|
||||
}
|
||||
|
||||
|
|
@ -47,7 +70,12 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
"longdv_m", "343332",
|
||||
"stringdv_m", "manchester \"city\"",
|
||||
"stringdv_m", "liverpool",
|
||||
"stringdv_m", "Everton"));
|
||||
"stringdv_m", "Everton",
|
||||
"datedv", "2017-06-16T07:00:00Z",
|
||||
"datedv_m", "2017-06-16T01:00:00Z",
|
||||
"datedv_m", "2017-06-16T02:00:00Z",
|
||||
"datedv_m", "2017-06-16T03:00:00Z",
|
||||
"datedv_m", "2017-06-16T04:00:00Z"));
|
||||
|
||||
assertU(adoc("id","7",
|
||||
"floatdv","2.1",
|
||||
|
|
@ -80,7 +108,11 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
"longdv_m", "343332",
|
||||
"stringdv_m", "manchester \"city\"",
|
||||
"stringdv_m", "liverpool",
|
||||
"stringdv_m", "everton"));
|
||||
"stringdv_m", "everton",
|
||||
"int_is_t", "1",
|
||||
"int_is_t", "1",
|
||||
"int_is_t", "1",
|
||||
"int_is_t", "1"));
|
||||
assertU(commit());
|
||||
assertU(adoc("id","8",
|
||||
"floatdv","2.1",
|
||||
|
|
@ -98,7 +130,14 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
"longdv_m", "343332",
|
||||
"stringdv_m", "manchester \"city\"",
|
||||
"stringdv_m", "liverpool",
|
||||
"stringdv_m", "everton"));
|
||||
"stringdv_m", "everton",
|
||||
"datedv", "2017-01-01T00:00:00Z",
|
||||
"datedv_m", "2017-01-01T01:00:00Z",
|
||||
"datedv_m", "2017-01-01T02:00:00Z",
|
||||
"int_is_p", "1",
|
||||
"int_is_p", "1",
|
||||
"int_is_p", "1",
|
||||
"int_is_p", "1"));
|
||||
assertU(commit());
|
||||
|
||||
|
||||
|
|
@ -192,4 +231,152 @@ public class TestExportWriter extends SolrTestCaseJ4 {
|
|||
// Interesting you don't even need to specify a "q" parameter.
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDates() throws Exception {
|
||||
String s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv", "sort", "datedv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv\":\"2017-06-16T07:00:00Z\"}]}}");
|
||||
s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv_m", "sort", "datedv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv_m\":[\"2017-06-16T01:00:00Z\",\"2017-06-16T02:00:00Z\",\"2017-06-16T03:00:00Z\",\"2017-06-16T04:00:00Z\"]}]}}");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDuplicates() throws Exception {
|
||||
RefCounted<SolrIndexSearcher> ref = null;
|
||||
try {
|
||||
ref = h.getCore().getSearcher();
|
||||
LeafReader reader = ref.get().getSlowAtomicReader();
|
||||
// MultiValued Trie fields use SortedSet
|
||||
assertNotNull(reader.getSortedSetDocValues("int_is_t"));
|
||||
assertNull(reader.getSortedNumericDocValues("int_is_t"));
|
||||
// MultiValued Point fields use SortedNumerics
|
||||
assertNull(reader.getSortedSetDocValues("int_is_p"));
|
||||
assertNotNull(reader.getSortedNumericDocValues("int_is_p"));
|
||||
} finally {
|
||||
if (ref != null) ref.decref();
|
||||
}
|
||||
String s = h.query(req("q", "id:3", "qt", "/export", "fl", "int_is_t", "sort", "intdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_t\":[1]}]}}");
|
||||
s = h.query(req("q", "id:8", "qt", "/export", "fl", "int_is_p", "sort", "intdv asc"));
|
||||
assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_p\":[1,1,1,1]}]}}");
|
||||
}
|
||||
|
||||
/**
|
||||
* This test doesn't validate the correctness of results, it just compares the response of the same request
|
||||
* when asking for Trie fields vs Point fields. Can be removed once Trie fields are no longer supported
|
||||
*/
|
||||
@Test
|
||||
@SuppressForbidden(reason="using new Date(time) to create random dates")
|
||||
public void testRandomNumerics() throws Exception {
|
||||
assertU(delQ("*:*"));
|
||||
assertU(commit());
|
||||
List<String> trieFields = new ArrayList<String>();
|
||||
List<String> pointFields = new ArrayList<String>();
|
||||
for (String mv:new String[]{"s", ""}) {
|
||||
for (String indexed:new String[]{"_ni", ""}) {
|
||||
for (String type:new String[]{"i", "l", "f", "d", "dt"}) {
|
||||
String field = "number_" + type + mv + indexed;
|
||||
SchemaField sf = h.getCore().getLatestSchema().getField(field + "_t");
|
||||
assertTrue(sf.hasDocValues());
|
||||
assertTrue(sf.getType().getNumberType() != null);
|
||||
assertFalse(sf.getType().isPointField());
|
||||
|
||||
sf = h.getCore().getLatestSchema().getField(field + "_p");
|
||||
assertTrue(sf.hasDocValues());
|
||||
assertTrue(sf.getType().getNumberType() != null);
|
||||
assertTrue(sf.getType().isPointField());
|
||||
|
||||
trieFields.add(field + "_t");
|
||||
pointFields.add(field + "_p");
|
||||
}
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < atLeast(100); i++) {
|
||||
if (random().nextInt(20) == 0) {
|
||||
//have some empty docs
|
||||
assertU(adoc("id", String.valueOf(i)));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (random().nextInt(20) == 0 && i > 0) {
|
||||
//delete some docs
|
||||
assertU(delI(String.valueOf(i - 1)));
|
||||
}
|
||||
|
||||
SolrInputDocument doc = new SolrInputDocument();
|
||||
doc.addField("id", String.valueOf(i));
|
||||
addInt(doc, random().nextInt(), false);
|
||||
addLong(doc, random().nextLong(), false);
|
||||
addFloat(doc, random().nextFloat() * 3000 * (random().nextBoolean()?1:-1), false);
|
||||
addDouble(doc, random().nextDouble() * 3000 * (random().nextBoolean()?1:-1), false);
|
||||
addDate(doc, new Date(), false);
|
||||
|
||||
// MV need to be unique in order to be the same in Trie vs Points
|
||||
Set<Integer> ints = new HashSet<>();
|
||||
Set<Long> longs = new HashSet<>();
|
||||
Set<Float> floats = new HashSet<>();
|
||||
Set<Double> doubles = new HashSet<>();
|
||||
Set<Date> dates = new HashSet<>();
|
||||
for (int j=0; j < random().nextInt(20); j++) {
|
||||
ints.add(random().nextInt());
|
||||
longs.add(random().nextLong());
|
||||
floats.add(random().nextFloat() * 3000 * (random().nextBoolean()?1:-1));
|
||||
doubles.add(random().nextDouble() * 3000 * (random().nextBoolean()?1:-1));
|
||||
dates.add(new Date(System.currentTimeMillis() + random().nextInt()));
|
||||
}
|
||||
ints.stream().forEach((val)->addInt(doc, val, true));
|
||||
longs.stream().forEach((val)->addLong(doc, val, true));
|
||||
floats.stream().forEach((val)->addFloat(doc, val, true));
|
||||
doubles.stream().forEach((val)->addDouble(doc, val, true));
|
||||
dates.stream().forEach((val)->addDate(doc, val, true));
|
||||
|
||||
assertU(adoc(doc));
|
||||
if (random().nextInt(20) == 0) {
|
||||
assertU(commit());
|
||||
}
|
||||
}
|
||||
assertU(commit());
|
||||
doTestQuery("id:1", trieFields, pointFields);
|
||||
doTestQuery("*:*", trieFields, pointFields);
|
||||
doTestQuery("id:[0 TO 2]", trieFields, pointFields);// "id" field is really a string, this is not a numeric range query
|
||||
doTestQuery("id:[0 TO 9]", trieFields, pointFields);
|
||||
doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields);
|
||||
}
|
||||
|
||||
private void doTestQuery(String query, List<String> trieFields, List<String> pointFields) throws Exception {
|
||||
String trieFieldsFl = String.join(",", trieFields);
|
||||
String pointFieldsFl = String.join(",", pointFields);
|
||||
String sort = pickRandom((String)pickRandom(trieFields.toArray()), (String)pickRandom(pointFields.toArray())).replace("s_", "_") + pickRandom(" asc", " desc");
|
||||
String resultPoints = h.query(req("q", query, "qt", "/export", "fl", pointFieldsFl, "sort", sort));
|
||||
String resultTries = h.query(req("q", query, "qt", "/export", "fl", trieFieldsFl, "sort", sort));
|
||||
assertJsonEquals(resultPoints.replaceAll("_p", ""), resultTries.replaceAll("_t", ""));
|
||||
}
|
||||
|
||||
private void addFloat(SolrInputDocument doc, float value, boolean mv) {
|
||||
addField(doc, "f", String.valueOf(value), mv);
|
||||
}
|
||||
|
||||
private void addDouble(SolrInputDocument doc, double value, boolean mv) {
|
||||
addField(doc, "d", String.valueOf(value), mv);
|
||||
}
|
||||
|
||||
private void addLong(SolrInputDocument doc, long value, boolean mv) {
|
||||
addField(doc, "l", String.valueOf(value), mv);
|
||||
}
|
||||
|
||||
private void addInt(SolrInputDocument doc, int value, boolean mv) {
|
||||
addField(doc, "i", String.valueOf(value), mv);
|
||||
}
|
||||
|
||||
private void addDate(SolrInputDocument doc, Date value, boolean mv) {
|
||||
addField(doc, "dt", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT).format(value), mv);
|
||||
}
|
||||
|
||||
private void addField(SolrInputDocument doc, String type, String value, boolean mv) {
|
||||
doc.addField("number_" + type + (mv?"s":"") + "_t", value);
|
||||
doc.addField("number_" + type + (mv?"s":"") + "_p", value);
|
||||
doc.addField("number_" + type + (mv?"s":"") + "_ni_t", value);
|
||||
doc.addField("number_" + type + (mv?"s":"") + "_ni_p", value);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue