diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f540292d9f8..2fc9c30eca3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -217,6 +217,21 @@ Jetty 9.2.13.v20150730 Upgrading from Solr 5.4 ----------------------- +* The Solr schema version has been increased to 1.6. Since schema version 1.6, all non-stored docValues fields + will be returned along with other stored fields when all fields (or pattern matching globs) are specified + to be returned (e.g. fl=*) for search queries. This behavior can be turned on and off by setting + 'useDocValuesAsStored' parameter for a field or a field type to true (default since schema version 1.6) + or false (default till schema version 1.5). + Note that enabling this property has performance implications because DocValues are column-oriented and may + therefore incur additional cost to retrieve for each returned document. All example schema are upgraded to + version 1.6 but any older schemas will default to useDocValuesAsStored=false and continue to work as in + older versions of Solr. If this new behavior is desirable, then you should set version attribute in your + schema file to '1.6'. + Also note that returning stored fields from docValues (default in schema versions 1.6+) returns multiValued + fields in sorted order. If you require the older behavior of multiValued fields being returned in the + original insertion order, set useDocValuesAsStored="false" for the individual fields or make + sure your schema version is < 1.6. This does not require re-indexing. + See SOLR-8220 for more details. Detailed Change List ---------------------- @@ -246,6 +261,9 @@ New Features child.facet.field parameter with {!parent ..}.. query. They count facets on children documents aggregating (deduplicating) counts by parent documents (Dr. Oleg Savrasov via Mikhail Khludnev) +* SOLR-8220: Read field from DocValues for non stored fields. + (Keith Laban, yonik, Erick Erickson, Ishan Chattopadhyaya, shalin) + Bug Fixes ---------------------- diff --git a/solr/contrib/analytics/src/test-files/solr/collection1/conf/schema-analytics.xml b/solr/contrib/analytics/src/test-files/solr/collection1/conf/schema-analytics.xml index 5e3fb713d18..0387e0904bb 100644 --- a/solr/contrib/analytics/src/test-files/solr/collection1/conf/schema-analytics.xml +++ b/solr/contrib/analytics/src/test-files/solr/collection1/conf/schema-analytics.xml @@ -25,7 +25,7 @@ --> - + - + - + diff --git a/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml b/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml index 0d8dc35a494..bc3e89daa66 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/minimr/conf/schema.xml @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml b/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml index aa0e1d816b7..e1dc3738503 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/mrunit/conf/schema.xml @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml b/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml index 29ea56e08d5..edc838c3d26 100644 --- a/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml +++ b/solr/contrib/morphlines-core/src/test-files/solr/solrcelltest/collection1/conf/schema.xml @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml index c3499be8189..45f0aafd0e6 100644 --- a/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml +++ b/solr/contrib/velocity/src/test-files/velocity/solr/collection1/conf/schema.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index 81d5bbb9880..9a535dd45de 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -61,6 +61,7 @@ public class DocsStreamer implements Iterator { private boolean onlyPseudoFields; private Set fnames; + private Set dvFieldsToReturn; private int idx = -1; public DocsStreamer(ResultContext rctx) { @@ -71,6 +72,41 @@ public class DocsStreamer implements Iterator { fnames = rctx.getReturnFields().getLuceneFieldNames(); onlyPseudoFields = (fnames == null && !rctx.getReturnFields().wantsAllFields() && !rctx.getReturnFields().hasPatternMatching()) || (fnames != null && fnames.size() == 1 && SolrReturnFields.SCORE.equals(fnames.iterator().next())); + + // add non-stored DV fields that may have been requested + if (rctx.getReturnFields().wantsAllFields()) { + // check whether there are no additional fields + Set fieldNames = rctx.getReturnFields().getLuceneFieldNames(true); + if (fieldNames == null) { + dvFieldsToReturn = rctx.getSearcher().getNonStoredDVs(true); + } else { + dvFieldsToReturn = new HashSet<>(rctx.getSearcher().getNonStoredDVs(true)); // copy + // add all requested fields that may be useDocValuesAsStored=false + for (String fl : fieldNames) { + if (rctx.getSearcher().getNonStoredDVs(false).contains(fl)) { + dvFieldsToReturn.add(fl); + } + } + } + } else { + if (rctx.getReturnFields().hasPatternMatching()) { + for (String s : rctx.getSearcher().getNonStoredDVs(true)) { + if (rctx.getReturnFields().wantsField(s)) { + if (null == dvFieldsToReturn) { + dvFieldsToReturn = new HashSet<>(); + } + dvFieldsToReturn.add(s); + } + } + } else if (fnames != null) { + dvFieldsToReturn = new HashSet<>(fnames); // copy + // here we get all non-stored dv fields because even if a user has set + // useDocValuesAsStored=false in schema, he may have requested a field + // explicitly using the fl parameter + dvFieldsToReturn.retainAll(rctx.getSearcher().getNonStoredDVs(false)); + } + } + if (transformer != null) transformer.setContext(rctx); } @@ -95,6 +131,11 @@ public class DocsStreamer implements Iterator { try { StoredDocument doc = rctx.getSearcher().doc(id, fnames); sdoc = getDoc(doc, rctx.getSearcher().getSchema()); // make sure to use the schema from the searcher and not the request (cross-core) + + // decorate the document with non-stored docValues fields + if (dvFieldsToReturn != null) { + rctx.getSearcher().decorateDocValueFields(sdoc, id, dvFieldsToReturn); + } } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document with docId " + id, e); } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java index 8498f47ddf5..fef1fea08d0 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java @@ -53,6 +53,7 @@ public abstract class FieldProperties { protected final static int DOC_VALUES = 0x00008000; protected final static int STORE_TERMPAYLOADS = 0x00010000; + protected final static int USE_DOCVALUES_AS_STORED = 0x00020000; static final String[] propertyNames = { "indexed", "tokenized", "stored", @@ -60,7 +61,7 @@ public abstract class FieldProperties { "termVectors", "termPositions", "termOffsets", "multiValued", "sortMissingFirst","sortMissingLast","required", "omitPositions", - "storeOffsetsWithPositions", "docValues", "termPayloads" + "storeOffsetsWithPositions", "docValues", "termPayloads", "useDocValuesAsStored" }; static final Map propertyMap = new HashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index cacadba5730..ea42ef8463e 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -46,8 +46,8 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.search.SortedSetSelector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.uninverting.UninvertingReader; @@ -124,7 +124,12 @@ public abstract class FieldType extends FieldProperties { return false; } - + /** + * Returns true if the fields' docValues should be used for obtaining stored value + */ + public boolean useDocValuesAsStored() { + return (properties & USE_DOCVALUES_AS_STORED) != 0; + } /** Returns true if a single field value of this type has multiple logical values * for the purposes of faceting, sorting, etc. Text fields normally return @@ -153,6 +158,7 @@ public abstract class FieldType extends FieldProperties { if (schemaVersion < 1.3) { args.remove("compressThreshold"); } + if (schemaVersion >= 1.6f) properties |= USE_DOCVALUES_AS_STORED; this.args = Collections.unmodifiableMap(args); Map initArgs = new HashMap<>(args); diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java index 27a642c7758..e38f6b62ef5 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java @@ -99,6 +99,8 @@ public final class SchemaField extends FieldProperties { public boolean omitPositions() { return (properties & OMIT_POSITIONS)!=0; } public boolean storeOffsetsWithPositions() { return (properties & STORE_OFFSETS)!=0; } + public boolean useDocValuesAsStored() { return (properties & USE_DOCVALUES_AS_STORED)!=0; } + public boolean multiValued() { return (properties & MULTIVALUED)!=0; } public boolean sortMissingFirst() { return (properties & SORT_MISSING_FIRST)!=0; } public boolean sortMissingLast() { return (properties & SORT_MISSING_LAST)!=0; } @@ -335,6 +337,7 @@ public final class SchemaField extends FieldProperties { } properties.add(getPropertyName(REQUIRED), isRequired()); properties.add(getPropertyName(TOKENIZED), isTokenized()); + properties.add(getPropertyName(USE_DOCVALUES_AS_STORED), useDocValuesAsStored()); // The BINARY property is always false // properties.add(getPropertyName(BINARY), isBinary()); } else { diff --git a/solr/core/src/java/org/apache/solr/search/ReturnFields.java b/solr/core/src/java/org/apache/solr/search/ReturnFields.java index d59afc5266a..dabfdd661ee 100644 --- a/solr/core/src/java/org/apache/solr/search/ReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/ReturnFields.java @@ -16,7 +16,7 @@ */ package org.apache.solr.search; -import java.util.*; +import java.util.Set; import org.apache.solr.response.transform.DocTransformer; @@ -35,6 +35,17 @@ public abstract class ReturnFields { */ public abstract Set getLuceneFieldNames(); + /** + * Set of field names with their exact names from the lucene index. + * + * @param ignoreWantsAll If true, it returns any additional specified field names, in spite of + * also wanting all fields. Example: when fl=*,field1, returns ["field1"]. + * If false, the method returns null when all fields are wanted. Example: when fl=*,field1, returns null. + * Note that this method returns null regardless of ignoreWantsAll if all fields + * are requested and no explicit field names are specified. + */ + public abstract Set getLuceneFieldNames(boolean ignoreWantsAll); + /** * The requested field names (includes pseudo fields) *

diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 2ff4572d23d..b2414ab2cbd 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -37,32 +37,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.document.Document; -import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.LazyDocument; +import org.apache.lucene.document.LegacyDoubleField; import org.apache.lucene.document.LegacyFloatField; import org.apache.lucene.document.LegacyIntField; -import org.apache.lucene.document.LazyDocument; import org.apache.lucene.document.LegacyLongField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.ExitableDirectoryReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiPostingsEnum; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.SlowCompositeReaderWrapper; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.index.StoredDocument; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.*; import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; @@ -70,6 +54,9 @@ import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.LegacyNumericUtils; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.common.SolrDocumentBase; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.ModifiableSolrParams; @@ -84,8 +71,15 @@ import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.EnumField; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.StrField; +import org.apache.solr.schema.TrieDateField; +import org.apache.solr.schema.TrieDoubleField; +import org.apache.solr.schema.TrieFloatField; +import org.apache.solr.schema.TrieIntField; +import org.apache.solr.schema.TrieLongField; import org.apache.solr.search.facet.UnInvertedField; import org.apache.solr.search.stats.StatsSource; import org.apache.solr.update.SolrIndexConfig; @@ -144,6 +138,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn private final FieldInfos fieldInfos; // TODO: do we need this separate set of field names? we can just use the fieldinfos? private final Collection fieldNames; + + /** + * Contains the names/patterns of all docValues=true,stored=false fields in the schema + */ + private final Set allNonStoredDVs; + + /** + * Contains the names/patterns of all docValues=true,stored=false,useDocValuesAsStored=true fields in the schema + */ + private final Set nonStoredDVsUsedAsStored; + private Collection storedHighlightFieldNames; private DirectoryFactory directoryFactory; @@ -298,11 +303,23 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn } fieldNames = new HashSet<>(); + Set nonStoredDVsUsedAsStored = new HashSet<>(); + Set allNonStoredDVs = new HashSet<>(); fieldInfos = leafReader.getFieldInfos(); for(FieldInfo fieldInfo : fieldInfos) { fieldNames.add(fieldInfo.name); + SchemaField schemaField = schema.getFieldOrNull(fieldInfo.name); + if (schemaField != null && !schemaField.stored() && schemaField.hasDocValues()) { + if (schemaField.useDocValuesAsStored()) { + nonStoredDVsUsedAsStored.add(fieldInfo.name); + } + allNonStoredDVs.add(fieldInfo.name); + } } + this.nonStoredDVsUsedAsStored = Collections.unmodifiableSet(nonStoredDVsUsedAsStored); + this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs); + // We already have our own filter cache setQueryCache(null); @@ -757,6 +774,88 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn return d; } + /** + * This will fetch and add the docValues fields to a given SolrDocument/SolrInputDocument + * + * @param doc A SolrDocument or SolrInputDocument instance where docValues will be added + * @param docid The lucene docid of the document to be populated + * @param fields The list of docValues fields to be decorated + */ + @SuppressWarnings("deprecation") + public void decorateDocValueFields(@SuppressWarnings("rawtypes") SolrDocumentBase doc, int docid, + Set fields) throws IOException { + for (String fieldName : fields) { + SchemaField schemaField = schema.getFieldOrNull(fieldName); + if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) { + log.warn("Couldn't decorate docValues for field: {}, schemaField: {}", fieldName, schemaField); + continue; + } + + if (schemaField.multiValued()) { + SortedSetDocValues values = getLeafReader().getSortedSetDocValues(fieldName); + if (values != null && values.getValueCount() > 0 && + DocValues.getDocsWithField(leafReader, fieldName).get(docid)) { + values.setDocument(docid); + List outValues = new LinkedList(); + for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) { + if (schemaField.getType() instanceof TrieIntField) { + outValues.add(LegacyNumericUtils.prefixCodedToInt(values.lookupOrd(ord))); + } else if (schemaField.getType() instanceof TrieLongField) { + outValues.add(LegacyNumericUtils.prefixCodedToLong(values.lookupOrd(ord))); + } else if (schemaField.getType() instanceof TrieFloatField) { + outValues.add(LegacyNumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(values.lookupOrd(ord)))); + } else if (schemaField.getType() instanceof TrieDoubleField) { + outValues.add(LegacyNumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(values.lookupOrd(ord)))); + } else if (schemaField.getType() instanceof TrieDateField) { + outValues.add(new Date(LegacyNumericUtils.prefixCodedToLong(values.lookupOrd(ord)))); + } else if (schemaField.getType() instanceof EnumField) { + outValues.add(((EnumField) schemaField.getType()).intValueToStringValue( + LegacyNumericUtils.prefixCodedToInt(values.lookupOrd(ord)))); + } else if (schemaField.getType() instanceof StrField) { + outValues.add(values.lookupOrd(ord).utf8ToString()); + } + } + if (outValues.size() > 0) + doc.addField(fieldName, outValues); + } + } else { + DocValuesType dvType = fieldInfos.fieldInfo(fieldName).getDocValuesType(); + switch (dvType) { + case NUMERIC: + if (DocValues.getDocsWithField(leafReader, fieldName).get(docid)) { + NumericDocValues ndv = leafReader.getNumericDocValues(fieldName); + Object val = ndv.get(docid); + if (schemaField.getType() instanceof TrieIntField) { + val = ((Long) val).intValue(); + } else if (schemaField.getType() instanceof TrieFloatField) { + val = NumericUtils.sortableIntToFloat(((Long) val).intValue()); + } else if (schemaField.getType() instanceof TrieDoubleField) { + val = NumericUtils.sortableLongToDouble((long) val); + } else if (schemaField.getType() instanceof TrieDateField) { + val = new Date((long) val); + } else if (schemaField.getType() instanceof EnumField) { + val = ((EnumField) schemaField.getType()).intValueToStringValue(((Long) val).intValue()); + } + doc.addField(fieldName, val); + } + break; + case BINARY: + if (DocValues.getDocsWithField(leafReader, fieldName).get(docid)) { + BinaryDocValues bdv = leafReader.getBinaryDocValues(fieldName); + doc.addField(fieldName, bdv.get(docid)); + } + break; + case SORTED: + SortedDocValues sdv = leafReader.getSortedDocValues(fieldName); + if (sdv.getOrd(docid) >= 0) { + doc.addField(fieldName, sdv.get(docid).utf8ToString()); + } + break; + } + } + } + } + /** * Takes a list of docs (the doc ids actually), and reads them into an array * of Documents. @@ -775,6 +874,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn } } + /** + * Returns an unmodifiable set of non-stored docValues field names. + * + * @param onlyUseDocValuesAsStored If false, returns all non-stored docValues. + * If true, returns only those non-stored docValues + * which have the {@link SchemaField#useDocValuesAsStored()} flag true. + */ + public Set getNonStoredDVs(boolean onlyUseDocValuesAsStored) { + return onlyUseDocValuesAsStored ? nonStoredDVsUsedAsStored : allNonStoredDVs; + } + /* ********************** end document retrieval *************************/ //////////////////////////////////////////////////////////////////////////////// diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java index f50120d7422..06e2a8aed14 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java @@ -398,7 +398,16 @@ public class SolrReturnFields extends ReturnFields { @Override public Set getLuceneFieldNames() { - return (_wantsAllFields || fields.isEmpty()) ? null : fields; + return getLuceneFieldNames(false); + } + + @Override + public Set getLuceneFieldNames(boolean ignoreWantsAll) + { + if (ignoreWantsAll) + return fields; + else + return (_wantsAllFields || fields.isEmpty()) ? null : fields; } @Override diff --git a/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml b/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml index 9a6733d97e8..e9b508c5621 100644 --- a/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml +++ b/solr/core/src/test-files/solr/analysisconfs/analysis-err-schema.xml @@ -20,7 +20,7 @@ text field that blows up in analysis, and an ID field for diagnosis. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-analysis-parameters.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-analysis-parameters.xml index 3f8e224ce1b..2f8963f1239 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-analysis-parameters.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-analysis-parameters.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-field-parameters.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-field-parameters.xml index 3575c438c72..6516875c199 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-field-parameters.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-bogus-field-parameters.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-init-error.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-init-error.xml index e18b048aed9..85a97ebe45e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-init-error.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-init-error.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-dest-should-fail-test.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-dest-should-fail-test.xml index a0496664b7a..83b4b4ef0c7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-dest-should-fail-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-dest-should-fail-test.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-source-should-fail-test.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-source-should-fail-test.xml index 27939f4e719..e29e351ad24 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-source-should-fail-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-misplaced-asterisk-copyfield-source-should-fail-test.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-dest-should-fail-test.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-dest-should-fail-test.xml index a3187f11a87..650d64cc2ee 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-dest-should-fail-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-dest-should-fail-test.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-source-should-fail-test.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-source-should-fail-test.xml index 1be1deff0e8..6737b1f3eb1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-source-should-fail-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-multiple-asterisk-copyfield-source-should-fail-test.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-non-glob-copyfield-source-matching-nothing-should-fail-test.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-non-glob-copyfield-source-matching-nothing-should-fail-test.xml index 4b35cf2598f..621ef296744 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-non-glob-copyfield-source-matching-nothing-should-fail-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-non-glob-copyfield-source-matching-nothing-should-fail-test.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml index 5f4d69a31a7..552e27d2f4a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-HighlighterMaxOffsetTest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-HighlighterMaxOffsetTest.xml index d091fa00771..ab5c7e8fa4a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-HighlighterMaxOffsetTest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-HighlighterMaxOffsetTest.xml @@ -21,7 +21,7 @@ Test for HighlighterMaxOffsetTest which requires the use of ReversedWildcardFilterFactory --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml b/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml index c1c0d2ee63f..16949d19678 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-add-schema-fields-update-processor.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-behavior.xml b/solr/core/src/test-files/solr/collection1/conf/schema-behavior.xml index 20b5a3533b9..474ea2983bf 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-behavior.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-behavior.xml @@ -27,6 +27,7 @@ + @@ -42,80 +43,94 @@ - + autoGeneratePhraseQueries="true"/> + + - - - + + + + - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + - - - - - - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-class-name-shortening-on-serialization.xml b/solr/core/src/test-files/solr/collection1/conf/schema-class-name-shortening-on-serialization.xml index 4f900451fec..82a6709e9ee 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-class-name-shortening-on-serialization.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-class-name-shortening-on-serialization.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml index d40d7ff1531..a0551d3cd08 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml index 792eea60542..9b1d3d679e6 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distrib-interval-faceting.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml index d6881c910a2..4fe84d281df 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml index ea81b8fa58d..851ff658a9b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -25,7 +25,7 @@ --> - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml index 126551bb402..8c848d8824d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml @@ -16,14 +16,26 @@ limitations under the License. --> - + + - + @@ -86,4 +98,5 @@ + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml index b6ef570ef1b..c87a1f4de6d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml @@ -18,7 +18,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml index ec392aad32e..f0a4007abf5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml b/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml index 60cab4f8601..46f89cdb325 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-eff.xml @@ -17,7 +17,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml b/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml index f4f06049ca3..3fdce4721a1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-field-sort-values.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-graph.xml b/solr/core/src/test-files/solr/collection1/conf/schema-graph.xml index 3f226d10d48..a75ce663818 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-graph.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-graph.xml @@ -13,7 +13,7 @@ - + - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml new file mode 100644 index 00000000000..aab4da4e2e8 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest-lucene-match-version.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest-lucene-match-version.xml index e4150fab149..dd9ded77a6f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-rest-lucene-match-version.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest-lucene-match-version.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml index d5b8b1533a8..e52473bfdb0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml @@ -16,9 +16,9 @@ limitations under the License. --> - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-single-dynamic-copy-field.xml b/solr/core/src/test-files/solr/collection1/conf/schema-single-dynamic-copy-field.xml index 204cc815a43..3b514dd1b81 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-single-dynamic-copy-field.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-single-dynamic-copy-field.xml @@ -16,9 +16,9 @@ limitations under the License. --> - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml index e9d8f3db475..a264c014fb4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml index 372305daed5..ec2d73c4367 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml @@ -22,7 +22,7 @@ NOTE: Tests expect every field in this schema to be sortable. --> - + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml index 216fa2ce85b..25a9bc074d8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml @@ -25,7 +25,7 @@ --> - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml index 9578a38dc64..214b4867770 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-xinclude.xml b/solr/core/src/test-files/solr/collection1/conf/schema-xinclude.xml index 94194df6192..e8bb5be3a4b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-xinclude.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-xinclude.xml @@ -18,7 +18,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index 37bc618a2ca..ea46d85553d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -16,9 +16,9 @@ limitations under the License. --> - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index cf8666f2f58..fc8f7c11dd1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java index 3509e8d7cb2..570e81082de 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java @@ -24,7 +24,7 @@ public class TestFieldResource extends SolrRestletTestBase { public void testGetField() throws Exception { assertQ("/schema/fields/test_postv?indent=on&wt=xml&showDefaults=true", "count(/response/lst[@name='field']) = 1", - "count(/response/lst[@name='field']/*) = 16", + "count(/response/lst[@name='field']/*) = 17", "/response/lst[@name='field']/str[@name='name'] = 'test_postv'", "/response/lst[@name='field']/str[@name='type'] = 'text'", "/response/lst[@name='field']/bool[@name='indexed'] = 'true'", @@ -40,7 +40,8 @@ public class TestFieldResource extends SolrRestletTestBase { "/response/lst[@name='field']/bool[@name='storeOffsetsWithPositions'] = 'false'", "/response/lst[@name='field']/bool[@name='multiValued'] = 'false'", "/response/lst[@name='field']/bool[@name='required'] = 'false'", - "/response/lst[@name='field']/bool[@name='tokenized'] = 'true'"); + "/response/lst[@name='field']/bool[@name='tokenized'] = 'true'", + "/response/lst[@name='field']/bool[@name='useDocValuesAsStored'] = 'true'"); } @Test diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java index fb59f1f7d41..dc0bc47ff36 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaResource.java @@ -29,7 +29,7 @@ public class TestSchemaResource extends SolrRestletTestBase { "/response/lst[@name='schema']/str[@name='name'][.='test-rest']", "count(/response/lst[@name='schema']/float[@name='version']) = 1", - "/response/lst[@name='schema']/float[@name='version'][.='1.5']", + "/response/lst[@name='schema']/float[@name='version'][.='1.6']", "count(/response/lst[@name='schema']/lst[@name='solrQueryParser']/str[@name='defaultOperator']) = 1", "/response/lst[@name='schema']/lst[@name='solrQueryParser']/str[@name='defaultOperator'][.='OR']", @@ -114,7 +114,7 @@ public class TestSchemaResource extends SolrRestletTestBase { assertJQ("/schema?wt=json", // Should work with or without a trailing slash "/schema/name=='test-rest'", - "/schema/version==1.5", + "/schema/version==1.6", "/schema/solrQueryParser/defaultOperator=='OR'", "/schema/uniqueKey=='id'", "/schema/defaultSearchField=='text'", @@ -155,7 +155,7 @@ public class TestSchemaResource extends SolrRestletTestBase { assertQ("/schema?wt=schema.xml", // should work with or without trailing slash on '/schema/' path "/schema/@name = 'test-rest'", - "/schema/@version = '1.5'", + "/schema/@version = '1.6'", "/schema/solrQueryParser/@defaultOperator = 'OR'", "/schema/uniqueKey = 'id'", "/schema/defaultSearchField = 'text'", diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaVersionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaVersionResource.java index 4dff32041bd..645843e9dac 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaVersionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestSchemaVersionResource.java @@ -24,7 +24,7 @@ public class TestSchemaVersionResource extends SolrRestletTestBase { public void testGetSchemaVersion() throws Exception { assertQ("/schema/version?indent=on&wt=xml", "count(/response/float[@name='version']) = 1", - "/response/float[@name='version'][.='1.5']"); + "/response/float[@name='version'][.='1.6']"); } } diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java index d1b04f5a8ab..2cdc7317b0f 100644 --- a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java +++ b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java @@ -23,7 +23,7 @@ import org.apache.solr.SolrTestCaseJ4; public class SchemaVersionSpecificBehaviorTest extends SolrTestCaseJ4 { public void testVersionBehavior() throws Exception { - for (float v : new float[] { 1.0F, 1.1F, 1.2F, 1.3F, 1.4F, 1.5F }) { + for (float v : new float[] { 1.0F, 1.1F, 1.2F, 1.3F, 1.4F, 1.5F, 1.6F }) { try { final IndexSchema schema = initCoreUsingSchemaVersion(v); final String ver = String.valueOf(v); @@ -32,7 +32,8 @@ public class SchemaVersionSpecificBehaviorTest extends SolrTestCaseJ4 { // have any properties set on them for (String f : new String[] { "text", "xx_dyn_text", "bool", "xx_dyn_bool", - "str", "xx_dyn_str" }) { + "str", "xx_dyn_str", + "int", "xx_dyn_int"}) { SchemaField field = schema.getField(f); @@ -59,6 +60,11 @@ public class SchemaVersionSpecificBehaviorTest extends SolrTestCaseJ4 { ( v < 1.5F ? false : ! (field.getType() instanceof TextField)), field.omitNorms()); + + // 1.6: useDocValuesAsStored defaults to true + assertEquals(f + " field's type has wrong useDocValuesAsStored for ver=" + ver, + ( v < 1.6F ? false : true), + field.useDocValuesAsStored()); } // regardless of version, explicit multiValued values on field or type @@ -90,6 +96,34 @@ public class SchemaVersionSpecificBehaviorTest extends SolrTestCaseJ4 { } } + // regardless of version, explicit useDocValuesAsStored values on field or type + // should be correct + for (String f : new String[] { "ft_intdvas_f", "ft_intdvas_t", + "intdvas_f", "intdvas_t", + "xx_dyn_ft_intdvas_f", "xx_dyn_ft_intdvas_f", + "xx_dyn_intdvas_f", "xx_dyn_intdvas_f"}) { + + boolean expected = f.endsWith("dvas_t"); + SchemaField field = schema.getField(f); + assertEquals(f + " field's useDocValuesAsStored is wrong for ver=" + ver, + expected, field.useDocValuesAsStored()); + + FieldType ft = field.getType(); + if (f.contains("ft_")) { + // sanity check that we really are inheriting from fieldtype + assertEquals(f + " field's omitTfP doesn't match type for ver=" + ver, + expected, ft.hasProperty(FieldType.USE_DOCVALUES_AS_STORED)); + } else { + // for fields where the property is explicit, make sure + // we aren't getting a false negative because someone changed the + // schema and we're inheriting from fieldType + assertEquals(f + " field's type has wrong useDocValuesAsStored for ver=" + ver, + ( v < 1.6F ? false : true), + ft.hasProperty(FieldType.USE_DOCVALUES_AS_STORED)); + + } + } + // regardless of version, explicit omitTfP values on field or type // should be correct for (String f : new String[] { "strTfP_f", "strTfP_t", diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java new file mode 100644 index 00000000000..e3914c0b4b7 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java @@ -0,0 +1,242 @@ +package org.apache.solr.schema; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; + +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.core.AbstractBadConfigTestBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Tests the useDocValuesAsStored functionality. + */ +public class TestUseDocValuesAsStored extends AbstractBadConfigTestBase { + + private int id = 1; + + private static File tmpSolrHome; + private static File tmpConfDir; + + private static final String collection = "collection1"; + private static final String confDir = collection + "/conf"; + + @Before + private void initManagedSchemaCore() throws Exception { + tmpSolrHome = createTempDir().toFile(); + tmpConfDir = new File(tmpSolrHome, confDir); + File testHomeConfDir = new File(TEST_HOME(), confDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig-managed-schema.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig-basic.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "solrconfig.snippet.randomindexconfig.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-one-field-no-dynamic-field.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-one-field-no-dynamic-field-unique-key.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "enumsConfig.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-non-stored-docvalues.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-minimal.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema_codec.xml"), tmpConfDir); + FileUtils.copyFileToDirectory(new File(testHomeConfDir, "schema-bm25.xml"), tmpConfDir); + + // initCore will trigger an upgrade to managed schema, since the solrconfig has + // + System.setProperty("enable.update.log", "false"); + System.setProperty("managed.schema.mutable", "true"); + initCore("solrconfig-managed-schema.xml", "schema-non-stored-docvalues.xml", tmpSolrHome.getPath()); + } + + @After + private void afterClass() throws Exception { + deleteCore(); + System.clearProperty("managed.schema.mutable"); + System.clearProperty("enable.update.log"); + } + + + public String getCoreName() { + return "basic"; + } + + @Test + public void testOnEmptyIndex() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + assertJQ(req("q", "*:*"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "*"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "test_nonstored_dv_str"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "*,test_nonstored_dv_str"), "/response/numFound==0"); + + assertU(adoc("id", "xyz", "test_nonstored_dv_str", "xyz")); + assertJQ(req("q", "*:*"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "*"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "test_nonstored_dv_str"), "/response/numFound==0"); + assertJQ(req("q", "*:*", "fl", "*,test_nonstored_dv_str"), "/response/numFound==0"); + + assertU(commit()); + assertJQ(req("q", "*:*"), "/response/numFound==1", + "/response/docs==[" + + "{'id':'xyz','test_nonstored_dv_str':'xyz'}" + + "]"); + assertJQ(req("q", "*:*", "fl", "*"), "/response/numFound==1", + "/response/docs==[" + + "{'id':'xyz','test_nonstored_dv_str':'xyz'}" + + "]"); + assertJQ(req("q", "*:*", "fl", "test_nonstored_dv_str"), "/response/numFound==1", + "/response/docs==[" + + "{'test_nonstored_dv_str':'xyz'}" + + "]"); + assertJQ(req("q", "*:*", "fl", "*,test_nonstored_dv_str"), "/response/numFound==1", + "/response/docs==[" + + "{'id':'xyz','test_nonstored_dv_str':'xyz'}" + + "]"); + + assertU(adoc("id", "xyz")); + assertU(commit()); + assertJQ(req("q", "*:*"), "/response/numFound==1", + "/response/docs==[" + + "{'id':'xyz'}" + + "]"); + } + + @Test + public void testSinglyValued() throws IOException { + clearIndex(); + doTest("check string value is correct", "test_s_dvo", "str", "keyword"); + doTest("check int value is correct", "test_i_dvo", "int", "1234"); + doTest("check double value is correct", "test_d_dvo", "double", "1.234"); + doTest("check long value is correct", "test_l_dvo", "long", "12345"); + doTest("check float value is correct", "test_f_dvo", "float", "1.234"); + doTest("check dt value is correct", "test_dt_dvo", "date", "1976-07-04T12:08:56.235Z"); + doTest("check stored and docValues value is correct", "test_s_dv", "str", "storedAndDocValues"); + doTest("check non-stored and non-indexed is accessible", "test_s_dvo2", "str", "gotIt"); + doTest("enumField", "enum_dvo", "str", "Critical"); + } + + @Test + public void testMultiValued() throws IOException { + clearIndex(); + doTest("check string value is correct", "test_ss_dvo", "str", "keyword", "keyword2"); + doTest("check int value is correct", "test_is_dvo", "int", "1234", "12345"); + doTest("check double value is correct", "test_ds_dvo", "double", "1.234", "12.34", "123.4"); + doTest("check long value is correct", "test_ls_dvo", "long", "12345", "123456"); + doTest("check float value is correct", "test_fs_dvo", "float", "1.234", "12.34"); + doTest("check dt value is correct", "test_dts_dvo", "date", "1976-07-04T12:08:56.235Z", "1978-07-04T12:08:56.235Z"); + doTest("check stored and docValues value is correct", "test_ss_dv", "str", "storedAndDocValues", "storedAndDocValues2"); + doTest("check non-stored and non-indexed is accessible", "test_ss_dvo2", "str", "gotIt", "gotIt2"); + doTest("enumField", "enums_dvo", "str", "High", "Critical"); + } + + @Test + public void testMultipleSearchResults() throws Exception { + + // Three documents with different numbers of values for a field + assertU(adoc("id", "myid1", "test_is_dvo", "101", "test_is_dvo", "102", "test_is_dvo", "103")); + assertU(adoc("id", "myid2", "test_is_dvo", "201", "test_is_dvo", "202")); + assertU(adoc("id", "myid3", "test_is_dvo", "301", "test_is_dvo", "302", + "test_is_dvo", "303", "test_is_dvo", "304")); + + // Multivalued and singly valued fields in the same document + assertU(adoc("id", "myid4", "test_s_dvo", "hello", "test_is_dvo", "401", "test_is_dvo", "402")); + + // Test a field which has useDocValuesAsStored=false + assertU(adoc("id", "myid5", "nonstored_dv_str", "dont see me")); + assertU(adoc("id", "myid6", "nonstored_dv_str", "dont see me", "test_s_dvo", "hello")); + assertU(commit()); + + assertJQ(req("q", "id:myid*", "fl", "*"), + "/response/docs==[" + + "{'id':'myid1','test_is_dvo':[101,102,103]}," + + "{'id':'myid2','test_is_dvo':[201,202]}," + + "{'id':'myid3','test_is_dvo':[301,302,303,304]}," + + "{'id':'myid4','test_s_dvo':'hello','test_is_dvo':[401,402]}," + + "{'id':'myid5'}," + + "{'id':'myid6','test_s_dvo':'hello'}" + + "]"); + } + + public void testManagedSchema() throws Exception { + IndexSchema oldSchema = h.getCore().getLatestSchema(); + StrField type = new StrField(); + type.setTypeName("str"); + SchemaField falseDVASField = new SchemaField("false_dvas", type, + SchemaField.INDEXED | SchemaField.DOC_VALUES, null); + SchemaField trueDVASField = new SchemaField("true_dvas", type, + SchemaField.INDEXED | SchemaField.DOC_VALUES | SchemaField.USE_DOCVALUES_AS_STORED, null); + IndexSchema newSchema = oldSchema.addField(falseDVASField).addField(trueDVASField); + h.getCore().setLatestSchema(newSchema); + + clearIndex(); + assertU(adoc("id", "myid1", "false_dvas", "101", "true_dvas", "102")); + assertU(commit()); + + assertJQ(req("q", "id:myid*", "fl", "*"), + "/response/docs==[" + + "{'id':'myid1', 'true_dvas':'102'}]"); + } + + private void doTest(String desc, String field, String type, String... value) { + String id = "" + this.id++; + + + String[] xpaths = new String[value.length + 1]; + + if (value.length > 1) { + String[] fieldAndValues = new String[value.length * 2 + 2]; + fieldAndValues[0] = "id"; + fieldAndValues[1] = id; + + for (int i = 0; i < value.length; ++i) { + fieldAndValues[i * 2 + 2] = field; + fieldAndValues[i * 2 + 3] = value[i]; + xpaths[i] = "//arr[@name='" + field + "']/" + type + "[" + (i + 1) + "][.='" + value[i] + "']"; + } + + xpaths[value.length] = "*[count(//arr[@name='" + field + "']/" + type + ") = " + value.length + "]"; + assertU(adoc(fieldAndValues)); + + } else { + assertU(adoc("id", id, field, value[0])); + xpaths[0] = "//" + type + "[@name='" + field + "'][.='" + value[0] + "']"; + xpaths[1] = "*[count(//" + type + "[@name='" + field + "']) = 1]"; + } + + assertU(commit()); + + String fl = field; + assertQ(desc + ": " + fl, req("q", "id:" + id, "fl", fl), xpaths); + + fl = field + ",*"; + assertQ(desc + ": " + fl, req("q", "id:" + id, "fl", fl), xpaths); + + fl = "*" + field.substring(field.length() - 3); + assertQ(desc + ": " + fl, req("q", "id:" + id, "fl", fl), xpaths); + + fl = "*"; + assertQ(desc + ": " + fl, req("q", "id:" + id, "fl", fl), xpaths); + + fl = field + ",fakeFieldName"; + assertQ(desc + ": " + fl, req("q", "id:" + id, "fl", fl), xpaths); + + fl = "*"; + assertQ(desc + ": " + fl, req("q", "*:*", "fl", fl), xpaths); + + } +} diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored2.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored2.java new file mode 100644 index 00000000000..3011141f501 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored2.java @@ -0,0 +1,185 @@ +package org.apache.solr.schema; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.StringReader; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.solr.rest.schema.TestBulkSchemaAPI; +import org.apache.solr.util.RestTestBase; +import org.apache.solr.util.RestTestHarness; +import org.junit.After; +import org.junit.Before; +import org.noggit.JSONParser; +import org.noggit.ObjectBuilder; + +/** + * Tests the useDocValuesAsStored functionality. + */ +public class TestUseDocValuesAsStored2 extends RestTestBase { + + @Before + public void before() throws Exception { + File tmpSolrHome = createTempDir().toFile(); + FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile()); + + System.setProperty("managed.schema.mutable", "true"); + System.setProperty("enable.update.log", "false"); + + createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml", + "/solr", true, null); + } + + @After + public void after() throws Exception { + if (jetty != null) { + jetty.stop(); + jetty = null; + } + client = null; + if (restTestHarness != null) { + restTestHarness.close(); + } + restTestHarness = null; + } + + + public void testSchemaAPI() throws Exception { + RestTestHarness harness = restTestHarness; + String payload = "{\n" + + " 'add-field' : {\n" + + " 'name':'a1',\n" + + " 'type': 'string',\n" + + " 'stored':false,\n" + + " 'docValues':true,\n" + + " 'indexed':false\n" + + " },\n" + + " 'add-field' : {\n" + + " 'name':'a2',\n" + + " 'type': 'string',\n" + + " 'stored':false,\n" + + " 'useDocValuesAsStored':true,\n" + + " 'docValues':true,\n" + + " 'indexed':true\n" + + " },\n" + + " 'add-field' : {\n" + + " 'name':'a3',\n" + + " 'type': 'string',\n" + + " 'stored':false,\n" + + " 'useDocValuesAsStored':false,\n" + + " 'docValues':true,\n" + + " 'indexed':true\n" + + " }\n" + + " }\n"; + + String response = harness.post("/schema?wt=json", json(payload)); + + Map m = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response))); + assertNull(response, m.get("errors")); + + // default value of useDocValuesAsStored + m = TestBulkSchemaAPI.getObj(harness, "a1", "fields"); + assertNotNull("field a1 not created", m); + assertNull(m.get("useDocValuesAsStored")); + + // useDocValuesAsStored=true + m = TestBulkSchemaAPI.getObj(harness,"a2", "fields"); + assertNotNull("field a2 not created", m); + assertEquals(Boolean.TRUE, m.get("useDocValuesAsStored")); + + // useDocValuesAsStored=false + m = TestBulkSchemaAPI.getObj(harness,"a3", "fields"); + assertNotNull("field a3 not created", m); + assertEquals(Boolean.FALSE, m.get("useDocValuesAsStored")); + + // Index documents to check the effect + assertU(adoc("id", "myid1", "a1", "1", "a2", "2", "a3", "3")); + assertU(commit()); + + RestTestBase.assertJQ("/select?q=id:myid*&fl=*", + "/response/docs==[{'id':'myid1', 'a1':'1', 'a2':'2'}]"); + + RestTestBase.assertJQ("/select?q=id:myid*&fl=id,a1,a2,a3", + "/response/docs==[{'id':'myid1', 'a1':'1', 'a2':'2', 'a3':'3'}]"); + + RestTestBase.assertJQ("/select?q=id:myid*&fl=a3", + "/response/docs==[{'a3':'3'}]"); + + // this will return a3 because it is explicitly requested even if '*' is specified + RestTestBase.assertJQ("/select?q=id:myid*&fl=*,a3", + "/response/docs==[{'id':'myid1', 'a1':'1', 'a2':'2', 'a3':'3'}]"); + + // this will not return a3 because the glob 'a*' will match only stored + useDocValuesAsStored=true fields + RestTestBase.assertJQ("/select?q=id:myid*&fl=id,a*", + "/response/docs==[{'id':'myid1', 'a1':'1', 'a2':'2'}]"); + + // Test replace-field + // Explicitly set useDocValuesAsStored to false + payload = "{\n" + + " 'replace-field' : {\n" + + " 'name':'a1',\n" + + " 'type': 'string',\n" + + " 'stored':false,\n" + + " 'useDocValuesAsStored':false,\n" + + " 'docValues':true,\n" + + " 'indexed':false\n" + + " }}"; + response = harness.post("/schema?wt=json", json(payload)); + m = TestBulkSchemaAPI.getObj(harness, "a1", "fields"); + assertNotNull("field a1 doesn't exist any more", m); + assertEquals(Boolean.FALSE, m.get("useDocValuesAsStored")); + + // Explicitly set useDocValuesAsStored to true + payload = "{\n" + + " 'replace-field' : {\n" + + " 'name':'a1',\n" + + " 'type': 'string',\n" + + " 'stored':false,\n" + + " 'useDocValuesAsStored':true,\n" + + " 'docValues':true,\n" + + " 'indexed':false\n" + + " }}"; + response = harness.post("/schema?wt=json", json(payload)); + m = TestBulkSchemaAPI.getObj(harness, "a1", "fields"); + assertNotNull("field a1 doesn't exist any more", m); + assertEquals(Boolean.TRUE, m.get("useDocValuesAsStored")); + + // add a field which is stored as well as docvalues + payload = "{ 'add-field' : {\n" + + " 'name':'a4',\n" + + " 'type': 'string',\n" + + " 'stored':true,\n" + + " 'useDocValuesAsStored':true,\n" + + " 'docValues':true,\n" + + " 'indexed':true\n" + + " }}"; + response = harness.post("/schema?wt=json", json(payload)); + m = TestBulkSchemaAPI.getObj(harness, "a4", "fields"); + assertNotNull("field a4 not found", m); + assertEquals(Boolean.TRUE, m.get("useDocValuesAsStored")); + + assertU(adoc("id", "myid1", "a1", "1", "a2", "2", "a3", "3", "a4", "4")); + assertU(commit()); + + RestTestBase.assertJQ("/select?q=id:myid*&fl=*", + "/response/docs==[{'id':'myid1', 'a1':'1', 'a2':'2', 'a4':'4'}]"); + + } +} diff --git a/solr/example/example-DIH/solr/db/conf/managed-schema b/solr/example/example-DIH/solr/db/conf/managed-schema index 37e37b9e3c4..156c7767a77 100644 --- a/solr/example/example-DIH/solr/db/conf/managed-schema +++ b/solr/example/example-DIH/solr/db/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/example/example-DIH/solr/mail/conf/managed-schema b/solr/example/example-DIH/solr/mail/conf/managed-schema index 859d7bb79a8..dc1ec6c4c04 100644 --- a/solr/example/example-DIH/solr/mail/conf/managed-schema +++ b/solr/example/example-DIH/solr/mail/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/example/example-DIH/solr/rss/conf/managed-schema b/solr/example/example-DIH/solr/rss/conf/managed-schema index 7a0f81091e5..790e97a2469 100644 --- a/solr/example/example-DIH/solr/rss/conf/managed-schema +++ b/solr/example/example-DIH/solr/rss/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/example/example-DIH/solr/solr/conf/managed-schema b/solr/example/example-DIH/solr/solr/conf/managed-schema index 5bd1791c399..a714be5b46e 100644 --- a/solr/example/example-DIH/solr/solr/conf/managed-schema +++ b/solr/example/example-DIH/solr/solr/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/example/example-DIH/solr/tika/conf/managed-schema b/solr/example/example-DIH/solr/tika/conf/managed-schema index 5f5b1e6335d..daf08c741f7 100644 --- a/solr/example/example-DIH/solr/tika/conf/managed-schema +++ b/solr/example/example-DIH/solr/tika/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/example/files/conf/managed-schema b/solr/example/files/conf/managed-schema index 24880b06690..e936bcde4e3 100644 --- a/solr/example/files/conf/managed-schema +++ b/solr/example/files/conf/managed-schema @@ -1,6 +1,6 @@ - + id @@ -514,4 +514,4 @@ - \ No newline at end of file + diff --git a/solr/server/solr/configsets/basic_configs/conf/managed-schema b/solr/server/solr/configsets/basic_configs/conf/managed-schema index 23d19977580..2599a28dffd 100644 --- a/solr/server/solr/configsets/basic_configs/conf/managed-schema +++ b/solr/server/solr/configsets/basic_configs/conf/managed-schema @@ -29,7 +29,7 @@ http://wiki.apache.org/solr/SchemaXml --> - + diff --git a/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema b/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema index 49c9c5b33d4..822e2074a3f 100644 --- a/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema +++ b/solr/server/solr/configsets/data_driven_schema_configs/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + + +false diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema index 3929b8b7c7e..8583fc54b31 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema @@ -45,7 +45,7 @@ that avoids logging every request --> - + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml index 216fa2ce85b..25a9bc074d8 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml @@ -25,7 +25,7 @@ --> - + - + - +