Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

drop old style Lucene compression support - but allow older configs to still work 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/branches/solr@923411 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2010-03-15 19:56:25 +00:00
parent 6be973b1a4
commit b0f25360bf
9 changed files with 22 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
example/solr/conf/schema.xml
View File

@ -45,7 +45,7 @@
that avoids logging every request
-->
<schema name="example" version="1.2">
<schema name="example" version="1.3">
<!-- attribute "name" is the name of this schema and is only used for display purposes.
Applications should change this to reflect the nature of the search collection.
version="1.2" is Solr's version number for the schema syntax and semantics. It should
@ -53,6 +53,7 @@
1.0: multiValued attribute did not exist, all fields are multiValued by nature
1.1: multiValued attribute introduced, false by default
1.2: omitTermFreqAndPositions attribute introduced, true by default except for text fields.
1.3: removed optional field compress feature
-->
<types>
@ -64,11 +65,7 @@
org.apache.solr.analysis package.
-->
<!-- The StrField type is not analyzed, but indexed/stored verbatim.
- StrField and TextField support an optional compressThreshold which
limits compression (if enabled in the derived fields) to values which
exceed a certain size (in characters).
-->
<!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<!-- boolean type: "true" or "false" -->
@ -436,9 +433,6 @@
<types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
compressed: [false] if this field should be stored using gzip compression
(this will only apply if the field type is compressable; among
the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time

1
src/common/org/apache/solr/common/luke/FieldFlag.java
View File

@ -33,7 +33,6 @@ public enum FieldFlag {
OMIT_TF('F', "Omit Tf"),
LAZY('L', "Lazy"),
BINARY('B', "Binary"),
COMPRESSED('C', "Compressed"),
SORT_MISSING_FIRST('f', "Sort Missing First"),
SORT_MISSING_LAST('l', "Sort Missing Last");

4
src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
View File

@ -172,8 +172,6 @@ public class LukeRequestHandler extends RequestHandlerBase
flags.append( (f != null && f.getOmitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' );
flags.append( (f != null && f.isLazy()) ? FieldFlag.LAZY.getAbbreviation() : '-' );
flags.append( (f != null && f.isBinary()) ? FieldFlag.BINARY.getAbbreviation() : '-' );
//nocommit: handle compressed
//flags.append( (f != null && f.isCompressed()) ? FieldFlag.COMPRESSED.getAbbreviation() : '-' );
flags.append( (false) ? FieldFlag.SORT_MISSING_FIRST.getAbbreviation() : '-' ); // SchemaField Specific
flags.append( (false) ? FieldFlag.SORT_MISSING_LAST.getAbbreviation() : '-' ); // SchemaField Specific
return flags.toString();
@ -202,7 +200,6 @@ public class LukeRequestHandler extends RequestHandlerBase
flags.append( (f != null && f.omitTf()) ? FieldFlag.OMIT_TF.getAbbreviation() : '-' );
flags.append( (lazy) ? FieldFlag.LAZY.getAbbreviation() : '-' );
flags.append( (binary) ? FieldFlag.BINARY.getAbbreviation() : '-' );
flags.append( (f != null && f.isCompressed()) ? FieldFlag.COMPRESSED.getAbbreviation() : '-' );
flags.append( (f != null && f.sortMissingFirst() ) ? FieldFlag.SORT_MISSING_FIRST.getAbbreviation() : '-' );
flags.append( (f != null && f.sortMissingLast() ) ? FieldFlag.SORT_MISSING_LAST.getAbbreviation() : '-' );
return flags.toString();
@ -224,7 +221,6 @@ public class LukeRequestHandler extends RequestHandlerBase
key.add( String.valueOf(FieldFlag.OMIT_NORMS.getAbbreviation()), FieldFlag.OMIT_NORMS.getDisplay() );
key.add( String.valueOf(FieldFlag.LAZY.getAbbreviation()), FieldFlag.LAZY.getDisplay() );
key.add( String.valueOf(FieldFlag.BINARY.getAbbreviation()), FieldFlag.BINARY.getDisplay() );
key.add( String.valueOf(FieldFlag.COMPRESSED.getAbbreviation()), FieldFlag.COMPRESSED.getDisplay() );
key.add( String.valueOf(FieldFlag.SORT_MISSING_FIRST.getAbbreviation()), FieldFlag.SORT_MISSING_FIRST.getDisplay() );
key.add( String.valueOf(FieldFlag.SORT_MISSING_LAST.getAbbreviation()), FieldFlag.SORT_MISSING_LAST.getDisplay() );
return key;

68
src/java/org/apache/solr/schema/CompressableField.java
View File

@ -1,68 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.schema;
import org.apache.lucene.document.Field;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import java.util.Map;
/** <code>CompressableField</code> is an abstract field type which enables a
* field to be compressed (by specifying <code>compressed="true"</code> at the
* field definition level) and provides optional support for specifying a
* threshold at which compression is enabled.
*
* Optional settings:
* <ul>
* <li><code>compressThreshold</code>: length, in characters, at which point the
* field contents should be compressed [default: 0]</li>
* </ul></p>
*
* TODO: Enable compression level specification (not yet in lucene)
*
* @version $Id$
*/
public abstract class CompressableField extends FieldType {
/* if field size (in characters) is greater than this threshold, the field
will be stored compressed */
public static int DEFAULT_COMPRESS_THRESHOLD = 0;
int compressThreshold;
private static String CT = "compressThreshold";
protected void init(IndexSchema schema, Map<String,String> args) {
SolrParams p = new MapSolrParams(args);
compressThreshold = p.getInt(CT, DEFAULT_COMPRESS_THRESHOLD);
args.remove(CT);
super.init(schema, args);
}
/* Helpers for field construction */
protected Field.Store getFieldStore(SchemaField field,
String internalVal) {
/* compress field if length exceeds threshold */
if(field.isCompressed()) {
// nocommit: handle compression
//return internalVal.length() >= compressThreshold ?
// Field.Store.COMPRESS : Field.Store.YES;
return Field.Store.YES;
} else
return super.getFieldStore(field, internalVal);
}
}

21
src/java/org/apache/solr/schema/FieldProperties.java
View File

@ -33,23 +33,22 @@ abstract class FieldProperties {
final static int TOKENIZED = 0x00000002;
final static int STORED = 0x00000004;
final static int BINARY = 0x00000008;
final static int COMPRESSED = 0x00000010;
final static int OMIT_NORMS = 0x00000020;
final static int OMIT_TF_POSITIONS = 0x00000040;
final static int STORE_TERMVECTORS = 0x00000080;
final static int STORE_TERMPOSITIONS = 0x00000100;
final static int STORE_TERMOFFSETS = 0x00000200;
final static int OMIT_NORMS = 0x00000010;
final static int OMIT_TF_POSITIONS = 0x00000020;
final static int STORE_TERMVECTORS = 0x00000040;
final static int STORE_TERMPOSITIONS = 0x00000080;
final static int STORE_TERMOFFSETS = 0x00000100;
final static int MULTIVALUED = 0x00000400;
final static int SORT_MISSING_FIRST = 0x00000800;
final static int SORT_MISSING_LAST = 0x00001000;
final static int MULTIVALUED = 0x00000200;
final static int SORT_MISSING_FIRST = 0x00000400;
final static int SORT_MISSING_LAST = 0x00000800;
final static int REQUIRED = 0x00002000;
final static int REQUIRED = 0x00001000;
static final String[] propertyNames = {
"indexed", "tokenized", "stored",
"binary", "compressed", "omitNorms", "omitTermFreqAndPositions",
"binary", "omitNorms", "omitTermFreqAndPositions",
"termVectors", "termPositions", "termOffsets",
"multiValued",
"sortMissingFirst","sortMissingLast","required"

8
src/java/org/apache/solr/schema/FieldType.java
View File

@ -127,8 +127,12 @@ public abstract class FieldType extends FieldProperties {
void setArgs(IndexSchema schema, Map<String,String> args) {
// default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version
properties = (STORED | INDEXED);
if (schema.getVersion()< 1.1f) properties |= MULTIVALUED;
if (schema.getVersion()> 1.1f) properties |= OMIT_TF_POSITIONS;
float schemaVersion = schema.getVersion();
if (schemaVersion < 1.1f) properties |= MULTIVALUED;
if (schemaVersion > 1.1f) properties |= OMIT_TF_POSITIONS;
if (schemaVersion < 1.3) {
args.remove("compressThreshold");
}
this.args=args;
Map<String,String> initArgs = new HashMap<String,String>(args);

3
src/java/org/apache/solr/schema/SchemaField.java
View File

@ -83,7 +83,6 @@ public final class SchemaField extends FieldProperties {
public boolean multiValued() { return (properties & MULTIVALUED)!=0; }
public boolean sortMissingFirst() { return (properties & SORT_MISSING_FIRST)!=0; }
public boolean sortMissingLast() { return (properties & SORT_MISSING_LAST)!=0; }
public boolean isCompressed() { return (properties & COMPRESSED)!=0; }
public boolean isRequired() { return required; }
// things that should be determined by field type, not set as options
@ -166,7 +165,7 @@ public final class SchemaField extends FieldProperties {
// that depend on that.
//
if (on(falseProps,STORED)) {
int pp = STORED | BINARY | COMPRESSED;
int pp = STORED | BINARY;
if (on(pp,trueProps)) {
throw new RuntimeException("SchemaField: " + name + " conflicting stored field options:" + props);
}

2
src/java/org/apache/solr/schema/StrField.java
View File

@ -33,7 +33,7 @@ import java.io.IOException;
/**
* @version $Id$
*/
public class StrField extends CompressableField {
public class StrField extends FieldType {
protected void init(IndexSchema schema, Map<String,String> args) {
super.init(schema, args);
}

2
src/java/org/apache/solr/schema/TextField.java
View File

@ -45,7 +45,7 @@ import java.io.StringReader;
* Analyzers for field types using this implementation should be defined in the schema.
* @version $Id$
*/
public class TextField extends CompressableField {
public class TextField extends FieldType {
protected void init(IndexSchema schema, Map<String,String> args) {
properties |= TOKENIZED;
if (schema.getVersion()> 1.1f) properties &= ~OMIT_TF_POSITIONS;