move sorting to work with new field data

src/main/java/org/elasticsearch/common/lucene/Lucene.java

@@ -32,7 +32,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
-import org.elasticsearch.index.field.data.FieldDataType;
+import org.elasticsearch.index.fielddata.IndexFieldData;
 
 import java.io.IOException;
 import java.lang.reflect.Field;
@@ -215,7 +215,7 @@ public class Lucene {
                     out.writeString(sortField.getField());
                 }
                 if (sortField.getComparatorSource() != null) {
-                    writeSortType(out, ((FieldDataType.ExtendedFieldComparatorSource) sortField.getComparatorSource()).reducedType());
+                    writeSortType(out, ((IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource()).reducedType());
                 } else {
                     writeSortType(out, sortField.getType());
                 }

src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java

@@ -56,7 +56,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
 
         buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
                 .put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
-                .put(Tuple.tuple("string", "packed_bytes"), new PackedBytesIndexFieldData.Builder())
+                .put(Tuple.tuple("string", "paged_bytes"), new PagesBytesIndexFieldData.Builder())
                 .put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
                 .put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())

src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class ByteArrayIndexFieldData extends AbstractIndexFieldData<ByteArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ByteArrayAtomicFieldData.Single(new byte[0], 0);
+            return new ByteArrayAtomicFieldData.SingleFixedSet(new byte[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java

@@ -131,7 +131,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<S
             } else {
                 ret.bytes = value.bytes;
                 ret.offset = value.offset;
-                ret.length = value.offset;
+                ret.length = value.length;
             }
             return ret;
         }

src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
+            return new DoubleArrayAtomicFieldData.SingleFixedSet(new double[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new FloatArrayAtomicFieldData.Single(new float[0], 0);
+            return new FloatArrayAtomicFieldData.SingleFixedSet(new float[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayIndexFieldData.java

@@ -77,7 +77,7 @@ public class GeoPointDoubleArrayIndexFieldData extends AbstractIndexFieldData<Ge
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new GeoPointDoubleArrayAtomicFieldData.Single(new double[0], new double[0], 0);
+            return new GeoPointDoubleArrayAtomicFieldData.SingleFixedSet(new double[1], new double[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class IntArrayIndexFieldData extends AbstractIndexFieldData<IntArrayAtomi
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new IntArrayAtomicFieldData.Single(new int[0], 0);
+            return new IntArrayAtomicFieldData.SingleFixedSet(new int[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class LongArrayIndexFieldData extends AbstractIndexFieldData<LongArrayAto
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new LongArrayAtomicFieldData.Single(new long[0], 0);
+            return new LongArrayAtomicFieldData.SingleFixedSet(new long[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java

@@ -32,7 +32,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
 
 /**
  */
-public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
+public class PagedBytesAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
 
     // 0 ordinal in values means no value (its null)
     private final PagedBytes.Reader bytes;
@@ -42,7 +42,7 @@ public class PackedBytesAtomicFieldData implements AtomicOrdinalFieldData<Script
     private int[] hashes;
     private long size = -1;
 
-    public PackedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
+    public PagedBytesAtomicFieldData(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals ordinals) {
         this.bytes = bytes;
         this.termOrdToBytesOffset = termOrdToBytesOffset;
         this.ordinals = ordinals;

src/main/java/org/elasticsearch/index/fielddata/plain/PagesBytesIndexFieldData.java

@@ -41,17 +41,17 @@ import java.util.ArrayList;
 
 /**
  */
-public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedBytesAtomicFieldData> implements IndexOrdinalFieldData<PackedBytesAtomicFieldData> {
+public class PagesBytesIndexFieldData extends AbstractIndexFieldData<PagedBytesAtomicFieldData> implements IndexOrdinalFieldData<PagedBytesAtomicFieldData> {
 
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
         public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new PackedBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
+            return new PagesBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
         }
     }
 
-    public PackedBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+    public PagesBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
         super(index, indexSettings, fieldNames, fieldDataType, cache);
     }
 
@@ -61,7 +61,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
     }
 
     @Override
-    public PackedBytesAtomicFieldData load(AtomicReaderContext context) {
+    public PagedBytesAtomicFieldData load(AtomicReaderContext context) {
         try {
             return cache.load(context, this);
         } catch (Throwable e) {
@@ -74,7 +74,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
     }
 
     @Override
-    public PackedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
+    public PagedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();
 
         Terms terms = reader.terms(getFieldNames().indexName());
@@ -83,7 +83,7 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
             // 0 is reserved for "unset"
             bytes.copyUsingLengthPrefix(new BytesRef());
             GrowableWriter termOrdToBytesOffset = new GrowableWriter(1, 2, PackedInts.FASTEST);
-            return new PackedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
+            return new PagedBytesAtomicFieldData(bytes.freeze(true), termOrdToBytesOffset.getMutable(), new EmptyOrdinals(reader.maxDoc()));
         }
 
         final PagedBytes bytes = new PagedBytes(15);
@@ -174,13 +174,13 @@ public class PackedBytesIndexFieldData extends AbstractIndexFieldData<PackedByte
         PackedInts.Reader termOrdToBytesOffsetReader = termOrdToBytesOffset.getMutable();
 
         if (ordinals.size() == 1) {
-            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
+            return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new SingleArrayOrdinals(ordinals.get(0), termOrd));
         } else {
             int[][] nativeOrdinals = new int[ordinals.size()][];
             for (int i = 0; i < nativeOrdinals.length; i++) {
                 nativeOrdinals[i] = ordinals.get(i);
             }
-            return new PackedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
+            return new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffsetReader, new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
         }
 
     }

src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayIndexFieldData.java

@@ -83,7 +83,7 @@ public class ShortArrayIndexFieldData extends AbstractIndexFieldData<ShortArrayA
 
         Terms terms = reader.terms(getFieldNames().indexName());
         if (terms == null) {
-            return new ShortArrayAtomicFieldData.Single(new short[0], 0);
+            return new ShortArrayAtomicFieldData.SingleFixedSet(new short[1], 0, new FixedBitSet(1));
         }
 
         // TODO: how can we guess the number of terms? numerics end up creating more terms per value...

src/main/java/org/elasticsearch/search/sort/SortParseElement.java

@@ -163,7 +163,7 @@ public class SortParseElement implements SearchParseElement {
                 }
                 throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on");
             }
-            sortFields.add(new SortField(fieldMapper.names().indexName(), fieldMapper.fieldDataType().newFieldComparatorSource(context.fieldDataCache(), missing), reverse));
+            sortFields.add(new SortField(fieldMapper.names().indexName(), context.fieldData().getForField(fieldMapper).comparatorSource(missing), reverse));
         }
     }
 }

src/test/java/org/elasticsearch/test/integration/search/sort/SimpleSortTests.java

@@ -531,22 +531,24 @@ public class SimpleSortTests extends AbstractNodesTests {
 
         client.admin().indices().prepareFlush().setRefresh(true).execute().actionGet();
 
-        logger.info("--> sort with no missing");
+        logger.info("--> sort with no missing (same as missing _last)");
         SearchResponse searchResponse = client.prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
-        assertThat(searchResponse.hits().getAt(1).id(), equalTo("2"));
+        assertThat(searchResponse.hits().getAt(1).id(), equalTo("3"));
-        assertThat(searchResponse.hits().getAt(2).id(), equalTo("3"));
+        assertThat(searchResponse.hits().getAt(2).id(), equalTo("2"));
 
         logger.info("--> sort with missing _last");
         searchResponse = client.prepareSearch()
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_last"))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("1"));
@@ -558,6 +560,7 @@ public class SimpleSortTests extends AbstractNodesTests {
                 .setQuery(matchAllQuery())
                 .addSort(SortBuilders.fieldSort("i_value").order(SortOrder.ASC).missing("_first"))
                 .execute().actionGet();
+        assertThat(Arrays.toString(searchResponse.shardFailures()), searchResponse.failedShards(), equalTo(0));
 
         assertThat(searchResponse.hits().getTotalHits(), equalTo(3l));
         assertThat(searchResponse.hits().getAt(0).id(), equalTo("2"));

src/test/java/org/elasticsearch/test/unit/index/fielddata/PagedBytesStringFieldDataTests.java

@@ -26,10 +26,10 @@ import org.testng.annotations.Test;
 /**
  */
 @Test
-public class PackedBytesStringFieldDataTests extends StringFieldDataTests {
+public class PagedBytesStringFieldDataTests extends StringFieldDataTests {
 
     @Override
     protected FieldDataType getFieldDataType() {
-        return new FieldDataType("string", "packed_bytes", ImmutableMap.<String, String>of());
+        return new FieldDataType("string", "paged_bytes", ImmutableMap.<String, String>of());
     }
 }

src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java

@@ -56,6 +56,13 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
         return "4";
     }
 
+    private String toString(Object value) {
+        if (value instanceof BytesRef) {
+            return ((BytesRef) value).utf8ToString();
+        }
+        return value.toString();
+    }
+
     protected void fillSingleValueAllSet() throws Exception {
         Document d = new Document();
         d.add(new StringField("_id", "1", Field.Store.NO));
@@ -218,8 +225,11 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
                 new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
         assertThat(topDocs.totalHits, equalTo(3));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[0]).fields[0]), equalTo(one()));
         assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[1]).fields[0]), equalTo(two()));
         assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
+        assertThat(toString(((FieldDoc) topDocs.scoreDocs[2]).fields[0]), equalTo(three()));
 
         topDocs = searcher.search(new MatchAllDocsQuery(), 10,
                 new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));