Make array value parsing flag more robust. (#63371)

When constructing a value fetcher, the 'parsesArrayValue' flag must match
`FieldMapper#parsesArrayValue`. However there is nothing in code or tests to
help enforce this.

This PR reworks the value fetcher constructors so that `parsesArrayValue` is
'false' by default. Just as for `FieldMapper#parsesArrayValue`, field types must
explicitly set it to true and ensure the behavior is covered by tests.

Follow-up to #62974.

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

@@ -118,7 +118,7 @@ public class RankFeatureFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Float parseSourceValue(Object value) {
                     return objectToFloat(value);

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

@@ -93,7 +93,7 @@ public class RankFeaturesFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -225,7 +225,7 @@ public class ScaledFloatFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Double parseSourceValue(Object value) {
                     double doubleValue;

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -227,7 +227,7 @@ public final class ParentJoinFieldMapper extends FieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -229,7 +229,7 @@ public class PercolatorFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -104,7 +104,7 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     String keywordValue = value.toString();

plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java

@@ -111,7 +111,7 @@ public class Murmur3FieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     return value.toString();

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -259,13 +259,21 @@ public abstract class AbstractGeometryFieldMapper<Parsed, Processed> extends Fie
             String geoFormat = format != null ? format : GeoJsonGeometryFormat.NAME;
 
             Function<Object, Object> valueParser = value -> geometryParser.parseAndFormatObject(value, geoFormat);
-
+            if (parsesArrayValue) {
-            return new SourceValueFetcher(name(), mapperService, parsesArrayValue) {
+                return new ArraySourceValueFetcher(name(), mapperService) {
-                @Override
+                    @Override
-                protected Object parseSourceValue(Object value) {
+                    protected Object parseSourceValue(Object value) {
-                    return valueParser.apply(value);
+                        return valueParser.apply(value);
-                }
+                    }
-            };
+                };
+            } else {
+                return new SourceValueFetcher(name(), mapperService) {
+                    @Override
+                    protected Object parseSourceValue(Object value) {
+                        return valueParser.apply(value);
+                    }
+                };
+            }
         }
     }
 

server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * An implementation of {@link ValueFetcher} that knows how to extract values
+ * from the document source.
+ *
+ * This class differs from {@link SourceValueFetcher} in that it directly handles
+ * array values in parsing. Field types should use this class if their corresponding
+ * mapper returns true for {@link FieldMapper#parsesArrayValue()}.
+ */
+public abstract class ArraySourceValueFetcher implements ValueFetcher {
+    private final Set<String> sourcePaths;
+    private final @Nullable Object nullValue;
+
+    public ArraySourceValueFetcher(String fieldName, MapperService mapperService) {
+        this(fieldName, mapperService, null);
+    }
+
+    /**
+     * @param fieldName The name of the field.
+     * @param mapperService A mapper service.
+     * @param nullValue A optional substitute value if the _source value is 'null'.
+     */
+    public ArraySourceValueFetcher(String fieldName, MapperService mapperService, Object nullValue) {
+        this.sourcePaths = mapperService.sourcePath(fieldName);
+        this.nullValue = nullValue;
+    }
+
+    @Override
+    public List<Object> fetchValues(SourceLookup lookup) {
+        List<Object> values = new ArrayList<>();
+        for (String path : sourcePaths) {
+            Object sourceValue = lookup.extractValue(path, nullValue);
+            if (sourceValue == null) {
+                return org.elasticsearch.common.collect.List.of();
+            }
+            values.addAll((List<?>) parseSourceValue(sourceValue));
+        }
+        return values;
+    }
+
+    /**
+     * Given a value that has been extracted from a document's source, parse it into a standard
+     * format. This parsing logic should closely mirror the value parsing in
+     * {@link FieldMapper#parseCreateField} or {@link FieldMapper#parse}.
+     */
+    protected abstract Object parseSourceValue(Object value);
+}

server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java

@@ -103,7 +103,7 @@ public class BinaryFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -135,7 +135,7 @@ public class BooleanFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected Boolean parseSourceValue(Object value) {
                     if (value instanceof Boolean) {

server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java

@@ -307,7 +307,7 @@ public class CompletionFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, true) {
+            return new ArraySourceValueFetcher(name(), mapperService) {
                 @Override
                 protected List<?> parseSourceValue(Object value) {
                     if (value instanceof List) {

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -330,7 +330,7 @@ public final class DateFieldMapper extends ParametrizedFieldMapper {
                 ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
                 : defaultFormatter;
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 public String parseSourceValue(Object value) {
                     String date = value.toString();

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -158,7 +158,7 @@ public class IpFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     InetAddress address;

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -243,7 +243,7 @@ public final class KeywordFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     String keywordValue = value.toString();

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -980,7 +980,7 @@ public class NumberFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     if (value.equals("")) {

server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java

@@ -205,7 +205,7 @@ public class RangeFieldMapper extends ParametrizedFieldMapper {
                 ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
                 : defaultFormatter;
 
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
 
                 @Override
                 @SuppressWarnings("unchecked")

server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java

@@ -32,25 +32,25 @@ import java.util.Set;
  * An implementation of {@link ValueFetcher} that knows how to extract values
  * from the document source. Most standard field mappers will use this class
  * to implement value fetching.
+ *
+ * Field types that handle arrays directly should instead use {@link ArraySourceValueFetcher}.
  */
 public abstract class SourceValueFetcher implements ValueFetcher {
     private final Set<String> sourcePaths;
     private final @Nullable Object nullValue;
-    private final boolean parsesArrayValue;
 
-    public SourceValueFetcher(String fieldName, MapperService mapperService, boolean parsesArrayValue) {
+    public SourceValueFetcher(String fieldName, MapperService mapperService) {
-        this(fieldName, mapperService, parsesArrayValue, null);
+        this(fieldName, mapperService, null);
     }
 
     /**
      * @param fieldName The name of the field.
-     * @param parsesArrayValue Whether the fetcher handles array values during document parsing.
+     * @param mapperService A mapper service.
      * @param nullValue A optional substitute value if the _source value is 'null'.
      */
-    public SourceValueFetcher(String fieldName, MapperService mapperService, boolean parsesArrayValue, Object nullValue) {
+    public SourceValueFetcher(String fieldName, MapperService mapperService, Object nullValue) {
         this.sourcePaths = mapperService.sourcePath(fieldName);
         this.nullValue = nullValue;
-        this.parsesArrayValue = parsesArrayValue;
     }
 
     @Override
@@ -62,22 +62,18 @@ public abstract class SourceValueFetcher implements ValueFetcher {
                 return org.elasticsearch.common.collect.List.of();
             }
 
-            if (parsesArrayValue) {
+            // We allow source values to contain multiple levels of arrays, such as `"field": [[1, 2]]`.
-                values.addAll((List<?>) parseSourceValue(sourceValue));
+            // So we need to unwrap these arrays before passing them on to be parsed.
-            } else {
+            Queue<Object> queue = new ArrayDeque<>();
-                // We allow source values to contain multiple levels of arrays, such as `"field": [[1, 2]]`.
+            queue.add(sourceValue);
-                // So we need to unwrap these arrays before passing them on to be parsed.
+            while (queue.isEmpty() == false) {
-                Queue<Object> queue = new ArrayDeque<>();
+                Object value = queue.poll();
-                queue.add(sourceValue);
+                if (value instanceof List) {
-                while (queue.isEmpty() == false) {
+                    queue.addAll((List<?>) value);
-                    Object value = queue.poll();
+                } else {
-                    if (value instanceof List) {
+                    Object parsedValue = parseSourceValue(value);
-                        queue.addAll((List<?>) value);
+                    if (parsedValue != null) {
-                    } else {
+                        values.add(parsedValue);
-                        Object parsedValue = parseSourceValue(value);
-                        if (parsedValue != null) {
-                            values.add(parsedValue);
-                        }
                     }
                 }
             }

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -645,7 +645,7 @@ public class TextFieldMapper extends FieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value.toString();

server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java

@@ -107,7 +107,7 @@ public class ExternalMapper extends ParametrizedFieldMapper {
 
         @Override
         public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java

@@ -95,7 +95,7 @@ public class FakeStringFieldMapper extends FieldMapper {
 
         @Override
         public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     return value.toString();

x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java

@@ -141,7 +141,7 @@ public class HistogramFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false) {
+            return new SourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java

@@ -479,7 +479,7 @@ public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+            return new SourceValueFetcher(name(), mapperService, nullValue) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java

@@ -235,7 +235,7 @@ public class UnsignedLongFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, nullValueFormatted) {
+            return new SourceValueFetcher(name(), mapperService, nullValueFormatted) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     if (value.equals("")) {

x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java

@@ -138,7 +138,7 @@ public class VersionStringFieldMapper extends ParametrizedFieldMapper {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
 
-            return new SourceValueFetcher(name(), mapperService, false, null) {
+            return new SourceValueFetcher(name(), mapperService, null) {
                 @Override
                 protected String parseSourceValue(Object value) {
                     return value.toString();

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java

@@ -15,13 +15,13 @@ import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.ArraySourceValueFetcher;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParametrizedFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -116,7 +116,7 @@ public class DenseVectorFieldMapper extends ParametrizedFieldMapper {
             if (format != null) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
-            return new SourceValueFetcher(name(), mapperService, true) {
+            return new ArraySourceValueFetcher(name(), mapperService) {
                 @Override
                 protected Object parseSourceValue(Object value) {
                     return value;

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java

@@ -8,7 +8,9 @@ package org.elasticsearch.xpack.vectors.mapper;
 
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 
+import java.io.IOException;
 import java.util.Collections;
+import java.util.List;
 
 public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
 
@@ -33,4 +35,10 @@ public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
         DenseVectorFieldMapper.DenseVectorFieldType ft = new DenseVectorFieldMapper.DenseVectorFieldType("f", 1, Collections.emptyMap());
         expectThrows(UnsupportedOperationException.class, () -> ft.docValueFormat(null, null));
     }
+
+    public void testFetchSourceValue() throws IOException {
+        DenseVectorFieldMapper.DenseVectorFieldType ft = new DenseVectorFieldMapper.DenseVectorFieldType("f", 5, Collections.emptyMap());
+        List<Double> vector = org.elasticsearch.common.collect.List.of(0.0, 1.0, 2.0, 3.0, 4.0);
+        assertEquals(vector, fetchSourceValue(ft, vector));
+    }
 }

x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java

@@ -982,7 +982,7 @@ public class WildcardFieldMapper extends FieldMapper {
                  throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
              }
 
-             return new SourceValueFetcher(name(), mapperService, false, nullValue) {
+             return new SourceValueFetcher(name(), mapperService, nullValue) {
                  @Override
                  protected String parseSourceValue(Object value) {
                      String keywordValue = value.toString();