Move dense_vector and sparse_vector to module (#43280) (#43333)

docs/reference/mapping/types/dense-vector.asciidoc

@@ -1,3 +1,5 @@
+[role="xpack"]
+[testenv="basic"]
 [[dense-vector]]
 === Dense vector datatype
 

docs/reference/mapping/types/sparse-vector.asciidoc

@@ -1,3 +1,5 @@
+[role="xpack"]
+[testenv="basic"]
 [[sparse-vector]]
 === Sparse vector datatype
 

docs/reference/query-dsl/script-score-query.asciidoc

@@ -72,6 +72,8 @@ to be the most efficient by using the internal mechanisms.
 --------------------------------------------------
 // NOTCONSOLE
 
+[role="xpack"]
+[testenv="basic"]
 [[vector-functions]]
 ===== Functions for vector fields
 

docs/reference/rest-api/info.asciidoc

@@ -107,6 +107,10 @@ Example response:
          "available" : true,
          "enabled" : true
       },
+      "vectors" : {
+         "available" : true,
+         "enabled" : true
+      },
       "watcher" : {
          "available" : true,
          "enabled" : true

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/71_context_api.yml

@@ -17,6 +17,6 @@
     - match: { classes.6.methods.0.parameters.0 : java.lang.CharSequence }
     - match: { classes.6.methods.0.parameters.1 : int }
     - match: { classes.6.methods.0.parameters.2 : int }
-    - match: { imported_methods.0.name: dotProduct }
+    - match: { imported_methods.0.name: saturation }
-    - match: { class_bindings.0.name: cosineSimilarity }
+    - match: { class_bindings.0.name: decayDateExp }
     - match: { instance_bindings: [] }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java

@@ -39,8 +39,6 @@ public class MapperExtrasPlugin extends Plugin implements MapperPlugin, SearchPl
         mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser());
         mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, new RankFeatureFieldMapper.TypeParser());
         mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, new RankFeaturesFieldMapper.TypeParser());
-        mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser());
-        mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser());
         mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, new SearchAsYouTypeFieldMapper.TypeParser());
         return Collections.unmodifiableMap(mappers);
     }

modules/mapper-extras/src/main/java/org/elasticsearch/index/query/DocValuesWhitelistExtension.java

@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.query;
-
-
-import org.elasticsearch.painless.spi.PainlessExtension;
-import org.elasticsearch.painless.spi.Whitelist;
-import org.elasticsearch.painless.spi.WhitelistLoader;
-import org.elasticsearch.script.ScoreScript;
-import org.elasticsearch.script.ScriptContext;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-public class DocValuesWhitelistExtension implements PainlessExtension {
-
-    private static final Whitelist WHITELIST =
-        WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "docvalues_whitelist.txt");
-
-    @Override
-    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
-        return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST));
-    }
-}

modules/mapper-extras/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension

@@ -1 +0,0 @@
-org.elasticsearch.index.query.DocValuesWhitelistExtension

modules/mapper-extras/src/main/resources/org/elasticsearch/index/query/docvalues_whitelist.txt

@@ -1,32 +0,0 @@
-#
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-class org.elasticsearch.index.query.VectorScriptDocValues {
-}
-class org.elasticsearch.index.query.VectorScriptDocValues$DenseVectorScriptDocValues {
-}
-class org.elasticsearch.index.query.VectorScriptDocValues$SparseVectorScriptDocValues {
-}
-
-static_import {
-    double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilarity
-    double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.index.query.ScoreScriptUtils
-    double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$DotProductSparse
-    double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.index.query.ScoreScriptUtils$CosineSimilaritySparse
-}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/DenseVectorFieldTypeTests.java

@@ -1,28 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
-
-    @Override
-    protected MappedFieldType createDefaultFieldType() {
-        return new DenseVectorFieldMapper.DenseVectorFieldType();
-    }
-}

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SparseVectorFieldTypeTests.java

@@ -1,28 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.mapper;
-
-public class SparseVectorFieldTypeTests extends FieldTypeTestCase {
-
-    @Override
-    protected MappedFieldType createDefaultFieldType() {
-        return new SparseVectorFieldMapper.SparseVectorFieldType();
-    }
-}

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -687,6 +687,15 @@ public class XPackLicenseState {
         return licensed && localStatus.active;
     }
 
+    /**
+     * Determine if Vectors support should be enabled.
+     * <p>
+     *  Vectors is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isVectorsAllowed() {
+        return status.active;
+    }
+
     /**
      * Determine if ODBC support should be enabled.
      * <p>

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -184,6 +184,7 @@ import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeAction;
 import org.elasticsearch.xpack.core.upgrade.actions.IndexUpgradeInfoAction;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherFeatureSetUsage;
 import org.elasticsearch.xpack.core.watcher.WatcherMetaData;
 import org.elasticsearch.xpack.core.watcher.transport.actions.ack.AckWatchAction;
@@ -465,7 +466,9 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.DATA_FRAME, DataFrameFeatureSetUsage::new),
                 new NamedWriteableRegistry.Entry(PersistentTaskParams.class, DataFrameField.TASK_NAME, DataFrameTransform::new),
                 new NamedWriteableRegistry.Entry(Task.Status.class, DataFrameField.TASK_NAME, DataFrameTransformState::new),
-                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new)
+                new NamedWriteableRegistry.Entry(PersistentTaskState.class, DataFrameField.TASK_NAME, DataFrameTransformState::new),
+                // Vectors
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.VECTORS, VectorsFeatureSetUsage::new)
         );
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -37,6 +37,8 @@ public final class XPackField {
     public static final String CCR = "ccr";
     /** Name constant for the data frame feature. */
     public static final String DATA_FRAME = "data_frame";
+    /** Name constant for the vectors feature. */
+    public static final String VECTORS = "vectors";
 
     private XPackField() {}
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -119,6 +119,9 @@ public class XPackSettings {
     /** Setting for enabling or disabling sql. Defaults to true. */
     public static final Setting<Boolean> SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope);
 
+    /** Setting for enabling or disabling vectors. Defaults to true. */
+    public static final Setting<Boolean> VECTORS_ENABLED = Setting.boolSetting("xpack.vectors.enabled", true, Setting.Property.NodeScope);
+
     public static final List<String> DEFAULT_SUPPORTED_PROTOCOLS;
 
     static {
@@ -249,6 +252,7 @@ public class XPackSettings {
         settings.add(PASSWORD_HASHING_ALGORITHM);
         settings.add(INDEX_LIFECYCLE_ENABLED);
         settings.add(DATA_FRAME_ENABLED);
+        settings.add(VECTORS_ENABLED);
         return Collections.unmodifiableList(settings);
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/vectors/VectorsFeatureSetUsage.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.vectors;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+
+public class VectorsFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    public VectorsFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+    }
+
+    public VectorsFeatureSetUsage(boolean available, boolean enabled) {
+        super(XPackField.VECTORS, available, enabled);
+    }
+}

x-pack/plugin/vectors/build.gradle

@@ -0,0 +1,22 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'vectors'
+    description 'A plugin for working with vectors'
+    classname 'org.elasticsearch.xpack.vectors.Vectors'
+    extendedPlugins = ['x-pack-core', 'lang-painless']
+}
+archivesBaseName = 'x-pack-vectors'
+
+dependencies {
+    compileOnly project(':modules:lang-painless:spi')
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    if (isEclipse) {
+        testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
+    }
+}
+
+integTest.enabled = false

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/Vectors.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.vectors.mapper.DenseVectorFieldMapper;
+import org.elasticsearch.xpack.vectors.mapper.SparseVectorFieldMapper;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+
+public class Vectors extends Plugin implements MapperPlugin {
+
+    public static final String NAME = "vectors";
+    protected final boolean enabled;
+
+    public Vectors(Settings settings) {
+        this.enabled = XPackSettings.VECTORS_ENABLED.get(settings);
+    }
+
+    public Collection<Module> createGuiceModules() {
+        return Collections.singletonList(b -> {
+            XPackPlugin.bindFeatureSet(b, VectorsFeatureSet.class);
+        });
+    }
+
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        if (enabled == false) {
+            return emptyMap();
+        }
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+        mappers.put(DenseVectorFieldMapper.CONTENT_TYPE, new DenseVectorFieldMapper.TypeParser());
+        mappers.put(SparseVectorFieldMapper.CONTENT_TYPE, new SparseVectorFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+}

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/VectorsFeatureSet.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
+
+import java.util.Map;
+
+public class VectorsFeatureSet implements XPackFeatureSet {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public VectorsFeatureSet(Settings settings, XPackLicenseState licenseState) {
+        this.enabled = XPackSettings.VECTORS_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.VECTORS;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState != null && licenseState.isVectorsAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+    @Override
+    public Map<String, Object> nativeCodeInfo() {
+        return null;
+    }
+
+    @Override
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
+        listener.onResponse(new VectorsFeatureSetUsage(available(), enabled()));
+    }
+}

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexOptions;
@@ -29,8 +17,14 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.ArrayValueMapperParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.VectorDVIndexFieldData;
+import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData;
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexOptions;
@@ -29,8 +17,13 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.VectorDVIndexFieldData;
+import org.elasticsearch.xpack.vectors.query.VectorDVIndexFieldData;
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoder.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InPlaceMergeSorter;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/DocValuesWhitelistExtension.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.query;
+
+
+import org.elasticsearch.painless.spi.PainlessExtension;
+import org.elasticsearch.painless.spi.Whitelist;
+import org.elasticsearch.painless.spi.WhitelistLoader;
+import org.elasticsearch.script.ScoreScript;
+import org.elasticsearch.script.ScriptContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class DocValuesWhitelistExtension implements PainlessExtension {
+
+    private static final Whitelist WHITELIST =
+        WhitelistLoader.loadFromResourceFiles(DocValuesWhitelistExtension.class, "whitelist.txt");
+
+    @Override
+    public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
+        return Collections.singletonMap(ScoreScript.CONTEXT, Collections.singletonList(WHITELIST));
+    }
+}

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtils.java

@@ -1,32 +1,20 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.mapper.VectorEncoderDecoder;
+import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder;
 
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues;
+import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder.sortSparseDimsDoubleValues;
 
 public class ScoreScriptUtils {
 

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVAtomicFieldData.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DocValues;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorDVIndexFieldData.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.SortField;

x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/query/VectorScriptDocValues.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.query;
+
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.util.BytesRef;

x-pack/plugin/vectors/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension

@@ -0,0 +1 @@
+org.elasticsearch.xpack.vectors.query.DocValuesWhitelistExtension

x-pack/plugin/vectors/src/main/resources/org/elasticsearch/xpack/vectors/query/whitelist.txt

@@ -0,0 +1,18 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues {
+}
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues {
+}
+class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues {
+}
+
+static_import {
+    double cosineSimilarity(List, VectorScriptDocValues.DenseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilarity
+    double dotProduct(List, VectorScriptDocValues.DenseVectorScriptDocValues) from_class org.elasticsearch.xpack.vectors.query.ScoreScriptUtils
+    double dotProductSparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$DotProductSparse
+    double cosineSimilaritySparse(Map, VectorScriptDocValues.SparseVectorScriptDocValues) bound_to org.elasticsearch.xpack.vectors.query.ScoreScriptUtils$CosineSimilaritySparse
+}

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/VectorsFeatureSetTests.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.vectors;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.vectors.VectorsFeatureSetUsage;
+import org.junit.Before;
+
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class VectorsFeatureSetTests extends ESTestCase {
+
+    private XPackLicenseState licenseState;
+
+    @Before
+    public void init() {
+        licenseState = mock(XPackLicenseState.class);
+    }
+
+    public void testAvailable() throws Exception {
+        VectorsFeatureSet featureSet = new VectorsFeatureSet(Settings.EMPTY, licenseState);
+        boolean available = randomBoolean();
+        when(licenseState.isVectorsAllowed()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.available(), is(available));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.available(), is(available));
+    }
+
+    public void testEnabled() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings.Builder settings = Settings.builder();
+        if (enabled) {
+            if (randomBoolean()) {
+                settings.put("xpack.vectors.enabled", enabled);
+            }
+        } else {
+            settings.put("xpack.vectors.enabled", enabled);
+        }
+        VectorsFeatureSet featureSet = new VectorsFeatureSet(settings.build(), licenseState);
+        assertThat(featureSet.enabled(), is(enabled));
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.enabled(), is(enabled));
+
+        BytesStreamOutput out = new BytesStreamOutput();
+        usage.writeTo(out);
+        XPackFeatureSet.Usage serializedUsage = new VectorsFeatureSetUsage(out.bytes().streamInput());
+        assertThat(serializedUsage.enabled(), is(enabled));
+    }
+
+}

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexableField;
@@ -28,8 +16,16 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.vectors.Vectors;
+
 import org.junit.Before;
 
 import java.io.IOException;
@@ -59,7 +55,7 @@ public class DenseVectorFieldMapperTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(MapperExtrasPlugin.class);
+        return pluginList(Vectors.class, XPackPlugin.class);
     }
 
     public void testDefaults() throws Exception {

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldTypeTests.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class DenseVectorFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new DenseVectorFieldMapper.DenseVectorFieldType();
+    }
+}

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java

@@ -1,23 +1,11 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.index.IndexableField;
@@ -28,8 +16,15 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.vectors.Vectors;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
@@ -64,7 +59,7 @@ public class SparseVectorFieldMapperTests extends ESSingleNodeTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(MapperExtrasPlugin.class);
+        return pluginList(Vectors.class, XPackPlugin.class);
     }
 
     public void testDefaults() throws Exception {

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldTypeTests.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+package org.elasticsearch.xpack.vectors.mapper;
+
+import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.MappedFieldType;
+
+public class SparseVectorFieldTypeTests extends FieldTypeTestCase {
+
+    @Override
+    protected MappedFieldType createDefaultFieldType() {
+        return new SparseVectorFieldMapper.SparseVectorFieldType();
+    }
+}

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/VectorEncoderDecoderTests.java

@@ -1,23 +1,10 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.mapper;
+package org.elasticsearch.xpack.vectors.mapper;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.test.ESTestCase;

x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/query/ScoreScriptUtilsTests.java

@@ -1,38 +1,25 @@
 /*
- * Licensed to Elasticsearch under one or more contributor
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * license agreements. See the NOTICE file distributed with
+ * or more contributor license agreements. Licensed under the Elastic License;
- * this work for additional information regarding copyright
+ * you may not use this file except in compliance with the Elastic License.
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.xpack.vectors.query;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.mapper.VectorEncoderDecoder;
+import org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoder;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilarity;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilarity;
-import org.elasticsearch.index.query.ScoreScriptUtils.DotProductSparse;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.DotProductSparse;
-import org.elasticsearch.index.query.ScoreScriptUtils.CosineSimilaritySparse;
+import org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.CosineSimilaritySparse;
 
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector;
+import static org.elasticsearch.xpack.vectors.mapper.VectorEncoderDecoderTests.mockEncodeDenseVector;
-import static org.elasticsearch.index.query.ScoreScriptUtils.dotProduct;
+import static org.elasticsearch.xpack.vectors.query.ScoreScriptUtils.dotProduct;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;