Scripting: Convert domainSplit function for ML to whitelist (#34426)
This commit moves the definition of domainSplit into java and exposes it as a painless whitelist extension. The method also no longer needs params, and version which ignores params is added and deprecated.
This commit is contained in:
Ryan Ernst
GitHub
parent
8734540345
commit
d445785f1a
|
|
@ -490,10 +490,10 @@ PUT _xpack/ml/datafeeds/datafeed-test3
|
|||
},
|
||||
"script_fields":{
|
||||
"sub":{
|
||||
"script":"return domainSplit(doc['query'].value, params).get(0);"
|
||||
"script":"return domainSplit(doc['query'].value).get(0);"
|
||||
},
|
||||
"hrd":{
|
||||
"script":"return domainSplit(doc['query'].value, params).get(1);"
|
||||
"script":"return domainSplit(doc['query'].value).get(1);"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -511,10 +511,6 @@ registered domain. For example, the highest registered domain of
|
|||
`domainSplit()` function returns an array of two values: the first value is the
|
||||
subdomain; the second value is the highest registered domain.
|
||||
|
||||
NOTE: The `domainSplit()` function takes two arguments. The first argument is
|
||||
the string you want to split. The second argument is always `params`. This is a
|
||||
technical implementation detail related to how Painless operates internally.
|
||||
|
||||
The preview {dfeed} API returns the following results, which show that
|
||||
"www.ml.elastic.co" has been split into "elastic.co" and "www.ml":
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,8 @@ esplugin {
|
|||
description 'Elasticsearch Expanded Pack Plugin - Machine Learning'
|
||||
classname 'org.elasticsearch.xpack.ml.MachineLearning'
|
||||
hasNativeController true
|
||||
extendedPlugins = ['x-pack-core']
|
||||
extendedPlugins = ['x-pack-core', 'lang-painless']
|
||||
}
|
||||
archivesBaseName = 'x-pack-ml'
|
||||
|
||||
configurations {
|
||||
nativeBundle {
|
||||
|
|
@ -42,6 +41,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
|
|||
dependencies {
|
||||
// "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
|
||||
compileOnly project(path: xpackModule('core'), configuration: 'default')
|
||||
compileOnly "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${versions.elasticsearch}"
|
||||
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
|
||||
// This should not be here
|
||||
testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import org.elasticsearch.common.Strings;
|
|||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.test.rest.ESRestTestCase;
|
||||
import org.elasticsearch.xpack.ml.MachineLearning;
|
||||
import org.elasticsearch.xpack.ml.utils.DomainSplitFunction;
|
||||
import org.joda.time.DateTime;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -190,8 +189,7 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
|
|||
|
||||
Pattern pattern = Pattern.compile("domain_split\":\\[(.*?),(.*?)\\]");
|
||||
|
||||
Map<String, Object> params = new HashMap<>(DomainSplitFunction.params.size() + 1);
|
||||
params.putAll(DomainSplitFunction.params);
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
for (TestConfiguration testConfig : tests) {
|
||||
params.put("host", testConfig.hostName);
|
||||
String mapAsJson = Strings.toString(jsonBuilder().map(params));
|
||||
|
|
@ -207,8 +205,8 @@ public class PainlessDomainSplitIT extends ESRestTestCase {
|
|||
" \"domain_split\" : {\n" +
|
||||
" \"script\" : {\n" +
|
||||
" \"lang\": \"painless\",\n" +
|
||||
" \"inline\": \"" + DomainSplitFunction.function +
|
||||
" return domainSplit(params['host'], params); \",\n" +
|
||||
" \"inline\": \"" +
|
||||
" return domainSplit(params['host']); \",\n" +
|
||||
" \"params\": " + mapAsJson + "\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
package org.elasticsearch.xpack.ml;
|
||||
|
||||
import org.elasticsearch.painless.spi.PainlessExtension;
|
||||
import org.elasticsearch.painless.spi.Whitelist;
|
||||
import org.elasticsearch.painless.spi.WhitelistLoader;
|
||||
import org.elasticsearch.script.ScriptContext;
|
||||
import org.elasticsearch.script.SearchScript;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class MachineLearningPainlessExtension implements PainlessExtension {
|
||||
private static final Whitelist WHITELIST =
|
||||
WhitelistLoader.loadFromResourceFiles(MachineLearningPainlessExtension.class, "whitelist.txt");
|
||||
|
||||
@Override
|
||||
public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
|
||||
return Collections.singletonMap(SearchScript.CONTEXT, Collections.singletonList(WHITELIST));
|
||||
}
|
||||
}
|
||||
|
|
@ -17,7 +17,6 @@ import org.elasticsearch.action.search.SearchScrollRequestBuilder;
|
|||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.common.logging.Loggers;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.script.Script;
|
||||
import org.elasticsearch.search.SearchHit;
|
||||
import org.elasticsearch.search.fetch.StoredFieldsContext;
|
||||
import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
|
||||
|
|
@ -25,14 +24,11 @@ import org.elasticsearch.search.sort.SortOrder;
|
|||
import org.elasticsearch.xpack.core.ClientHelper;
|
||||
import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor;
|
||||
import org.elasticsearch.xpack.core.ml.datafeed.extractor.ExtractorUtils;
|
||||
import org.elasticsearch.xpack.ml.utils.DomainSplitFunction;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
|
|
@ -130,26 +126,10 @@ class ScrollDataExtractor implements DataExtractor {
|
|||
} else {
|
||||
searchRequestBuilder.setFetchSource(sourceFields, null);
|
||||
}
|
||||
context.scriptFields.forEach(f -> searchRequestBuilder.addScriptField(
|
||||
f.fieldName(), injectDomainSplit(f.script())));
|
||||
context.scriptFields.forEach(f -> searchRequestBuilder.addScriptField(f.fieldName(), f.script()));
|
||||
return searchRequestBuilder;
|
||||
}
|
||||
|
||||
private Script injectDomainSplit(Script script) {
|
||||
String code = script.getIdOrCode();
|
||||
if (code.contains("domainSplit(") && script.getLang().equals("painless")) {
|
||||
String modifiedCode = DomainSplitFunction.function + code;
|
||||
Map<String, Object> modifiedParams = new HashMap<>(script.getParams().size()
|
||||
+ DomainSplitFunction.params.size());
|
||||
|
||||
modifiedParams.putAll(script.getParams());
|
||||
modifiedParams.putAll(DomainSplitFunction.params);
|
||||
|
||||
return new Script(script.getType(), script.getLang(), modifiedCode, modifiedParams);
|
||||
}
|
||||
return script;
|
||||
}
|
||||
|
||||
private InputStream processSearchResponse(SearchResponse searchResponse) throws IOException {
|
||||
|
||||
if (searchResponse.getFailedShards() > 0 && searchHasShardFailure == false) {
|
||||
|
|
|
|||
|
|
@ -5,25 +5,33 @@
|
|||
*/
|
||||
package org.elasticsearch.xpack.ml.utils;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.elasticsearch.common.io.Streams;
|
||||
import org.elasticsearch.common.logging.DeprecationLogger;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
public final class DomainSplitFunction {
|
||||
|
||||
public static final String function;
|
||||
public static final Map<String, Object> params;
|
||||
private static final DeprecationLogger DEPRECATION_LOGGER =
|
||||
new DeprecationLogger(LogManager.getLogger(DomainSplitFunction.class));
|
||||
|
||||
DomainSplitFunction() {}
|
||||
private static final int MAX_DOMAIN_PART_LENGTH = 63;
|
||||
|
||||
private static final Map<String, String> exact;
|
||||
private static final Map<String, String> under;
|
||||
private static final Map<String, String> excluded;
|
||||
static {
|
||||
Map<String, Object> paramsMap = new HashMap<>();
|
||||
|
||||
Map<String, String> exact = new HashMap<>(2048);
|
||||
Map<String, String> exactMap = new HashMap<>(2048);
|
||||
|
||||
String exactResourceName = "org/elasticsearch/xpack/ml/transforms/exact.properties";
|
||||
|
||||
|
|
@ -31,253 +39,205 @@ public final class DomainSplitFunction {
|
|||
List<String> lines = Streams.readAllLines(resource);
|
||||
for (String line : lines) {
|
||||
String[] split = line.split("=");
|
||||
exact.put(split[0].trim(), split[1].trim());
|
||||
exactMap.put(split[0].trim(), split[1].trim());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException("Could not load DomainSplit resource", e);
|
||||
}
|
||||
exact = Collections.unmodifiableMap(exact);
|
||||
exact = Collections.unmodifiableMap(exactMap);
|
||||
|
||||
Map<String, Object> under = new HashMap<>(30);
|
||||
under.put("bd", "i");
|
||||
under.put("np", "i");
|
||||
under.put("jm", "i");
|
||||
under.put("fj", "i");
|
||||
under.put("fk", "i");
|
||||
under.put("ye", "i");
|
||||
under.put("sch.uk", "i");
|
||||
under.put("bn", "i");
|
||||
under.put("kitakyushu.jp", "i");
|
||||
under.put("kobe.jp", "i");
|
||||
under.put("ke", "i");
|
||||
under.put("sapporo.jp", "i");
|
||||
under.put("kh", "i");
|
||||
under.put("mm", "i");
|
||||
under.put("il", "i");
|
||||
under.put("yokohama.jp", "i");
|
||||
under.put("ck", "i");
|
||||
under.put("nagoya.jp", "i");
|
||||
under.put("sendai.jp", "i");
|
||||
under.put("kw", "i");
|
||||
under.put("er", "i");
|
||||
under.put("mz", "i");
|
||||
under.put("platform.sh", "p");
|
||||
under.put("gu", "i");
|
||||
under.put("nom.br", "i");
|
||||
under.put("zm", "i");
|
||||
under.put("pg", "i");
|
||||
under.put("ni", "i");
|
||||
under.put("kawasaki.jp", "i");
|
||||
under.put("zw", "i");
|
||||
under = Collections.unmodifiableMap(under);
|
||||
Map<String, String> underMap = new HashMap<>(30);
|
||||
underMap.put("bd", "i");
|
||||
underMap.put("np", "i");
|
||||
underMap.put("jm", "i");
|
||||
underMap.put("fj", "i");
|
||||
underMap.put("fk", "i");
|
||||
underMap.put("ye", "i");
|
||||
underMap.put("sch.uk", "i");
|
||||
underMap.put("bn", "i");
|
||||
underMap.put("kitakyushu.jp", "i");
|
||||
underMap.put("kobe.jp", "i");
|
||||
underMap.put("ke", "i");
|
||||
underMap.put("sapporo.jp", "i");
|
||||
underMap.put("kh", "i");
|
||||
underMap.put("mm", "i");
|
||||
underMap.put("il", "i");
|
||||
underMap.put("yokohama.jp", "i");
|
||||
underMap.put("ck", "i");
|
||||
underMap.put("nagoya.jp", "i");
|
||||
underMap.put("sendai.jp", "i");
|
||||
underMap.put("kw", "i");
|
||||
underMap.put("er", "i");
|
||||
underMap.put("mz", "i");
|
||||
underMap.put("platform.sh", "p");
|
||||
underMap.put("gu", "i");
|
||||
underMap.put("nom.br", "i");
|
||||
underMap.put("zm", "i");
|
||||
underMap.put("pg", "i");
|
||||
underMap.put("ni", "i");
|
||||
underMap.put("kawasaki.jp", "i");
|
||||
underMap.put("zw", "i");
|
||||
under = Collections.unmodifiableMap(underMap);
|
||||
|
||||
Map<String, String> excluded = new HashMap<>(9);
|
||||
excluded.put("city.yokohama.jp", "i");
|
||||
excluded.put("teledata.mz", "i");
|
||||
excluded.put("city.kobe.jp", "i");
|
||||
excluded.put("city.sapporo.jp", "i");
|
||||
excluded.put("city.kawasaki.jp", "i");
|
||||
excluded.put("city.nagoya.jp", "i");
|
||||
excluded.put("www.ck", "i");
|
||||
excluded.put("city.sendai.jp", "i");
|
||||
excluded.put("city.kitakyushu.jp", "i");
|
||||
excluded = Collections.unmodifiableMap(excluded);
|
||||
|
||||
|
||||
paramsMap.put("excluded", excluded);
|
||||
paramsMap.put("under", under);
|
||||
paramsMap.put("exact", exact);
|
||||
params = Collections.unmodifiableMap(paramsMap);
|
||||
Map<String, String> excludedMap = new HashMap<>(9);
|
||||
excludedMap.put("city.yokohama.jp", "i");
|
||||
excludedMap.put("teledata.mz", "i");
|
||||
excludedMap.put("city.kobe.jp", "i");
|
||||
excludedMap.put("city.sapporo.jp", "i");
|
||||
excludedMap.put("city.kawasaki.jp", "i");
|
||||
excludedMap.put("city.nagoya.jp", "i");
|
||||
excludedMap.put("www.ck", "i");
|
||||
excludedMap.put("city.sendai.jp", "i");
|
||||
excludedMap.put("city.kitakyushu.jp", "i");
|
||||
excluded = Collections.unmodifiableMap(excludedMap);
|
||||
}
|
||||
|
||||
static {
|
||||
String fn = "String replaceDots(String input) {\n" +
|
||||
" String output = input;\n" +
|
||||
" if (output.indexOf('。') >= 0) {\n" +
|
||||
" output = output.replace('。', '.');\n" +
|
||||
" }\n" +
|
||||
" if (output.indexOf('．') >= 0) {\n" +
|
||||
" output = output.replace('．', '.');\n" +
|
||||
" }\n" +
|
||||
" if (output.indexOf('｡') >= 0) {\n" +
|
||||
" output = output.replace('｡', '.');\n" +
|
||||
" }\n" +
|
||||
" return output;\n" +
|
||||
"}\n" +
|
||||
"List split(String value) {\n" +
|
||||
" int nextWord = 0;\n" +
|
||||
" List splits = [];\n" +
|
||||
" for(int i = 0; i < value.length(); i++) {\n" +
|
||||
" if(value.charAt(i) == (char)'.') {\n" +
|
||||
" splits.add(value.substring(nextWord, i));\n" +
|
||||
" nextWord = i+1;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" if (nextWord != value.length()) {\n" +
|
||||
" splits.add(value.substring(nextWord, value.length()));\n" +
|
||||
" }\n" +
|
||||
" return splits;\n" +
|
||||
"}\n" +
|
||||
"List splitDomain(String domain) {\n" +
|
||||
" String dotDomain = replaceDots(domain);\n" +
|
||||
" return split(dotDomain);\n" +
|
||||
"}\n" +
|
||||
"boolean validateSyntax(List parts) {\n" +
|
||||
" int lastIndex = parts.length - 1;\n" +
|
||||
" /* Validate the last part specially, as it has different syntax rules. */\n" +
|
||||
" if (!validatePart(parts[lastIndex], true)) {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" for (int i = 0; i < lastIndex; i++) {\n" +
|
||||
" String part = parts[i];\n" +
|
||||
" if (!validatePart(part, false)) {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" return true;\n" +
|
||||
"}\n" +
|
||||
"boolean validatePart(String part, boolean isFinalPart) {\n" +
|
||||
" int MAX_DOMAIN_PART_LENGTH = 63;\n" +
|
||||
" if (part.length() < 1 || part.length() > MAX_DOMAIN_PART_LENGTH) {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" int offset = 0;\n" +
|
||||
" int strLen = part.length();\n" +
|
||||
" while (offset < strLen) {\n" +
|
||||
" int curChar = part.charAt(offset);\n" +
|
||||
" offset += 1;\n" +
|
||||
" if (!(Character.isLetterOrDigit(curChar) || curChar == (char)'-' || curChar == (char)'_')) {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" if (part.charAt(0) == (char)'-' || part.charAt(0) == (char)'_' ||\n" +
|
||||
" part.charAt(part.length() - 1) == (char)'-' || part.charAt(part.length() - 1) == (char)'_') {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" if (isFinalPart && Character.isDigit(part.charAt(0))) {\n" +
|
||||
" return false;\n" +
|
||||
" }\n" +
|
||||
" return true;\n" +
|
||||
"}\n" +
|
||||
"int findPublicSuffix(Map params, List parts) {\n" +
|
||||
" int partsSize = parts.size();\n" +
|
||||
"\n" +
|
||||
" for (int i = 0; i < partsSize; i++) {\n" +
|
||||
" StringJoiner joiner = new StringJoiner('.');\n" +
|
||||
" for (String s : parts.subList(i, partsSize)) {\n" +
|
||||
" joiner.add(s);\n" +
|
||||
" }\n" +
|
||||
" /* parts.subList(i, partsSize).each(joiner::add); */\n" +
|
||||
" String ancestorName = joiner.toString();\n" +
|
||||
"\n" +
|
||||
" if (params['exact'].containsKey(ancestorName)) {\n" +
|
||||
" return i;\n" +
|
||||
" }\n" +
|
||||
"\n" +
|
||||
" /* Excluded domains (e.g. !nhs.uk) use the next highest\n" +
|
||||
" domain as the effective public suffix (e.g. uk). */\n" +
|
||||
"\n" +
|
||||
" if (params['excluded'].containsKey(ancestorName)) {\n" +
|
||||
" return i + 1;\n" +
|
||||
" }\n" +
|
||||
"\n" +
|
||||
" List pieces = split(ancestorName);\n" +
|
||||
" if (pieces.length >= 2 && params['under'].containsKey(pieces[1])) {\n" +
|
||||
" return i;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
"\n" +
|
||||
" return -1;\n" +
|
||||
"}\n" +
|
||||
"String ancestor(List parts, int levels) {\n" +
|
||||
" StringJoiner joiner = new StringJoiner('.');\n" +
|
||||
" for (String s : parts.subList(levels, parts.size())) {\n" +
|
||||
" joiner.add(s);\n" +
|
||||
" }\n" +
|
||||
" String name = joiner.toString();\n" +
|
||||
" if (name.endsWith('.')) {\n" +
|
||||
" name = name.substring(0, name.length() - 1);\n" +
|
||||
" }\n" +
|
||||
" return name;\n" +
|
||||
"}\n" +
|
||||
"String topPrivateDomain(String name, List parts, int publicSuffixIndex) {\n" +
|
||||
" if (publicSuffixIndex == 1) {\n" +
|
||||
" return name;\n" +
|
||||
" }\n" +
|
||||
" if (!(publicSuffixIndex > 0)) {\n" +
|
||||
" throw new IllegalArgumentException('Not under a public suffix: ' + name);\n" +
|
||||
" }\n" +
|
||||
" return ancestor(parts, publicSuffixIndex - 1);\n" +
|
||||
"}\n" +
|
||||
"List domainSplit(String host, Map params) {\n" +
|
||||
" int MAX_DNS_NAME_LENGTH = 253;\n" +
|
||||
" int MAX_LENGTH = 253;\n" +
|
||||
" int MAX_PARTS = 127;\n" +
|
||||
" if ('host'.isEmpty()) {\n" +
|
||||
" return ['',''];\n" +
|
||||
" }\n" +
|
||||
" host = host.trim();\n" +
|
||||
" if (host.contains(':')) {\n" +
|
||||
" return ['', host];\n" +
|
||||
" }\n" +
|
||||
" boolean tentativeIP = true;\n" +
|
||||
" for(int i = 0; i < host.length(); i++) {\n" +
|
||||
" if (!(Character.isDigit(host.charAt(i)) || host.charAt(i) == (char)'.')) {\n" +
|
||||
" tentativeIP = false;\n" +
|
||||
" break;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" if (tentativeIP) {\n" +
|
||||
" /* special-snowflake rules now... */\n" +
|
||||
" if (host == '.') {\n" +
|
||||
" return ['',''];\n" +
|
||||
" }\n" +
|
||||
" return ['', host];\n" +
|
||||
" }\n" +
|
||||
" def normalizedHost = host;\n" +
|
||||
" normalizedHost = normalizedHost.toLowerCase();\n" +
|
||||
" List parts = splitDomain(normalizedHost);\n" +
|
||||
" int publicSuffixIndex = findPublicSuffix(params, parts);\n" +
|
||||
" if (publicSuffixIndex == 0) {\n" +
|
||||
" return ['', host];\n" +
|
||||
" }\n" +
|
||||
" String highestRegistered = '';\n" +
|
||||
" /* for the case where the host is internal like .local so is not a recognised public suffix */\n" +
|
||||
" if (publicSuffixIndex == -1) {\n" +
|
||||
" if (!parts.isEmpty()) {\n" +
|
||||
" if (parts.size() == 1) {\n" +
|
||||
" return ['', host];\n" +
|
||||
" }\n" +
|
||||
" if (parts.size() > 2) {\n" +
|
||||
" boolean allNumeric = true;\n" +
|
||||
" String value = parts.get(parts.size() - 1);\n" +
|
||||
" for (int i = 0; i < value.length(); i++) {\n" +
|
||||
" if (!Character.isDigit(value.charAt(i))) {\n" +
|
||||
" allNumeric = false;\n" +
|
||||
" break;\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" if (allNumeric) {\n" +
|
||||
" highestRegistered = parts.get(parts.size() - 2) + '.' + parts.get(parts.size() - 1);\n" +
|
||||
" } else {\n" +
|
||||
" highestRegistered = parts.get(parts.size() - 1);\n" +
|
||||
" }\n" +
|
||||
"\n" +
|
||||
" } else {\n" +
|
||||
" highestRegistered = parts.get(parts.size() - 1);\n" +
|
||||
" }\n" +
|
||||
" }\n" +
|
||||
" } else {\n" +
|
||||
" /* HRD is the top private domain */\n" +
|
||||
" highestRegistered = topPrivateDomain(normalizedHost, parts, publicSuffixIndex);\n" +
|
||||
" }\n" +
|
||||
" String subDomain = host.substring(0, host.length() - highestRegistered.length());\n" +
|
||||
" if (subDomain.endsWith('.')) {\n" +
|
||||
" subDomain = subDomain.substring(0, subDomain.length() - 1);\n" +
|
||||
" }\n" +
|
||||
" return [subDomain, highestRegistered];\n" +
|
||||
"}\n";
|
||||
fn = fn.replace("\n"," ");
|
||||
function = fn;
|
||||
private DomainSplitFunction() {}
|
||||
|
||||
private static String replaceDots(String input) {
|
||||
String output = input;
|
||||
if (output.indexOf('。') >= 0) {
|
||||
output = output.replace('。', '.');
|
||||
}
|
||||
if (output.indexOf('．') >= 0) {
|
||||
output = output.replace('．', '.');
|
||||
}
|
||||
if (output.indexOf('｡') >= 0) {
|
||||
output = output.replace('｡', '.');
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
private static List<String> splitDomain(String domain) {
|
||||
String dotDomain = replaceDots(domain);
|
||||
return Arrays.asList(dotDomain.split("\\."));
|
||||
}
|
||||
|
||||
private static int findPublicSuffix(List<String> parts) {
|
||||
int partsSize = parts.size();
|
||||
for (int i = 0; i < partsSize; i++) {
|
||||
StringJoiner joiner = new StringJoiner(".");
|
||||
for (String s : parts.subList(i, partsSize)) {
|
||||
joiner.add(s);
|
||||
}
|
||||
/* parts.subList(i, partsSize).each(joiner::add); */
|
||||
String ancestorName = joiner.toString();
|
||||
if (exact.containsKey(ancestorName)) {
|
||||
return i;
|
||||
}
|
||||
/* Excluded domains (e.g. !nhs.uk) use the next highest
|
||||
domain as the effective public suffix (e.g. uk). */
|
||||
if (excluded.containsKey(ancestorName)) {
|
||||
return i + 1;
|
||||
}
|
||||
String [] pieces = ancestorName.split("\\.");
|
||||
if (pieces.length >= 2 && under.containsKey(pieces[1])) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
private static String ancestor(List<String> parts, int levels) {
|
||||
StringJoiner joiner = new StringJoiner(".");
|
||||
for (String s : parts.subList(levels, parts.size())) {
|
||||
joiner.add(s);
|
||||
}
|
||||
String name = joiner.toString();
|
||||
if (name.endsWith(".")) {
|
||||
name = name.substring(0, name.length() - 1);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
private static String topPrivateDomain(String name, List<String> parts, int publicSuffixIndex) {
|
||||
if (publicSuffixIndex == 1) {
|
||||
return name;
|
||||
}
|
||||
if (!(publicSuffixIndex > 0)) {
|
||||
throw new IllegalArgumentException("Not under a public suffix: " + name);
|
||||
}
|
||||
return ancestor(parts, publicSuffixIndex - 1);
|
||||
}
|
||||
|
||||
public static List<String> domainSplit(String host, Map<String, Object> params) {
|
||||
// NOTE: we don't check SpecialPermission because this will be called (indirectly) from scripts
|
||||
AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
|
||||
DEPRECATION_LOGGER.deprecatedAndMaybeLog("domainSplit",
|
||||
"Method [domainSplit] taking params is deprecated. Remove the params argument.");
|
||||
return null;
|
||||
});
|
||||
return domainSplit(host);
|
||||
}
|
||||
|
||||
/**
|
||||
* Split {@code host} into sub domain and highest registered domain.
|
||||
* The result is a list containing exactly 2 items the first is the sub domain
|
||||
* and the second the highest registered domain.
|
||||
*
|
||||
* @param host The hostname to split
|
||||
* @return The sub domain and highest registered domain
|
||||
*/
|
||||
public static List<String> domainSplit(String host) {
|
||||
host = host.trim();
|
||||
if (host.contains(":")) {
|
||||
return Arrays.asList("", host);
|
||||
}
|
||||
boolean tentativeIP = true;
|
||||
for(int i = 0; i < host.length(); i++) {
|
||||
if (!(Character.isDigit(host.charAt(i)) || host.charAt(i) == '.')) {
|
||||
tentativeIP = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (tentativeIP) {
|
||||
/* special-snowflake rules now... */
|
||||
if (host.equals(".")) {
|
||||
return Arrays.asList("","");
|
||||
}
|
||||
return Arrays.asList("", host);
|
||||
}
|
||||
String normalizedHost = host;
|
||||
normalizedHost = normalizedHost.toLowerCase(Locale.ROOT);
|
||||
List<String> parts = splitDomain(normalizedHost);
|
||||
int publicSuffixIndex = findPublicSuffix(parts);
|
||||
if (publicSuffixIndex == 0) {
|
||||
return Arrays.asList("", host);
|
||||
}
|
||||
String highestRegistered = "";
|
||||
/* for the case where the host is internal like .local so is not a recognised public suffix */
|
||||
if (publicSuffixIndex == -1) {
|
||||
if (!parts.isEmpty()) {
|
||||
if (parts.size() == 1) {
|
||||
return Arrays.asList("", host);
|
||||
}
|
||||
if (parts.size() > 2) {
|
||||
boolean allNumeric = true;
|
||||
String value = parts.get(parts.size() - 1);
|
||||
for (int i = 0; i < value.length(); i++) {
|
||||
if (!Character.isDigit(value.charAt(i))) {
|
||||
allNumeric = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (allNumeric) {
|
||||
highestRegistered = parts.get(parts.size() - 2) + '.' + parts.get(parts.size() - 1);
|
||||
} else {
|
||||
highestRegistered = parts.get(parts.size() - 1);
|
||||
}
|
||||
|
||||
} else {
|
||||
highestRegistered = parts.get(parts.size() - 1);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
/* HRD is the top private domain */
|
||||
highestRegistered = topPrivateDomain(normalizedHost, parts, publicSuffixIndex);
|
||||
}
|
||||
String subDomain = host.substring(0, host.length() - highestRegistered.length());
|
||||
if (subDomain.endsWith(".")) {
|
||||
subDomain = subDomain.substring(0, subDomain.length() - 1);
|
||||
}
|
||||
return Arrays.asList(subDomain, highestRegistered);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
org.elasticsearch.xpack.ml.MachineLearningPainlessExtension
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
|
||||
static_import {
|
||||
List domainSplit(String) from_class org.elasticsearch.xpack.ml.utils.DomainSplitFunction
|
||||
List domainSplit(String,Map) from_class org.elasticsearch.xpack.ml.utils.DomainSplitFunction
|
||||
}
|
||||
|
|
@ -454,8 +454,6 @@ public class ScrollDataExtractorTests extends ESTestCase {
|
|||
// Check for the scripts
|
||||
assertThat(searchRequest, containsString("{\"script\":{\"source\":\"return 1 + 1;\",\"lang\":\"mockscript\"}"
|
||||
.replaceAll("\\s", "")));
|
||||
assertThat(searchRequest, containsString("List domainSplit(String host, Map params)".replaceAll("\\s", "")));
|
||||
assertThat(searchRequest, containsString("String replaceDots(String input) {".replaceAll("\\s", "")));
|
||||
|
||||
assertThat(capturedContinueScrollIds.size(), equalTo(1));
|
||||
assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId()));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.xpack.ml.utils;
|
||||
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class DomainSplitFunctionTests extends ESTestCase {
|
||||
|
||||
public void testDomainSplit() {
|
||||
// Test cases from https://github.com/john-kurkowski/tldextract/tree/master/tldextract/tests
|
||||
assertDomainSplit("www", "google.com", "www.google.com");
|
||||
assertDomainSplit("www.maps", "google.co.uk", "www.maps.google.co.uk");
|
||||
assertDomainSplit("www", "theregister.co.uk", "www.theregister.co.uk");
|
||||
assertDomainSplit("", "gmail.com", "gmail.com");
|
||||
assertDomainSplit("media.forums", "theregister.co.uk", "media.forums.theregister.co.uk");
|
||||
assertDomainSplit("www", "www.com", "www.www.com");
|
||||
assertDomainSplit("", "www.com", "www.com");
|
||||
assertDomainSplit("", "internalunlikelyhostname", "internalunlikelyhostname");
|
||||
assertDomainSplit("internalunlikelyhostname", "bizarre", "internalunlikelyhostname.bizarre");
|
||||
assertDomainSplit("", "internalunlikelyhostname.info", "internalunlikelyhostname.info"); // .info is a valid TLD
|
||||
assertDomainSplit("internalunlikelyhostname", "information", "internalunlikelyhostname.information");
|
||||
assertDomainSplit("", "216.22.0.192", "216.22.0.192");
|
||||
assertDomainSplit("", "::1", "::1");
|
||||
assertDomainSplit("", "FE80:0000:0000:0000:0202:B3FF:FE1E:8329", "FE80:0000:0000:0000:0202:B3FF:FE1E:8329");
|
||||
assertDomainSplit("216.22", "project.coop", "216.22.project.coop");
|
||||
assertDomainSplit("www", "xn--h1alffa9f.xn--p1ai", "www.xn--h1alffa9f.xn--p1ai");
|
||||
assertDomainSplit("", "", "");
|
||||
assertDomainSplit("www", "parliament.uk", "www.parliament.uk");
|
||||
assertDomainSplit("www", "parliament.co.uk", "www.parliament.co.uk");
|
||||
assertDomainSplit("www.a", "cgs.act.edu.au", "www.a.cgs.act.edu.au");
|
||||
assertDomainSplit("www", "google.com.au", "www.google.com.au");
|
||||
assertDomainSplit("www", "metp.net.cn", "www.metp.net.cn");
|
||||
assertDomainSplit("www", "waiterrant.blogspot.com", "www.waiterrant.blogspot.com");
|
||||
assertDomainSplit("", "kittens.blogspot.co.uk", "kittens.blogspot.co.uk");
|
||||
assertDomainSplit("example", "example", "example.example");
|
||||
assertDomainSplit("b.example", "example", "b.example.example");
|
||||
assertDomainSplit("a.b.example", "example", "a.b.example.example");
|
||||
assertDomainSplit("example", "local", "example.local");
|
||||
assertDomainSplit("b.example", "local", "b.example.local");
|
||||
assertDomainSplit("a.b.example", "local", "a.b.example.local");
|
||||
assertDomainSplit("r192494180984795-1-1041782-channel-live.ums", "ustream.tv",
|
||||
"r192494180984795-1-1041782-channel-live.ums.ustream.tv");
|
||||
}
|
||||
|
||||
private void assertDomainSplit(String expectedSubDomain, String expectedDomain, String hostName) {
|
||||
List<String> split = DomainSplitFunction.domainSplit(hostName);
|
||||
assertEquals(expectedSubDomain, split.get(0));
|
||||
assertEquals(expectedDomain, split.get(1));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue