From 5b40acdc529fdb9498fd2e3e85498212caef9f3a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 14 Sep 2016 11:05:31 +0000 Subject: [PATCH 001/407] disable gradle deamon Original commit: elastic/x-pack-elasticsearch@f93c69bf4044d136cfc2dd15c5d99cb723862644 --- gradle.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 gradle.properties diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000000..6b1823d86a6 --- /dev/null +++ b/gradle.properties @@ -0,0 +1 @@ +org.gradle.daemon=false From f367ecf1e24870c2d18ca6f9831c3c51d95add23 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 16 Sep 2016 17:30:45 +0100 Subject: [PATCH 002/407] Build Native C++ binaries and Java code in a single gradle project. C++ is built by calling make Original commit: elastic/x-pack-elasticsearch@bd52bfd316438b9f3df3db49ad34a966ad016a7d --- build.gradle | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ settings.gradle | 17 ++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 build.gradle create mode 100644 settings.gradle diff --git a/build.gradle b/build.gradle new file mode 100644 index 00000000000..554e4a46ad9 --- /dev/null +++ b/build.gradle @@ -0,0 +1,52 @@ +description = 'Builds the Prelert Engine native binaries and Java classes' + +project.ext.numCpus = Runtime.runtime.availableProcessors() -1 + + +task clean(type: Exec) { + commandLine 'make' + args 'clean', 'BUILD_ENGINE_API=1' +} + +task objcompile(type: Exec) { + commandLine 'make' + args '-j' + numCpus, 'objcompile', 'BUILD_ENGINE_API=1' +} + +task make(type: Exec) { + commandLine 'make' + args '-j' + numCpus, 'BUILD_ENGINE_API=1' +} + +task test(type: Exec) { + commandLine 'make' + args '-j' + numCpus, 'BUILD_ENGINE_API=1', 'test' +} + +subprojects { + apply plugin: 'maven' + apply plugin: 'java' + + group = 'com.prelert' + version = '2.1.1' + sourceCompatibility = 1.8 + targetCompatibility = 1.8 + + + repositories { + mavenLocal() + maven { url "http://repo.maven.apache.org/maven2" } + } + + configurations.all { + } + + dependencies { + compile group: 'log4j', name: 'log4j', version:'1.2.17' + testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3' + testCompile group: 'junit', name: 'junit', version:'4.11' + testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' + testCompile group: 'org.mockito', name: 'mockito-all', version:'1.10.19' + } +} + diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000000..91ed0e442ac --- /dev/null +++ b/settings.gradle @@ -0,0 +1,17 @@ +rootProject.name = 'prelert-engine-api' +include ':engine-api-java' +include ':prelert-engine-api-common' +include ':data-extractors' +include ':engine-api' +include ':prelert-engine-api-client' +include ':engine-api-server' +include ':elasticsearch-persistence' + +project(':engine-api-java').projectDir = "$rootDir/java/apps/engineApi" as File +project(':prelert-engine-api-common').projectDir = "$rootDir/java/apps/engineApi/api-common" as File +project(':data-extractors').projectDir = "$rootDir/java/apps/engineApi/data-extractors" as File +project(':engine-api').projectDir = "$rootDir/java/apps/engineApi/engine-api" as File +project(':prelert-engine-api-client').projectDir = "$rootDir/java/apps/engineApi/apiClient" as File +project(':engine-api-server').projectDir = "$rootDir/java/apps/engineApi/apiServer" as File +project(':elasticsearch-persistence').projectDir = "$rootDir/java/apps/engineApi/elasticsearch-persistence" as File + From c0b1ac948c91499a21372d7eae6007a40cacb7f0 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 19 Sep 2016 14:08:35 +0100 Subject: [PATCH 003/407] More Gradle migration changes Original commit: elastic/x-pack-elasticsearch@2cdd8252246f340b4ad91ae02fc4cd288784f9ea --- build.gradle | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 554e4a46ad9..dd94dad7368 100644 --- a/build.gradle +++ b/build.gradle @@ -1,26 +1,28 @@ description = 'Builds the Prelert Engine native binaries and Java classes' -project.ext.numCpus = Runtime.runtime.availableProcessors() -1 +import org.gradle.internal.os.OperatingSystem +project.ext.make = OperatingSystem.current().isLinux() ? "make" : "gnumake" +project.ext.numCpus = Runtime.runtime.availableProcessors() task clean(type: Exec) { - commandLine 'make' - args 'clean', 'BUILD_ENGINE_API=1' + commandLine make + args 'clean' } task objcompile(type: Exec) { - commandLine 'make' - args '-j' + numCpus, 'objcompile', 'BUILD_ENGINE_API=1' + commandLine make + args '-j' + numCpus, 'objcompile' } task make(type: Exec) { - commandLine 'make' - args '-j' + numCpus, 'BUILD_ENGINE_API=1' + commandLine make + args '-j' + numCpus } -task test(type: Exec) { - commandLine 'make' - args '-j' + numCpus, 'BUILD_ENGINE_API=1', 'test' +task cpptest(type: Exec) { + commandLine make + args '-j' + numCpus, 'test' } subprojects { From 1b667b2584955ab38197d5c854176c7a75a8c05e Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 19 Sep 2016 14:36:40 +0100 Subject: [PATCH 004/407] Further split out C++ Gradle targets to make life easier for people without a C++ build environment Original commit: elastic/x-pack-elasticsearch@eeca100a3b8f418340f19af11d65c53c235d6d90 --- build.gradle | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index dd94dad7368..38313bef07f 100644 --- a/build.gradle +++ b/build.gradle @@ -5,17 +5,17 @@ import org.gradle.internal.os.OperatingSystem project.ext.make = OperatingSystem.current().isLinux() ? "make" : "gnumake" project.ext.numCpus = Runtime.runtime.availableProcessors() -task clean(type: Exec) { +task cppclean(type: Exec) { commandLine make args 'clean' } -task objcompile(type: Exec) { +task cppobjcompile(type: Exec) { commandLine make args '-j' + numCpus, 'objcompile' } -task make(type: Exec) { +task cppmake(type: Exec) { commandLine make args '-j' + numCpus } @@ -29,6 +29,8 @@ subprojects { apply plugin: 'maven' apply plugin: 'java' + compileJava.options.encoding = 'UTF-8' + group = 'com.prelert' version = '2.1.1' sourceCompatibility = 1.8 From f37a86c8803f9c0fc488da786ff0bc8a92a66a5f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 19 Sep 2016 15:00:38 +0100 Subject: [PATCH 005/407] Explicitly state source code encoding is UTF-8 in Gradle build files Original commit: elastic/x-pack-elasticsearch@2f59b2c3a9f619956d73f1d5416d24e6f2ffdd1c --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 38313bef07f..7ea11eeb760 100644 --- a/build.gradle +++ b/build.gradle @@ -30,13 +30,13 @@ subprojects { apply plugin: 'java' compileJava.options.encoding = 'UTF-8' + compileTestJava.options.encoding = 'UTF-8' group = 'com.prelert' version = '2.1.1' sourceCompatibility = 1.8 targetCompatibility = 1.8 - repositories { mavenLocal() maven { url "http://repo.maven.apache.org/maven2" } From b991b34f0e9d2a0f6c798506ed1d128a46c08f2e Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 19 Sep 2016 16:23:45 +0100 Subject: [PATCH 006/407] Add engine-node project to the root gradle project Builds the classes etc but does not copy any artefacts to the build area as the other projects do Original commit: elastic/x-pack-elasticsearch@abc1301c7048e9a9255e19d8483d83eceed0cd9b --- settings.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 91ed0e442ac..7b98a0a82d0 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,6 +6,7 @@ include ':engine-api' include ':prelert-engine-api-client' include ':engine-api-server' include ':elasticsearch-persistence' +include ':engine-node' project(':engine-api-java').projectDir = "$rootDir/java/apps/engineApi" as File project(':prelert-engine-api-common').projectDir = "$rootDir/java/apps/engineApi/api-common" as File @@ -14,4 +15,4 @@ project(':engine-api').projectDir = "$rootDir/java/apps/engineApi/engine-api" as project(':prelert-engine-api-client').projectDir = "$rootDir/java/apps/engineApi/apiClient" as File project(':engine-api-server').projectDir = "$rootDir/java/apps/engineApi/apiServer" as File project(':elasticsearch-persistence').projectDir = "$rootDir/java/apps/engineApi/elasticsearch-persistence" as File - +project(':engine-node').projectDir = "$rootDir/java/apps/engine-node" as File From 2d7b536848381b43224c2141ebb422c5821bc84c Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 20 Sep 2016 15:07:14 +0100 Subject: [PATCH 007/407] Updating version number Original commit: elastic/x-pack-elasticsearch@6427ca02555ed7a8b6520bb7098081a8c4b567dd --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7ea11eeb760..36ccc81ba8c 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ subprojects { compileTestJava.options.encoding = 'UTF-8' group = 'com.prelert' - version = '2.1.1' + version = '2.2.0' sourceCompatibility = 1.8 targetCompatibility = 1.8 From 9414b83e25f4af94094e76a62ce3297f4cde3c63 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 4 Oct 2016 09:18:52 +0100 Subject: [PATCH 008/407] Updating version number Original commit: elastic/x-pack-elasticsearch@027e5a6126f61b123d648afcc022c17861a18a6f --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 36ccc81ba8c..afc40192dce 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ subprojects { compileTestJava.options.encoding = 'UTF-8' group = 'com.prelert' - version = '2.2.0' + version = '2.3.0' sourceCompatibility = 1.8 targetCompatibility = 1.8 From 9fa9dc3fa5fd0cc7927ce4c4352b40a64cd07c97 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 3 Oct 2016 22:36:15 +0200 Subject: [PATCH 009/407] removed forked base test classes and let tests depend on provided base classes. During tests we run now only with jar hell enabled, security manager remains to be disabled. Original commit: elastic/x-pack-elasticsearch@06aebc5ec572633931f87d128d806036bdbafe73 --- build.gradle | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index afc40192dce..55ad5e93360 100644 --- a/build.gradle +++ b/build.gradle @@ -47,10 +47,9 @@ subprojects { dependencies { compile group: 'log4j', name: 'log4j', version:'1.2.17' - testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3' - testCompile group: 'junit', name: 'junit', version:'4.11' testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' - testCompile group: 'org.mockito', name: 'mockito-all', version:'1.10.19' + // Includes: junit, hamcrest and mockito + testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-alpha5' } } From aba1447b674c7f81611ee62787ed7f8d382c822d Mon Sep 17 00:00:00 2001 From: Dimitrios Athanasiou Date: Tue, 4 Oct 2016 16:11:30 +0100 Subject: [PATCH 010/407] Upgrade engine-node to beta1 and clean up after merge Original commit: elastic/x-pack-elasticsearch@745043a99afeb1eb349bfd287911592194e324c5 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 55ad5e93360..7c571f39338 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ subprojects { compile group: 'log4j', name: 'log4j', version:'1.2.17' testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' // Includes: junit, hamcrest and mockito - testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-alpha5' + testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-beta1' } } From 2af02b04e1ffeaf39b404707fd7d27edfd709ef5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 4 Oct 2016 20:08:24 +0200 Subject: [PATCH 011/407] really remove old log4j from old engine-node module. (only the engine-api-java module and its submodules remain to use old log4j) Original commit: elastic/x-pack-elasticsearch@18a761eb9aee05c200b0ebf92b3d6a0261966644 --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7c571f39338..97d28d4ef1d 100644 --- a/build.gradle +++ b/build.gradle @@ -46,7 +46,6 @@ subprojects { } dependencies { - compile group: 'log4j', name: 'log4j', version:'1.2.17' testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' // Includes: junit, hamcrest and mockito testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-beta1' From 19868a97bf277f91d35a962b4af2125a460a3b9a Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 5 Oct 2016 10:00:13 -0400 Subject: [PATCH 012/407] Add Vagrant build environment (elastic/elasticsearch#38) This adds a vagrant configuration under /vagrant. This allows the user to provision a virtual machine with the entire build environment needed for compiling the C++ portion of the code (as well as the Java side, but that can be done easily outside of vagrant). Original commit: elastic/x-pack-elasticsearch@12754bd80e1d2e40dc3e70fd2dc405cf5abb1b0d --- vagrant/.gitignore | 1 + vagrant/README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++ vagrant/Vagrantfile | 29 ++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 vagrant/.gitignore create mode 100644 vagrant/README.md create mode 100644 vagrant/Vagrantfile diff --git a/vagrant/.gitignore b/vagrant/.gitignore new file mode 100644 index 00000000000..8000dd9db47 --- /dev/null +++ b/vagrant/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/vagrant/README.md b/vagrant/README.md new file mode 100644 index 00000000000..831c2b42c25 --- /dev/null +++ b/vagrant/README.md @@ -0,0 +1,83 @@ +## Vagrant build environment + +This provides a vagrant box for building the C++ side of Prelert (and the Java side, +although that is easily accomplished outside vagrant). + +Provisioning the box will take a fair amount of time, since it needs to download +and compile a number of dependencies. + +A pre-provisioned box can be downloaded from: TODO + +### Details +- Ubuntu Trusty64 (14.04.5 LTS) +- 25% of host's memory +- 100% host's cores +- Maps prelert source repository to `/home/vagrant/prelert/src` + - Directory is shared with the host, so you can point your IDE to the prelert repo + and build inside vagrant +- Maps prelert build directory to `/home/vagrant/prelert/build` +- Changes into `/home/vagrant/prelert/src` on login + +### Usage + +```bash +$ cd prelert-legacy +$ cd vagrant +$ vagrant up + # ... + # wait while vagrant provisions + # ... +$ vagrant ssh + +Welcome to Ubuntu 14.04.5 LTS (GNU/Linux 3.13.0-96-generic x86_64) + + * Documentation: https://help.ubuntu.com/ + + System information as of Tue Oct 4 16:06:31 UTC 2016 + + System load: 0.0 Processes: 196 + Usage of /: 12.0% of 39.34GB Users logged in: 0 + Memory usage: 2% IP address for eth0: 10.0.2.15 + Swap usage: 0% + + Graph this data and manage this system at: + https://landscape.canonical.com/ + + Get cloud support with Ubuntu Advantage Cloud Guest: + http://www.ubuntu.com/business/services/cloud + +New release '16.04.1 LTS' available. +Run 'do-release-upgrade' to upgrade to it. + + +Last login: Tue Oct 4 16:06:32 2016 from 10.0.2.2 +vagrant@vagrant-ubuntu-trusty-64:~/prelert/src$ +``` + +Once you've logged into the box, you'll be in the prelert source directory. You +can build immediately via: + +```bash +vagrant@vagrant-ubuntu-trusty-64:~/prelert/src$ gradle cppmake + # ... + # much building + # ... +``` + +### Suspending your box +Once you've provisioned a box, you can use `vagrant suspend` to "sleep" the box. +This has the advantage of rebooting quickly when you `vagrant up`, and returns you +to exactly what you were doing. On the downside, it eats more disk space since it +needs to sleep the entire image. + +You can alternatively use `vagrant halt`, which gracefully powers down the machine. +Rebooting via `vagrant up` takes longer since you are rebooting the entire OS, +but it saves more disk space. + +### Fixing a broken box +If you accidentally kill the provisioning process before it completes, you can +attempt to reprovision it with `vagrant reload --provision`. That will run +the provisioners that did not complete previously. + +If your box is still horribly broken, you can destroy it with `vagrant destroy` +and try again with `vagrant up` diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile new file mode 100644 index 00000000000..b2caf1ada6e --- /dev/null +++ b/vagrant/Vagrantfile @@ -0,0 +1,29 @@ +Vagrant.configure(2) do |config| + config.vm.box = "ubuntu/trusty64" + config.vm.provider "virtualbox" do |v| + host = RbConfig::CONFIG['host_os'] + + # Give VM 1/4 system memory + linux = RUBY_PLATFORM =~ /linux/ + osx = RUBY_PLATFORM =~ /darwin/ + windows = (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil + if osx + cpus = `sysctl -n hw.ncpu`.to_i + mem = `sysctl -n hw.memsize`.to_i / 1024 / 1024 + end + if linux + cpus = `nproc`.to_i + mem = `sed -n -e '/^MemTotal/s/^[^0-9]*//p' /proc/meminfo`.to_i / 1024 + end + if windows + cpus = `wmic computersystem get numberofprocessors`.split("\n")[2].to_i + mem = `wmic OS get TotalVisibleMemorySize`.split("\n")[2].to_i / 1024 + end + + mem = mem / 4 + v.customize ["modifyvm", :id, "--memory", mem] + v.customize ["modifyvm", :id, "--cpus", cpus] + end + config.vm.provision :shell, path: "provision.sh" + config.vm.synced_folder "../", "/home/vagrant/prelert/src", mount_options: ["dmode=777,fmode=777"] +end From a2bc51da13cdeea88a71b24e44331ab710c672ab Mon Sep 17 00:00:00 2001 From: polyfractal Date: Thu, 6 Oct 2016 10:09:27 -0400 Subject: [PATCH 013/407] Add details about pre-baked vagrant box Original commit: elastic/x-pack-elasticsearch@be7d4efad1d426c1dc66145b46502ddd785e4920 --- vagrant/README.md | 69 ++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/vagrant/README.md b/vagrant/README.md index 831c2b42c25..75658f84109 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -6,7 +6,6 @@ although that is easily accomplished outside vagrant). Provisioning the box will take a fair amount of time, since it needs to download and compile a number of dependencies. -A pre-provisioned box can be downloaded from: TODO ### Details - Ubuntu Trusty64 (14.04.5 LTS) @@ -18,39 +17,59 @@ A pre-provisioned box can be downloaded from: TODO - Maps prelert build directory to `/home/vagrant/prelert/build` - Changes into `/home/vagrant/prelert/src` on login -### Usage +### Pre-baked box +Don't feel like compiling the entire box? No fear, there's a pre-baked box available +on S3. It is ~1.1gb to download: ```bash -$ cd prelert-legacy -$ cd vagrant +# must change into the vagrant directory first so that +# synced folders continue working +$ cd prelert-legacy/vagrant +$ s3cmd get s3://prelert-elastic-dump/prelert_env.box + # ... + # Downloading... + # ... + +$ vagrant box add prelert prelert_env.box +$ vagrant init prelert +$ vagrant up +$ vagrant ssh + + Welcome to Ubuntu 14.04.5 LTS (GNU/Linux 3.13.0-96-generic x86_64) + ... + ... + Last login: Tue Oct 4 16:06:32 2016 from 10.0.2.2 + +vagrant@vagrant-ubuntu-trusty-64:~/prelert/src$ +``` + +Once you've logged into the box, you'll be in the prelert source directory. You +can build immediately via: + +```bash +vagrant@vagrant-ubuntu-trusty-64:~/prelert/src$ gradle cppmake +``` +The pre-baked box has already compiled prelert once, so subsequent compilations +should happen considerably faster. + +### Compiling from Scratch + +If you feel like compiling everything from scratch instead of downloading the pre-baked +box, simply `vagrant up` and let the provisioners run: + +```bash +$ cd prelert-legacy/vagrant $ vagrant up # ... # wait while vagrant provisions # ... $ vagrant ssh -Welcome to Ubuntu 14.04.5 LTS (GNU/Linux 3.13.0-96-generic x86_64) + Welcome to Ubuntu 14.04.5 LTS (GNU/Linux 3.13.0-96-generic x86_64) + ... + ... + Last login: Tue Oct 4 16:06:32 2016 from 10.0.2.2 - * Documentation: https://help.ubuntu.com/ - - System information as of Tue Oct 4 16:06:31 UTC 2016 - - System load: 0.0 Processes: 196 - Usage of /: 12.0% of 39.34GB Users logged in: 0 - Memory usage: 2% IP address for eth0: 10.0.2.15 - Swap usage: 0% - - Graph this data and manage this system at: - https://landscape.canonical.com/ - - Get cloud support with Ubuntu Advantage Cloud Guest: - http://www.ubuntu.com/business/services/cloud - -New release '16.04.1 LTS' available. -Run 'do-release-upgrade' to upgrade to it. - - -Last login: Tue Oct 4 16:06:32 2016 from 10.0.2.2 vagrant@vagrant-ubuntu-trusty-64:~/prelert/src$ ``` From 55a828e15b1674bfcedcd27814c3ce992f59eee3 Mon Sep 17 00:00:00 2001 From: polyfractal Date: Tue, 11 Oct 2016 11:07:41 -0400 Subject: [PATCH 014/407] Tweak Vagrant to use env variables for S3 boxes, update instructions Original commit: elastic/x-pack-elasticsearch@20e657113e9f76ab2900db2cbe5fb7eb5f488c86 --- vagrant/README.md | 12 +++++++++--- vagrant/Vagrantfile | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/vagrant/README.md b/vagrant/README.md index 75658f84109..41b25d16c6d 100644 --- a/vagrant/README.md +++ b/vagrant/README.md @@ -22,9 +22,15 @@ Don't feel like compiling the entire box? No fear, there's a pre-baked box avai on S3. It is ~1.1gb to download: ```bash -# must change into the vagrant directory first so that -# synced folders continue working -$ cd prelert-legacy/vagrant +# Change into some random directory to download the box. +# Doesn't matter where this goes, but *cannot* go into prelert-legacy/vagrant +$ cd ~/some_directory + +# Export the path to your prelert-legacy repo. This is so the box knows where +# to sync the folders +$ export PRELERT_SRC_HOME=/path/to/prelert-legacy + +# Download the box from S3 $ s3cmd get s3://prelert-elastic-dump/prelert_env.box # ... # Downloading... diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index b2caf1ada6e..481b23d3f97 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -24,6 +24,14 @@ Vagrant.configure(2) do |config| v.customize ["modifyvm", :id, "--memory", mem] v.customize ["modifyvm", :id, "--cpus", cpus] end + + if File.expand_path(File.dirname(__FILE__)).include? "prelert-legacy/vagrant" + puts "Syncing host's source directory [" + File.expand_path("../") + "] to [/home/vagrant/prelert/src]" + config.vm.synced_folder "../", "/home/vagrant/prelert/src", mount_options: ["dmode=777,fmode=777"] + else + puts "Syncing host's source directory [" + File.expand_path(ENV['PRELERT_SRC_HOME']) + "] to [/home/vagrant/prelert/src] (via $PRELERT_SRC_HOME)" + config.vm.synced_folder ENV['PRELERT_SRC_HOME'], "/home/vagrant/prelert/src", mount_options: ["dmode=777,fmode=777"] + end + config.vm.provision :shell, path: "provision.sh" - config.vm.synced_folder "../", "/home/vagrant/prelert/src", mount_options: ["dmode=777,fmode=777"] end From 926c04c00d74c2b1527bbcd43cd66ddea4b6d538 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 14 Oct 2016 14:46:47 +0200 Subject: [PATCH 015/407] upgraded to 5.0.0-rc1 Original commit: elastic/x-pack-elasticsearch@687f9669db2462a5a5f26bf5cce7fbdc9564603b --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 97d28d4ef1d..44807bbd11f 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ subprojects { dependencies { testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' // Includes: junit, hamcrest and mockito - testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-beta1' + testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-rc1' } } From 791f74ded9693f5986edae6d2de2e0978d67dc08 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 26 Oct 2016 16:42:27 +0200 Subject: [PATCH 016/407] 5.0 upgrade Original commit: elastic/x-pack-elasticsearch@9627dadb00b99c2567ac698931aa011d6514cbdb --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 44807bbd11f..6a3f8c4b91e 100644 --- a/build.gradle +++ b/build.gradle @@ -48,7 +48,7 @@ subprojects { dependencies { testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' // Includes: junit, hamcrest and mockito - testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0-rc1' + testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0' } } From 3ea5e62e989f989d8cf870cb0922da37e5843970 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 28 Oct 2016 11:16:35 +0100 Subject: [PATCH 017/407] Upgrade the analytics-dev branch to use GA Elasticsearch and Kibana 5.0 (elastic/elasticsearch#188) Disabled the build of the incomplete engine-node project as it no longer builds with the GA versions Original commit: elastic/x-pack-elasticsearch@e07820f228c1022710b039da5371a08654f4b97c --- settings.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/settings.gradle b/settings.gradle index 7b98a0a82d0..fda9a221702 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,7 +6,6 @@ include ':engine-api' include ':prelert-engine-api-client' include ':engine-api-server' include ':elasticsearch-persistence' -include ':engine-node' project(':engine-api-java').projectDir = "$rootDir/java/apps/engineApi" as File project(':prelert-engine-api-common').projectDir = "$rootDir/java/apps/engineApi/api-common" as File @@ -15,4 +14,3 @@ project(':engine-api').projectDir = "$rootDir/java/apps/engineApi/engine-api" as project(':prelert-engine-api-client').projectDir = "$rootDir/java/apps/engineApi/apiClient" as File project(':engine-api-server').projectDir = "$rootDir/java/apps/engineApi/apiServer" as File project(':elasticsearch-persistence').projectDir = "$rootDir/java/apps/engineApi/elasticsearch-persistence" as File -project(':engine-node').projectDir = "$rootDir/java/apps/engine-node" as File From 68ecc10ea70ef7c4a2324e064faeccd1703db876 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Mon, 31 Oct 2016 13:05:32 +0000 Subject: [PATCH 018/407] Adds ide build gradle config (elastic/elasticsearch#203) Makes the ides work the same as the main ES project Original commit: elastic/x-pack-elasticsearch@3ca9d78ea96f7c02317b7c5a0e468bbc5613e4b4 --- build.gradle | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/build.gradle b/build.gradle index 6a3f8c4b91e..bb06ceaffb2 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,8 @@ description = 'Builds the Prelert Engine native binaries and Java classes' import org.gradle.internal.os.OperatingSystem +import org.gradle.plugins.ide.eclipse.model.SourceFolder +import org.apache.tools.ant.taskdefs.condition.Os project.ext.make = OperatingSystem.current().isLinux() ? "make" : "gnumake" project.ext.numCpus = Runtime.runtime.availableProcessors() @@ -52,3 +54,97 @@ subprojects { } } +allprojects { + // injecting groovy property variables into all projects + project.ext { + // for ide hacks... + isEclipse = System.getProperty("eclipse.launcher") != null || gradle.startParameter.taskNames.contains('eclipse') || gradle.startParameter.taskNames.contains('cleanEclipse') + isIdea = System.getProperty("idea.active") != null || gradle.startParameter.taskNames.contains('idea') || gradle.startParameter.taskNames.contains('cleanIdea') + } +} + +allprojects { + apply plugin: 'idea' + + if (isIdea) { + project.buildDir = file('build-idea') + } + idea { + module { + inheritOutputDirs = false + outputDir = file('build-idea/classes/main') + testOutputDir = file('build-idea/classes/test') + + // also ignore other possible build dirs + excludeDirs += file('build') + excludeDirs += file('build-eclipse') + + iml { + // fix so that Gradle idea plugin properly generates support for resource folders + // see also https://issues.gradle.org/browse/GRADLE-2975 + withXml { + it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/main/resources' }.each { + it.attributes().remove('isTestSource') + it.attributes().put('type', 'java-resource') + } + it.asNode().component.content.sourceFolder.findAll { it.@url == 'file://$MODULE_DIR$/src/test/resources' }.each { + it.attributes().remove('isTestSource') + it.attributes().put('type', 'java-test-resource') + } + } + } + } + } +} + +// Make sure gradle idea was run before running anything in intellij (including import). +File ideaMarker = new File(projectDir, '.local-idea-is-configured') +tasks.idea.doLast { + ideaMarker.setText('', 'UTF-8') +} +if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) { + throw new GradleException('You must run gradle idea from the root of elasticsearch before importing into IntelliJ') +} + +// eclipse configuration +allprojects { + apply plugin: 'eclipse' + // Name all the non-root projects after their path so that paths get grouped together when imported into eclipse. + if (path != ':') { + eclipse.project.name = path + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + eclipse.project.name = eclipse.project.name.replace(':', '_') + } + } + + plugins.withType(JavaBasePlugin) { + File eclipseBuild = project.file('build-eclipse') + eclipse.classpath.defaultOutputDir = eclipseBuild + if (isEclipse) { + // set this so generated dirs will be relative to eclipse build + project.buildDir = eclipseBuild + } + eclipse.classpath.file.whenMerged { classpath -> + // give each source folder a unique corresponding output folder + int i = 0; + classpath.entries.findAll { it instanceof SourceFolder }.each { folder -> + i++; + // this is *NOT* a path or a file. + folder.output = "build-eclipse/" + i + } + } + } + task copyEclipseSettings(type: Copy) { + // TODO: "package this up" for external builds + from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings') + into '.settings' + } + // otherwise .settings is not nuked entirely + task wipeEclipseSettings(type: Delete) { + delete '.settings' + } + tasks.cleanEclipse.dependsOn(wipeEclipseSettings) + // otherwise the eclipse merging is *super confusing* + tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings) +} + From f08faff0409422f65eaef0653791279b2bcb0b5f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 1 Nov 2016 10:19:17 +0000 Subject: [PATCH 019/407] Removed old Java projects (elastic/elasticsearch#208) They'll soon cease to work at all when $PRELERT_HOME is removed Any remaining code that needs to be ported can be copied from the analytics-dev branch Original commit: elastic/x-pack-elasticsearch@1f2d97b429cddf34475d15c5058fb5164a98f7bb --- settings.gradle | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/settings.gradle b/settings.gradle index 7b98a0a82d0..39e8e209c2a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,18 +1,4 @@ rootProject.name = 'prelert-engine-api' -include ':engine-api-java' -include ':prelert-engine-api-common' -include ':data-extractors' -include ':engine-api' -include ':prelert-engine-api-client' -include ':engine-api-server' -include ':elasticsearch-persistence' include ':engine-node' -project(':engine-api-java').projectDir = "$rootDir/java/apps/engineApi" as File -project(':prelert-engine-api-common').projectDir = "$rootDir/java/apps/engineApi/api-common" as File -project(':data-extractors').projectDir = "$rootDir/java/apps/engineApi/data-extractors" as File -project(':engine-api').projectDir = "$rootDir/java/apps/engineApi/engine-api" as File -project(':prelert-engine-api-client').projectDir = "$rootDir/java/apps/engineApi/apiClient" as File -project(':engine-api-server').projectDir = "$rootDir/java/apps/engineApi/apiServer" as File -project(':elasticsearch-persistence').projectDir = "$rootDir/java/apps/engineApi/elasticsearch-persistence" as File project(':engine-node').projectDir = "$rootDir/java/apps/engine-node" as File From 3033674607587434cf4a7fc7c2704d1aab239c4d Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 2 Nov 2016 15:41:12 +0000 Subject: [PATCH 020/407] Fix some Jenkins build issues (elastic/elasticsearch#221) 1) The C++ 3rd party libraries need to be copied to cppdistribution before gradle assemble runs 2) Use gradle check instead of gradle test in the build 3) We don't need Maven or pbzip2 any more 4) Make new_version.sh script update the right files 5) Keep a copy of the built plugin after the build Outstanding TODOs: 1) Versioning needs to be brought in line with Elasticsearch 2) We are building a plugin per platform rather than one containing all platforms - we need a step that runs when all platforms have completed to create a single plugin 3) Strip the C++ binaries so the uploads aren't so big Original commit: elastic/x-pack-elasticsearch@0cf32e134ff08d4783d50a98e7c08203a11ca891 --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index bb06ceaffb2..62dbda7d9ea 100644 --- a/build.gradle +++ b/build.gradle @@ -35,7 +35,6 @@ subprojects { compileTestJava.options.encoding = 'UTF-8' group = 'com.prelert' - version = '2.3.0' sourceCompatibility = 1.8 targetCompatibility = 1.8 From c6dadacb29416260568b1ebae37851e892d602f8 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Mon, 7 Nov 2016 09:27:22 +0000 Subject: [PATCH 021/407] Changes the build to track elasticsearch 5.1.0-SNAPSHOT (elastic/elasticsearch#248) Original commit: elastic/x-pack-elasticsearch@af689298f71960b0a2abe6ccce3ee8a2cd891453 --- build.gradle | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index 62dbda7d9ea..9ffde5f7a8b 100644 --- a/build.gradle +++ b/build.gradle @@ -28,19 +28,41 @@ task cpptest(type: Exec) { } subprojects { + apply plugin: 'eclipse' + apply plugin: 'idea' apply plugin: 'maven' apply plugin: 'java' + + buildscript { + repositories { + if (System.getProperty("repos.mavenlocal") != null) { + // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is useful for building against + // elasticsearch snapshots + mavenLocal() + } + mavenCentral() + maven { + name 'sonatype-snapshots' + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + jcenter() + } + } - compileJava.options.encoding = 'UTF-8' - compileTestJava.options.encoding = 'UTF-8' - - group = 'com.prelert' - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + group = 'org.elasticsearch.prelert' repositories { - mavenLocal() - maven { url "http://repo.maven.apache.org/maven2" } + if (System.getProperty("repos.mavenlocal") != null) { + // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is useful for building against + // elasticsearch snapshots + mavenLocal() + } + mavenCentral() + maven { + name 'sonatype-snapshots' + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + jcenter() } configurations.all { @@ -48,8 +70,6 @@ subprojects { dependencies { testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' - // Includes: junit, hamcrest and mockito - testCompile group: 'org.elasticsearch.test', name: 'framework', version: '5.0.0' } } From cd04814cfb16d88d24e6040e833ed659795b7fcd Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 17 Nov 2016 13:12:26 +0000 Subject: [PATCH 022/407] Change the Jenkins build entry point from shell script to gradle (elastic/elasticsearch#317) Other related changes: 1) The Windows build is now done on Windows Server 2012r2 2) The Windows build is now done using Visual Studio 2013 Express (closes elastic/elasticsearch#256) 3) set_env.sh is only used with the C++ build 4) It is no longer necessary to set PRELERT_SRC_HOME if building via gradle - this environment variable IS still required for make, but gradle sets PRELERT_SRC_HOME before calling make 5) No build tools are loaded from the Prelert NAS 6) The version number for C++ components is now picked up from the same gradle.properties as the version number for Java components (closes elastic/elasticsearch#37) 7) "make test" now propagates error returns even when PRELERT_KEEP_GOING is defined - this means gradle only needs to check the return code of make 8) The C++ processes now print out Elasticsearch BV as the copyright owner Jenkins implications: 1) Jenkins installs gradle 2.13 as part of the build process 2) Jenkins now does the build by executing these gradle tasks: cppAll, pluginAll 3) Jenkins must define the JAVA_HOME environment variables 4) The Jenkins slave running on Windows now has a standard setup, i.e. using cmd for its shell instead of Git bash 5) 64 bit Git for Windows is used on Windows build slaves instead of 32 bit msysgit Original commit: elastic/x-pack-elasticsearch@01c863bfd0c3cbe3310e05fea1b3d9502ef52a0a --- build.gradle | 43 ++++++++++++++++++++++++++++--------------- gradle.properties | 2 ++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/build.gradle b/build.gradle index 9ffde5f7a8b..8654f59fdf8 100644 --- a/build.gradle +++ b/build.gradle @@ -2,29 +2,41 @@ description = 'Builds the Prelert Engine native binaries and Java classes' import org.gradle.internal.os.OperatingSystem import org.gradle.plugins.ide.eclipse.model.SourceFolder -import org.apache.tools.ant.taskdefs.condition.Os -project.ext.make = OperatingSystem.current().isLinux() ? "make" : "gnumake" +boolean isLinux = OperatingSystem.current().isLinux() +boolean isMacOsX = OperatingSystem.current().isMacOsX() +boolean isWindows = OperatingSystem.current().isWindows() + +project.ext.bash = isWindows ? "C:\\Program Files\\Git\\bin\\bash" : "/bin/bash" +project.ext.make = (isMacOsX || isWindows) ? "gnumake" : (isLinux ? "make" : "gmake") project.ext.numCpus = Runtime.runtime.availableProcessors() -task cppclean(type: Exec) { - commandLine make - args 'clean' +task cppClean(type: Exec) { + commandLine bash + args '-c', 'source cpp/set_env.sh && ' + make + ' clean' } -task cppobjcompile(type: Exec) { - commandLine make - args '-j' + numCpus, 'objcompile' +task cppObjCompile(type: Exec) { + commandLine bash + args '-c', 'source cpp/set_env.sh && ' + make + ' -j' + numCpus + ' objcompile' } -task cppmake(type: Exec) { - commandLine make - args '-j' + numCpus +task cppMake(type: Exec) { + commandLine bash + args '-c', 'source cpp/set_env.sh && ' + make + ' -j' + numCpus } -task cpptest(type: Exec) { - commandLine make - args '-j' + numCpus, 'test' +task cppTest(type: Exec) { + commandLine bash + args '-c', 'source cpp/set_env.sh && ' + make + ' -j' + numCpus + ' test' +} + +task cppAll { + dependsOn 'cppObjCompile' + dependsOn 'cppMake' + dependsOn 'cppTest' + tasks.findByName('cppMake').mustRunAfter 'cppObjCompile' + tasks.findByName('cppTest').mustRunAfter 'cppMake' } subprojects { @@ -82,6 +94,7 @@ allprojects { } } +// intellij configuration allprojects { apply plugin: 'idea' @@ -131,7 +144,7 @@ allprojects { // Name all the non-root projects after their path so that paths get grouped together when imported into eclipse. if (path != ':') { eclipse.project.name = path - if (Os.isFamily(Os.FAMILY_WINDOWS)) { + if (isWindows) { eclipse.project.name = eclipse.project.name.replace(':', '_') } } diff --git a/gradle.properties b/gradle.properties index 6b1823d86a6..dcb55036615 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1,3 @@ org.gradle.daemon=false + +elasticsearchVersion=5.1.0-SNAPSHOT From 83f2997ee0db1f471776b52f9bdab87a6cc74f93 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 17 Nov 2016 14:07:20 +0000 Subject: [PATCH 023/407] Add infrastructure to the build for documentation (elastic/elasticsearch#320) Original commit: elastic/x-pack-elasticsearch@7efa8c3f98b3acbcf24755c4867a1651b454fadf --- build.gradle | 39 ++++++++++++++++++++++++++++++++------- settings.gradle | 2 ++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 8654f59fdf8..fc016a8b0ca 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,35 @@ description = 'Builds the Prelert Engine native binaries and Java classes' import org.gradle.internal.os.OperatingSystem import org.gradle.plugins.ide.eclipse.model.SourceFolder +import org.elasticsearch.gradle.precommit.LicenseHeadersTask + +buildscript { + repositories { + if (System.getProperty("repos.mavenlocal") != null) { + // with -Drepos.mavenlocal=true we can force checking the local .m2 repo which is useful for building against + // elasticsearch snapshots + mavenLocal() + } + mavenCentral() + maven { + name 'sonatype-snapshots' + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + jcenter() + } + dependencies { + classpath "org.elasticsearch.gradle:build-tools:${elasticsearchVersion}" + } +} + +subprojects { + // we must not publish to sonatype until we have set up x-plugins to only publish the parts we want to publish! + project.afterEvaluate { + if (project.plugins.hasPlugin('com.bmuschko.nexus') && project.nexus.repositoryUrl.startsWith('file://') == false) { + uploadArchives.enabled = false + } + } +} boolean isLinux = OperatingSystem.current().isLinux() boolean isMacOsX = OperatingSystem.current().isMacOsX() @@ -43,7 +72,6 @@ subprojects { apply plugin: 'eclipse' apply plugin: 'idea' apply plugin: 'maven' - apply plugin: 'java' buildscript { repositories { @@ -77,11 +105,9 @@ subprojects { jcenter() } - configurations.all { - } - - dependencies { - testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' + tasks.withType(LicenseHeadersTask.class) { + approvedLicenses = ['Elasticsearch Confidential'] + additionalLicense 'ESCON', 'Elasticsearch Confidential', 'ELASTICSEARCH CONFIDENTIAL' } } @@ -179,4 +205,3 @@ allprojects { // otherwise the eclipse merging is *super confusing* tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings) } - diff --git a/settings.gradle b/settings.gradle index 39e8e209c2a..71c1ed0e7e7 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,6 @@ rootProject.name = 'prelert-engine-api' include ':engine-node' +include ':docs' project(':engine-node').projectDir = "$rootDir/java/apps/engine-node" as File +project(':docs').projectDir = "$rootDir/docs" as File From 155ee948efe2414f91bdc9c03f83d78429d777a2 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Thu, 17 Nov 2016 19:34:39 +0000 Subject: [PATCH 024/407] Strip the C++ binaries when built via gradle to reduce download size (elastic/elasticsearch#323) By default this isn't done when building directly with make, so developers should still have debugging symbols in binaries they build themselves Original commit: elastic/x-pack-elasticsearch@64cd92431b8dbf9b591c77b6602949c1df531564 --- build.gradle | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fc016a8b0ca..41093322d19 100644 --- a/build.gradle +++ b/build.gradle @@ -55,6 +55,11 @@ task cppMake(type: Exec) { args '-c', 'source cpp/set_env.sh && ' + make + ' -j' + numCpus } +task cppStrip(type: Exec) { + commandLine bash + args '-c', 'source cpp/set_env.sh && cpp/strip_binaries.sh' +} + task cppTest(type: Exec) { commandLine bash args '-c', 'source cpp/set_env.sh && ' + make + ' -j' + numCpus + ' test' @@ -63,9 +68,11 @@ task cppTest(type: Exec) { task cppAll { dependsOn 'cppObjCompile' dependsOn 'cppMake' + dependsOn 'cppStrip' dependsOn 'cppTest' tasks.findByName('cppMake').mustRunAfter 'cppObjCompile' - tasks.findByName('cppTest').mustRunAfter 'cppMake' + tasks.findByName('cppStrip').mustRunAfter 'cppMake' + tasks.findByName('cppTest').mustRunAfter 'cppStrip' } subprojects { From a77f6df3f2e56c211949c7819727e36592cac067 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 18 Nov 2016 09:34:18 +0100 Subject: [PATCH 025/407] attempt to always download latest snapshot Original commit: elastic/x-pack-elasticsearch@869760c9dfbfc32954cf78e0e7415817950b0f9e --- build.gradle | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 41093322d19..8f2f82d43c3 100644 --- a/build.gradle +++ b/build.gradle @@ -4,6 +4,11 @@ import org.gradle.internal.os.OperatingSystem import org.gradle.plugins.ide.eclipse.model.SourceFolder import org.elasticsearch.gradle.precommit.LicenseHeadersTask +configurations.all { + // check for updates every build + resolutionStrategy.cacheChangingModulesFor 0, 'seconds' +} + buildscript { repositories { if (System.getProperty("repos.mavenlocal") != null) { @@ -19,7 +24,7 @@ buildscript { jcenter() } dependencies { - classpath "org.elasticsearch.gradle:build-tools:${elasticsearchVersion}" + classpath group: 'org.elasticsearch.gradle', name: 'build-tools', version: "${elasticsearchVersion}", changing: true } } From 85b5220c595a519f621b13c7349af9346d205a4c Mon Sep 17 00:00:00 2001 From: David Roberts Date: Fri, 18 Nov 2016 13:44:41 +0000 Subject: [PATCH 026/407] cppClean task should remove old cppdistribution contents The `git clean` we used to do at the beginning of the build used to take care of this, but due to long paths on Windows we can no longer use this Original commit: elastic/x-pack-elasticsearch@3210c7507ba4ce1e02f521883b69fb96f6b75d04 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8f2f82d43c3..23c3e09ad38 100644 --- a/build.gradle +++ b/build.gradle @@ -47,7 +47,7 @@ project.ext.numCpus = Runtime.runtime.availableProcessors() task cppClean(type: Exec) { commandLine bash - args '-c', 'source cpp/set_env.sh && ' + make + ' clean' + args '-c', 'source cpp/set_env.sh && rm -rf cppdistribution && ' + make + ' clean' } task cppObjCompile(type: Exec) { From 06df439db0219c9934382a413ce17c5709858350 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Fri, 18 Nov 2016 15:21:39 +0000 Subject: [PATCH 027/407] Moves Java code to fit x-pack style structure This change moves the Java code from `/java/apps/engine-node` to `/elasticsearch` so it matches the structure required by X-Pack. This will make it easier to get the gradle build working correct and building the full Pack zip. Original commit: elastic/x-pack-elasticsearch@2fd6539e851f19bf6ddc2928597ccb9d113e5dd5 --- elasticsearch/.gitignore | 44 + elasticsearch/README.asciidoc | 3 + elasticsearch/build.gradle | 91 ++ elasticsearch/gradle.properties | 1 + .../licenses/super-csv-2.4.0.jar.sha1 | 1 + elasticsearch/licenses/super-csv-LICENSE | 203 +++ elasticsearch/licenses/super-csv-NOTICE | 0 .../xpack/prelert/PrelertPlugin.java | 264 ++++ .../xpack/prelert/action/DeleteJobAction.java | 168 +++ .../action/DeleteModelSnapshotAction.java | 203 +++ .../xpack/prelert/action/GetBucketAction.java | 414 ++++++ .../action/GetCategoryDefinitionAction.java | 209 +++ .../prelert/action/GetInfluencersAction.java | 329 +++++ .../xpack/prelert/action/GetJobAction.java | 376 ++++++ .../xpack/prelert/action/GetJobsAction.java | 228 ++++ .../xpack/prelert/action/GetListAction.java | 264 ++++ .../action/GetModelSnapshotsAction.java | 349 +++++ .../prelert/action/GetRecordsAction.java | 373 ++++++ .../xpack/prelert/action/PauseJobAction.java | 169 +++ .../xpack/prelert/action/PostDataAction.java | 253 ++++ .../prelert/action/PostDataCloseAction.java | 149 +++ .../prelert/action/PostDataFlushAction.java | 255 ++++ .../xpack/prelert/action/PutJobAction.java | 232 ++++ .../xpack/prelert/action/PutListAction.java | 206 +++ .../PutModelSnapshotDescriptionAction.java | 302 +++++ .../xpack/prelert/action/ResumeJobAction.java | 169 +++ .../action/RevertModelSnapshotAction.java | 413 ++++++ .../action/StartJobSchedulerAction.java | 212 +++ .../action/StopJobSchedulerAction.java | 164 +++ .../UpdateJobSchedulerStatusAction.java | 191 +++ .../prelert/action/UpdateJobStatusAction.java | 183 +++ .../action/ValidateDetectorAction.java | 165 +++ .../action/ValidateTransformAction.java | 165 +++ .../action/ValidateTransformsAction.java | 174 +++ .../xpack/prelert/job/AnalysisConfig.java | 714 ++++++++++ .../xpack/prelert/job/AnalysisLimits.java | 131 ++ .../xpack/prelert/job/CategorizerState.java | 24 + .../xpack/prelert/job/DataCounts.java | 413 ++++++ .../xpack/prelert/job/DataDescription.java | 358 +++++ .../xpack/prelert/job/Detector.java | 780 +++++++++++ .../xpack/prelert/job/IgnoreDowntime.java | 51 + .../elasticsearch/xpack/prelert/job/Job.java | 884 ++++++++++++ .../xpack/prelert/job/JobSchedulerStatus.java | 35 + .../xpack/prelert/job/JobStatus.java | 48 + .../xpack/prelert/job/ModelDebugConfig.java | 166 +++ .../xpack/prelert/job/ModelSizeStats.java | 334 +++++ .../xpack/prelert/job/ModelSnapshot.java | 297 ++++ .../xpack/prelert/job/ModelState.java | 30 + .../xpack/prelert/job/SchedulerConfig.java | 962 +++++++++++++ .../xpack/prelert/job/SchedulerState.java | 116 ++ .../prelert/job/audit/AuditActivity.java | 182 +++ .../xpack/prelert/job/audit/AuditMessage.java | 199 +++ .../xpack/prelert/job/audit/Auditor.java | 15 + .../xpack/prelert/job/audit/Level.java | 51 + .../prelert/job/condition/Condition.java | 132 ++ .../xpack/prelert/job/condition/Operator.java | 115 ++ .../config/DefaultDetectorDescription.java | 83 ++ .../prelert/job/config/DefaultFrequency.java | 62 + .../xpack/prelert/job/data/DataProcessor.java | 53 + .../xpack/prelert/job/data/DataStreamer.java | 66 + .../prelert/job/data/DataStreamerThread.java | 87 ++ .../job/detectionrules/Connective.java | 52 + .../job/detectionrules/DetectionRule.java | 169 +++ .../job/detectionrules/RuleAction.java | 24 + .../job/detectionrules/RuleCondition.java | 248 ++++ .../job/detectionrules/RuleConditionType.java | 55 + .../prelert/job/extraction/DataExtractor.java | 47 + .../job/extraction/DataExtractorFactory.java | 13 + .../prelert/job/logging/CppLogMessage.java | 232 ++++ .../job/logging/CppLogMessageHandler.java | 172 +++ .../xpack/prelert/job/logs/JobLogs.java | 122 ++ .../job/manager/AutodetectProcessManager.java | 207 +++ .../xpack/prelert/job/manager/JobManager.java | 559 ++++++++ .../xpack/prelert/job/messages/Messages.java | 304 +++++ .../prelert/job/metadata/Allocation.java | 206 +++ .../prelert/job/metadata/JobAllocator.java | 114 ++ .../job/metadata/JobLifeCycleService.java | 174 +++ .../prelert/job/metadata/PrelertMetadata.java | 251 ++++ .../persistence/BatchedDocumentsIterator.java | 53 + .../job/persistence/BucketQueryBuilder.java | 112 ++ .../job/persistence/BucketsQueryBuilder.java | 210 +++ .../job/persistence/ElasticsearchAuditor.java | 105 ++ .../ElasticsearchBatchedBucketsIterator.java | 47 + ...ElasticsearchBatchedDocumentsIterator.java | 115 ++ ...asticsearchBatchedInfluencersIterator.java | 49 + ...searchBatchedModelDebugOutputIterator.java | 47 + ...icsearchBatchedModelSizeStatsIterator.java | 48 + ...ticsearchBatchedModelSnapshotIterator.java | 46 + .../persistence/ElasticsearchBulkDeleter.java | 255 ++++ .../ElasticsearchBulkDeleterFactory.java | 28 + .../ElasticsearchDotNotationReverser.java | 163 +++ .../ElasticsearchJobDataCountsPersister.java | 57 + .../ElasticsearchJobDetailsMapper.java | 100 ++ .../persistence/ElasticsearchJobProvider.java | 1192 +++++++++++++++++ .../persistence/ElasticsearchMappings.java | 892 ++++++++++++ .../persistence/ElasticsearchPersister.java | 558 ++++++++ .../job/persistence/ElasticsearchScripts.java | 125 ++ .../ElasticsearchUsagePersister.java | 86 ++ .../persistence/InfluencersQueryBuilder.java | 168 +++ .../persistence/JobDataCountsPersister.java | 23 + .../job/persistence/JobDataDeleter.java | 63 + .../persistence/JobDataDeleterFactory.java | 11 + .../prelert/job/persistence/JobProvider.java | 112 ++ .../job/persistence/JobProviderFactory.java | 19 + .../job/persistence/JobRenormaliser.java | 46 + .../job/persistence/JobResultsPersister.java | 77 ++ .../job/persistence/JobResultsProvider.java | 143 ++ .../job/persistence/OldDataRemover.java | 65 + .../prelert/job/persistence/QueryPage.java | 93 ++ .../job/persistence/RecordsQueryBuilder.java | 174 +++ .../job/persistence/ResultsFilterBuilder.java | 103 ++ .../job/persistence/UsagePersister.java | 17 + .../prelert/job/process/NativeController.java | 79 ++ .../prelert/job/process/ProcessCtrl.java | 304 +++++ .../prelert/job/process/ProcessPipes.java | 215 +++ .../process/autodetect/AutodetectBuilder.java | 187 +++ .../autodetect/AutodetectCommunicator.java | 159 +++ .../process/autodetect/AutodetectProcess.java | 99 ++ .../autodetect/AutodetectProcessFactory.java | 22 + .../BlackHoleAutodetectProcess.java | 122 ++ .../autodetect/NativeAutodetectProcess.java | 158 +++ .../NativeAutodetectProcessFactory.java | 128 ++ .../output/FlushAcknowledgement.java | 81 ++ .../parsing/AutoDetectResultProcessor.java | 197 +++ .../parsing/AutodetectResultsParser.java | 110 ++ .../output/parsing/FlushListener.java | 57 + .../output/parsing/StateReader.java | 85 ++ .../autodetect/params/DataLoadParams.java | 39 + .../params/InterimResultsParams.java | 142 ++ .../process/autodetect/params/TimeRange.java | 123 ++ .../writer/AbstractDataToProcessWriter.java | 486 +++++++ .../writer/AbstractJsonRecordReader.java | 112 ++ .../writer/AggregatedJsonRecordReader.java | 204 +++ .../writer/AnalysisLimitsWriter.java | 50 + .../writer/ControlMsgToProcessWriter.java | 167 +++ .../writer/CsvDataToProcessWriter.java | 159 +++ .../autodetect/writer/CsvRecordWriter.java | 47 + .../writer/DataToProcessWriter.java | 33 + .../writer/DataToProcessWriterFactory.java | 54 + .../autodetect/writer/FieldConfigWriter.java | 160 +++ .../writer/JsonDataToProcessWriter.java | 156 +++ .../autodetect/writer/JsonRecordReader.java | 25 + .../writer/LengthEncodedWriter.java | 96 ++ .../writer/ModelDebugConfigWriter.java | 52 + .../autodetect/writer/RecordWriter.java | 37 + .../writer/SimpleJsonRecordReader.java | 177 +++ .../writer/SingleLineDataToProcessWriter.java | 72 + .../autodetect/writer/WriterConstants.java | 17 + .../job/process/normalizer/Renormaliser.java | 35 + .../normalizer/RenormaliserFactory.java | 11 + .../normalizer/noop/NoOpRenormaliser.java | 37 + .../prelert/job/quantiles/Quantiles.java | 130 ++ .../prelert/job/results/AnomalyCause.java | 383 ++++++ .../prelert/job/results/AnomalyRecord.java | 578 ++++++++ .../prelert/job/results/AutodetectResult.java | 218 +++ .../xpack/prelert/job/results/Bucket.java | 439 ++++++ .../prelert/job/results/BucketInfluencer.java | 228 ++++ .../job/results/CategoryDefinition.java | 161 +++ .../xpack/prelert/job/results/Influence.java | 110 ++ .../xpack/prelert/job/results/Influencer.java | 244 ++++ .../prelert/job/results/ModelDebugOutput.java | 338 +++++ .../xpack/prelert/job/results/PageParams.java | 99 ++ .../prelert/job/results/PartitionScore.java | 141 ++ .../job/results/ReservedFieldNames.java | 179 +++ .../prelert/job/scheduler/ProblemTracker.java | 112 ++ .../prelert/job/scheduler/ScheduledJob.java | 198 +++ .../job/scheduler/ScheduledJobService.java | 250 ++++ .../http/ElasticsearchDataExtractor.java | 250 ++++ .../http/ElasticsearchQueryBuilder.java | 140 ++ .../http/ElasticsearchUrlBuilder.java | 82 ++ .../http/HttpDataExtractorFactory.java | 98 ++ .../job/scheduler/http/HttpRequester.java | 160 +++ .../job/scheduler/http/HttpResponse.java | 53 + .../job/scheduler/http/ScrollState.java | 166 +++ .../job/status/CountingInputStream.java | 75 ++ .../prelert/job/status/StatusReporter.java | 326 +++++ .../xpack/prelert/job/transform/IntRange.java | 105 ++ .../job/transform/TransformConfig.java | 191 +++ .../job/transform/TransformConfigs.java | 108 ++ .../prelert/job/transform/TransformType.java | 156 +++ .../verification/ArgumentVerifier.java | 16 + .../verification/RegexExtractVerifier.java | 31 + .../verification/RegexPatternVerifier.java | 26 + .../verification/TransformConfigVerifier.java | 151 +++ .../TransformConfigsVerifier.java | 141 ++ .../xpack/prelert/job/usage/Usage.java | 21 + .../prelert/job/usage/UsageReporter.java | 122 ++ .../xpack/prelert/lists/ListDocument.java | 91 ++ .../prelert/rest/data/RestPostDataAction.java | 47 + .../rest/data/RestPostDataCloseAction.java | 41 + .../rest/data/RestPostDataFlushAction.java | 62 + .../influencers/RestGetInfluencersAction.java | 63 + .../prelert/rest/job/RestDeleteJobAction.java | 37 + .../prelert/rest/job/RestGetJobAction.java | 51 + .../prelert/rest/job/RestGetJobsAction.java | 53 + .../prelert/rest/job/RestPauseJobAction.java | 38 + .../prelert/rest/job/RestPutJobsAction.java | 42 + .../prelert/rest/job/RestResumeJobAction.java | 38 + .../prelert/rest/list/RestGetListAction.java | 39 + .../prelert/rest/list/RestPutListAction.java | 43 + .../RestDeleteModelSnapshotAction.java | 43 + .../RestGetModelSnapshotsAction.java | 84 ++ ...RestPutModelSnapshotDescriptionAction.java | 54 + .../RestRevertModelSnapshotAction.java | 63 + .../rest/results/RestGetBucketAction.java | 91 ++ .../rest/results/RestGetCategoryAction.java | 56 + .../rest/results/RestGetRecordsAction.java | 57 + .../RestStartJobSchedulerAction.java | 75 ++ .../RestStopJobSchedulerAction.java | 41 + .../validate/RestValidateDetectorAction.java | 46 + .../validate/RestValidateTransformAction.java | 46 + .../RestValidateTransformsAction.java | 46 + .../xpack/prelert/transforms/Concat.java | 56 + .../prelert/transforms/DependencySorter.java | 173 +++ .../prelert/transforms/ExcludeFilter.java | 34 + .../transforms/ExcludeFilterNumeric.java | 85 ++ .../transforms/ExcludeFilterRegex.java | 49 + .../transforms/HighestRegisteredDomain.java | 47 + .../prelert/transforms/RegexExtract.java | 46 + .../xpack/prelert/transforms/RegexSplit.java | 53 + .../prelert/transforms/StringTransform.java | 49 + .../xpack/prelert/transforms/Transform.java | 103 ++ .../transforms/TransformException.java | 13 + .../prelert/transforms/TransformFactory.java | 123 ++ .../transforms/date/DateFormatTransform.java | 47 + .../transforms/date/DateTransform.java | 62 + .../transforms/date/DoubleDateTransform.java | 42 + .../date/ParseTimestampException.java | 16 + .../prelert/utils/CloseableIterator.java | 19 + .../xpack/prelert/utils/ExceptionsHelper.java | 47 + .../xpack/prelert/utils/NamedPipeHelper.java | 317 +++++ .../xpack/prelert/utils/PrelertStrings.java | 55 + .../xpack/prelert/utils/SingleDocument.java | 144 ++ .../DateTimeFormatterTimestampConverter.java | 115 ++ .../xpack/prelert/utils/time/TimeUtils.java | 47 + .../utils/time/TimestampConverter.java | 37 + .../plugin-metadata/plugin-security.policy | 4 + .../src/main/resources/log4j2.properties | 9 + .../job/messages/prelert_messages.properties | 226 ++++ .../action/CreateListActionRequestTests.java | 40 + .../prelert/action/DeleteJobRequestTests.java | 22 + .../action/GetBucketActionRequestTests.java | 71 + .../action/GetBucketActionResponseTests.java | 139 ++ .../GetCategoryDefinitionRequestTests.java | 32 + .../GetCategoryDefinitionResponseTests.java | 28 + .../GetInfluencersActionRequestTests.java | 51 + .../GetInfluencersActionResponseTests.java | 43 + .../action/GetJobActionRequestTests.java | 27 + .../action/GetJobActionResponseTests.java | 97 ++ .../action/GetJobsActionRequestTests.java | 38 + .../action/GetJobsActionResponseTests.java | 37 + .../action/GetListActionRequestTests.java | 24 + .../action/GetListActionResponseTests.java | 34 + .../GetModelSnapshotsActionRequestTests.java | 53 + .../GetModelSnapshotsActionResponseTests.java | 36 + .../action/GetRecordsActionRequestTests.java | 57 + .../action/GetRecordsActionResponseTests.java | 38 + .../prelert/action/PauseJobRequestTests.java | 22 + .../action/PostDataActionRequestTests.java | 28 + .../action/PostDataActionResponseTests.java | 28 + .../action/PostDataCloseRequestTests.java | 22 + .../action/PostDataFlushRequestTests.java | 33 + .../action/PutJobActionRequestTests.java | 35 + .../action/PutJobActionResponseTests.java | 30 + ...SnapshotDescriptionActionRequestTests.java | 31 + ...napshotDescriptionActionResponseTests.java | 26 + .../prelert/action/ResumeJobRequestTests.java | 22 + ...RevertModelSnapshotActionRequestTests.java | 43 + ...evertModelSnapshotActionResponseTests.java | 30 + .../xpack/prelert/action/ScheduledJobsIT.java | 212 +++ .../StartJobSchedulerActionRequestTests.java | 33 + .../StopJobSchedulerActionRequestTests.java | 23 + .../UpdateJobSchedulerStatusRequestTests.java | 23 + .../action/UpdateJobStatusRequestTests.java | 23 + .../ValidateDetectorActionRequestTests.java | 37 + .../ValidateTransformActionRequestTests.java | 34 + .../ValidateTransformsActionRequestTests.java | 42 + .../prelert/integration/PrelertJobIT.java | 280 ++++ .../integration/PrelertYamlTestSuiteIT.java | 36 + .../prelert/integration/ScheduledJobIT.java | 186 +++ .../prelert/job/AnalysisConfigTests.java | 804 +++++++++++ .../prelert/job/AnalysisLimitsTests.java | 78 ++ .../xpack/prelert/job/DataCountsTests.java | 169 +++ .../prelert/job/DataDescriptionTests.java | 250 ++++ .../xpack/prelert/job/DataFormatTests.java | 109 ++ .../xpack/prelert/job/DetectorTests.java | 643 +++++++++ .../prelert/job/IgnoreDowntimeTests.java | 51 + .../prelert/job/JobSchedulerStatusTests.java | 26 + .../xpack/prelert/job/JobStatusTests.java | 42 + .../xpack/prelert/job/JobTests.java | 647 +++++++++ .../xpack/prelert/job/MemoryStatusTests.java | 96 ++ .../prelert/job/ModelDebugConfigTests.java | 71 + .../prelert/job/ModelSizeStatsTests.java | 98 ++ .../xpack/prelert/job/ModelSnapshotTests.java | 215 +++ .../prelert/job/SchedulerConfigTests.java | 565 ++++++++ .../prelert/job/SchedulerStateTests.java | 75 ++ .../prelert/job/audit/AuditActivityTests.java | 79 ++ .../prelert/job/audit/AuditMessageTests.java | 98 ++ .../xpack/prelert/job/audit/LevelTests.java | 105 ++ .../prelert/job/condition/ConditionTests.java | 98 ++ .../prelert/job/condition/OperatorTests.java | 180 +++ .../DefaultDetectorDescriptionTests.java | 39 + .../job/config/DefaultFrequencyTests.java | 43 + .../prelert/job/data/DataStreamerTests.java | 89 ++ .../job/data/DataStreamerThreadTests.java | 79 ++ .../job/detectionrules/ConnectiveTests.java | 73 + .../detectionrules/DetectionRuleTests.java | 116 ++ .../job/detectionrules/RuleActionTests.java | 17 + .../detectionrules/RuleConditionTests.java | 223 +++ .../RuleConditionTypeTests.java | 106 ++ .../logging/CppLogMessageHandlerTests.java | 50 + .../job/logging/CppLogMessageTests.java | 53 + .../xpack/prelert/job/logs/JobLogsTests.java | 69 + .../AutodetectProcessManagerTests.java | 222 +++ .../prelert/job/manager/JobManagerTests.java | 199 +++ .../prelert/job/messages/MessagesTests.java | 54 + .../prelert/job/metadata/AllocationTests.java | 41 + .../job/metadata/JobAllocatorTests.java | 179 +++ .../metadata/JobLifeCycleServiceTests.java | 133 ++ .../job/metadata/PrelertMetadataTests.java | 102 ++ .../persistence/BucketQueryBuilderTests.java | 64 + .../persistence/BucketsQueryBuilderTests.java | 106 ++ .../ElasticsearchAuditorTests.java | 155 +++ ...icsearchBatchedDocumentsIteratorTests.java | 205 +++ ...ElasticsearchDotNotationReverserTests.java | 55 + .../ElasticsearchJobDetailsMapperTests.java | 127 ++ .../ElasticsearchJobProviderTests.java | 1044 +++++++++++++++ .../ElasticsearchMappingsTests.java | 217 +++ .../ElasticsearchPersisterTests.java | 171 +++ .../ElasticsearchScriptsTests.java | 128 ++ .../ElasticsearchUsagePersisterTests.java | 98 ++ .../InfluencersQueryBuilderTests.java | 87 ++ .../MockBatchedDocumentsIterator.java | 72 + .../job/persistence/MockClientBuilder.java | 408 ++++++ .../job/persistence/QueryPageTests.java | 30 + .../ResultsFilterBuilderTests.java | 155 +++ .../job/process/NativeControllerTests.java | 52 + .../prelert/job/process/ProcessCtrlTests.java | 150 +++ .../job/process/ProcessPipesTests.java | 90 ++ .../AutodetectCommunicatorTests.java | 140 ++ .../BlackHoleAutodetectProcessTests.java | 32 + .../NativeAutodetectProcessTests.java | 117 ++ .../output/FlushAcknowledgementTests.java | 30 + .../AutoDetectResultProcessorTests.java | 265 ++++ .../parsing/AutodetectResultsParserTests.java | 381 ++++++ .../output/parsing/FlushListenerTests.java | 56 + .../output/parsing/StateReaderTests.java | 56 + .../params/DataLoadParamsTests.java | 25 + .../params/InterimResultsParamsTests.java | 178 +++ .../autodetect/params/TimeRangeTests.java | 58 + .../AbstractDataToProcessWriterTests.java | 390 ++++++ .../AggregatedJsonRecordReaderTests.java | 250 ++++ .../writer/AnalysisLimitsWriterTests.java | 78 ++ .../ControlMsgToProcessWriterTests.java | 138 ++ .../writer/CsvDataToProcessWriterTests.java | 382 ++++++ .../autodetect/writer/CsvParserTests.java | 52 + .../writer/CsvRecordWriterTests.java | 82 ++ .../DataToProcessWriterFactoryTests.java | 55 + ...ataWithTransformsToProcessWriterTests.java | 142 ++ .../writer/FieldConfigWriterTests.java | 226 ++++ .../writer/JsonDataToProcessWriterTests.java | 373 ++++++ .../writer/LengthEncodedWriterTests.java | 239 ++++ .../writer/ModelDebugConfigWriterTests.java | 52 + .../writer/SimpleJsonRecordReaderTests.java | 256 ++++ .../SingleLineDataToProcessWriterTests.java | 182 +++ .../prelert/job/quantiles/QuantilesTests.java | 73 + .../job/results/AnomalyCauseTests.java | 96 ++ .../job/results/AutodetectResultTests.java | 85 ++ .../job/results/BucketInfluencerTests.java | 137 ++ .../prelert/job/results/BucketTests.java | 401 ++++++ .../job/results/CategoryDefinitionTests.java | 114 ++ .../prelert/job/results/InfluenceTests.java | 38 + .../prelert/job/results/InfluencerTests.java | 30 + .../job/results/ModelDebugOutputTests.java | 206 +++ .../prelert/job/results/PageParamsTests.java | 57 + .../job/results/PartitionScoreTests.java | 30 + .../job/scheduler/ProblemTrackerTests.java | 120 ++ .../scheduler/ScheduledJobServiceTests.java | 194 +++ .../job/scheduler/ScheduledJobTests.java | 169 +++ .../http/ElasticsearchDataExtractorTests.java | 755 +++++++++++ .../http/ElasticsearchQueryBuilderTests.java | 159 +++ .../http/ElasticsearchUrlBuilderTests.java | 61 + .../job/scheduler/http/HttpResponseTests.java | 40 + .../job/status/CountingInputStreamTests.java | 103 ++ .../job/status/DummyStatusReporter.java | 42 + .../job/status/StatusReporterTests.java | 237 ++++ .../job/transform/TransformConfigTests.java | 196 +++ .../job/transform/TransformConfigsTests.java | 81 ++ .../TransformSerialisationTests.java | 66 + .../job/transform/TransformTypeTests.java | 52 + .../prelert/job/usage/DummyUsageReporter.java | 66 + .../prelert/job/usage/UsageReporterTests.java | 66 + .../prelert/lists/ListDocumentTests.java | 49 + .../GetModelSnapshotsTests.java | 101 ++ .../PutModelSnapshotDescriptionTests.java | 29 + .../RestStartJobSchedulerActionTests.java | 49 + .../support/AbstractSerializingTestCase.java | 91 ++ .../support/AbstractStreamableTestCase.java | 89 ++ .../AbstractStreamableXContentTestCase.java | 86 ++ .../AbstractWireSerializingTestCase.java | 86 ++ .../xpack/prelert/transforms/ConcatTests.java | 97 ++ .../transforms/DependencySorterTests.java | 272 ++++ .../transforms/ExcludeFilterNumericTests.java | 115 ++ .../transforms/ExcludeFilterTests.java | 116 ++ .../HighestRegisteredDomainTests.java | 448 +++++++ .../prelert/transforms/RegexExtractTests.java | 40 + .../prelert/transforms/RegexSplitTests.java | 54 + .../transforms/StringTransformTests.java | 169 +++ .../transforms/TransformFactoryTests.java | 127 ++ .../transforms/TransformTestUtils.java | 83 ++ .../date/DateFormatTransformTests.java | 128 ++ .../date/DoubleDateTransformTests.java | 89 ++ .../NamedPipeHelperNoBootstrapTests.java | 311 +++++ .../prelert/utils/NamedPipeHelperTests.java | 78 ++ .../prelert/utils/PrelertStringsTests.java | 18 + .../prelert/utils/SingleDocumentTests.java | 35 + ...eTimeFormatterTimestampConverterTests.java | 106 ++ .../prelert/utils/time/TimeUtilsTests.java | 26 + .../api/xpack.prelert.close_data.json | 16 + .../api/xpack.prelert.delete_job.json | 17 + .../xpack.prelert.delete_model_snapshot.json | 22 + .../api/xpack.prelert.flush_data.json | 38 + .../api/xpack.prelert.get_bucket.json | 55 + .../api/xpack.prelert.get_category.json | 31 + .../api/xpack.prelert.get_influencers.json | 50 + .../api/xpack.prelert.get_job.json | 22 + .../api/xpack.prelert.get_jobs.json | 19 + .../api/xpack.prelert.get_list.json | 17 + .../xpack.prelert.get_model_snapshots.json | 47 + .../api/xpack.prelert.get_records.json | 54 + .../api/xpack.prelert.pause_job.json | 17 + .../api/xpack.prelert.post_data.json | 35 + .../api/xpack.prelert.put_job.json | 19 + .../api/xpack.prelert.put_list.json | 14 + ...relert.put_model_snapshot_description.json | 26 + .../api/xpack.prelert.resume_job.json | 17 + .../xpack.prelert.revert_model_snapshot.json | 39 + .../xpack.prelert.start_job_scheduler.json | 29 + .../api/xpack.prelert.stop_job_scheduler.json | 17 + .../api/xpack.prelert.validate_detector.json | 14 + .../api/xpack.prelert.validate_transform.json | 14 + .../xpack.prelert.validate_transforms.json | 14 + .../test/delete_model_snapshot.yaml | 138 ++ .../test/get_model_snapshots.yaml | 91 ++ .../rest-api-spec/test/job_get_stats.yaml | 75 ++ .../rest-api-spec/test/jobs_crud.yaml | 98 ++ .../test/jobs_get_result_buckets.yaml | 42 + .../test/jobs_get_result_categories.yaml | 45 + .../test/jobs_get_result_influencers.yaml | 42 + .../test/jobs_get_result_records.yaml | 52 + .../rest-api-spec/test/list_crud.yaml | 70 + .../rest-api-spec/test/post_data.yaml | 102 ++ .../test/put_model_snapshot_description.yaml | 108 ++ .../test/revert_model_snapshot.yaml | 218 +++ .../rest-api-spec/test/validate_detector.yaml | 21 + .../test/validate_transform.yaml | 63 + .../settings/all_comment_engine_api.yml | 0 .../resources/settings/empty_engine_api.yml | 0 .../resources/settings/invalid_engine_api.yml | 15 + .../resources/settings/valid_engine_api.yml | 47 + settings.gradle | 7 +- 461 files changed, 60365 insertions(+), 5 deletions(-) create mode 100644 elasticsearch/.gitignore create mode 100644 elasticsearch/README.asciidoc create mode 100644 elasticsearch/build.gradle create mode 100644 elasticsearch/gradle.properties create mode 100644 elasticsearch/licenses/super-csv-2.4.0.jar.sha1 create mode 100644 elasticsearch/licenses/super-csv-LICENSE create mode 100644 elasticsearch/licenses/super-csv-NOTICE create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/PrelertPlugin.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteModelSnapshotAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetBucketAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetInfluencersAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetListAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetRecordsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PauseJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataCloseAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataFlushAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutListAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ResumeJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisConfig.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisLimits.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataCounts.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataDescription.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Detector.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntime.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Job.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatus.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobStatus.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfig.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSizeStats.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSnapshot.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelState.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerConfig.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerState.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivity.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessage.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Auditor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Level.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Condition.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Operator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescription.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequency.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataProcessor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamer.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThread.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/Connective.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRule.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleCondition.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionType.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractorFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessage.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandler.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logs/JobLogs.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManager.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/JobManager.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/messages/Messages.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/Allocation.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleService.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BatchedDocumentsIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedBucketsIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedInfluencersIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelDebugOutputIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSizeStatsIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSnapshotIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleterFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverser.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDataCountsPersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapper.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProvider.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappings.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScripts.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchUsagePersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/InfluencersQueryBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataCountsPersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleterFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobRenormaliser.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsProvider.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/OldDataRemover.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/QueryPage.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/RecordsQueryBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ResultsFilterBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/UsagePersister.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/NativeController.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessCtrl.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessPipes.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectCommunicator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcess.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcessFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/BlackHoleAutodetectProcess.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcess.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcessFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/FlushAcknowledgement.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutoDetectResultProcessor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutodetectResultsParser.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/FlushListener.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/StateReader.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/DataLoadParams.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/InterimResultsParams.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/TimeRange.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractDataToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractJsonRecordReader.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AggregatedJsonRecordReader.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AnalysisLimitsWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ControlMsgToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvDataToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvRecordWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriterFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/FieldConfigWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonDataToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonRecordReader.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/LengthEncodedWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ModelDebugConfigWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/RecordWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SimpleJsonRecordReader.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SingleLineDataToProcessWriter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/WriterConstants.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/Renormaliser.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/RenormaliserFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/noop/NoOpRenormaliser.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyCause.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyRecord.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AutodetectResult.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Bucket.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/BucketInfluencer.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/CategoryDefinition.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influence.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influencer.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ModelDebugOutput.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PageParams.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PartitionScore.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ReservedFieldNames.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ProblemTracker.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJob.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobService.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchDataExtractor.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchQueryBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchUrlBuilder.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpDataExtractorFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpRequester.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpResponse.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ScrollState.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/CountingInputStream.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/StatusReporter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/IntRange.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfig.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigs.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformType.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/ArgumentVerifier.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexExtractVerifier.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexPatternVerifier.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigVerifier.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigsVerifier.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/usage/Usage.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/usage/UsageReporter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/lists/ListDocument.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataCloseAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataFlushAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/influencers/RestGetInfluencersAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestDeleteJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPauseJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPutJobsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestResumeJobAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestGetListAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestPutListAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestDeleteModelSnapshotAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestGetModelSnapshotsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestPutModelSnapshotDescriptionAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestRevertModelSnapshotAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetBucketAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetCategoryAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetRecordsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStartJobSchedulerAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStopJobSchedulerAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateDetectorAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformsAction.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Concat.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/DependencySorter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterNumeric.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterRegex.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/HighestRegisteredDomain.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexExtract.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexSplit.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/StringTransform.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Transform.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformException.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformFactory.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateFormatTransform.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateTransform.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DoubleDateTransform.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/ParseTimestampException.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/CloseableIterator.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/ExceptionsHelper.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelper.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/PrelertStrings.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/SingleDocument.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/DateTimeFormatterTimestampConverter.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimeUtils.java create mode 100644 elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimestampConverter.java create mode 100644 elasticsearch/src/main/plugin-metadata/plugin-security.policy create mode 100644 elasticsearch/src/main/resources/log4j2.properties create mode 100644 elasticsearch/src/main/resources/org/elasticsearch/xpack/prelert/job/messages/prelert_messages.properties create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/CreateListActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/DeleteJobRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PauseJobRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataCloseRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataFlushRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ResumeJobRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ScheduledJobsIT.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsActionRequestTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertJobIT.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertYamlTestSuiteIT.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/ScheduledJobIT.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisConfigTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisLimitsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataCountsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataDescriptionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataFormatTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DetectorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntimeTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatusTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobStatusTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/MemoryStatusTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfigTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSizeStatsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSnapshotTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerConfigTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerStateTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivityTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessageTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/LevelTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/ConditionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/OperatorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescriptionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequencyTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThreadTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/ConnectiveTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRuleTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleActionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTypeTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandlerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logs/JobLogsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManagerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/JobManagerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/messages/MessagesTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/AllocationTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocatorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleServiceTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadataTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIteratorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverserTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapperTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProviderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappingsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersisterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScriptsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchUsagePersisterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/InfluencersQueryBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/MockBatchedDocumentsIterator.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/MockClientBuilder.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/QueryPageTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ResultsFilterBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/NativeControllerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/ProcessCtrlTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/ProcessPipesTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectCommunicatorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/BlackHoleAutodetectProcessTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcessTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/FlushAcknowledgementTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutoDetectResultProcessorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutodetectResultsParserTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/FlushListenerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/StateReaderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/DataLoadParamsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/InterimResultsParamsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/TimeRangeTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractDataToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AggregatedJsonRecordReaderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AnalysisLimitsWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvDataToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvParserTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvRecordWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriterFactoryTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataWithTransformsToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/FieldConfigWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonDataToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/LengthEncodedWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ModelDebugConfigWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SimpleJsonRecordReaderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SingleLineDataToProcessWriterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/quantiles/QuantilesTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/AnomalyCauseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/AutodetectResultTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/BucketInfluencerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/BucketTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/CategoryDefinitionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/InfluenceTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/InfluencerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/ModelDebugOutputTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/PageParamsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/results/PartitionScoreTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/ProblemTrackerTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobServiceTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchDataExtractorTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchQueryBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchUrlBuilderTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpResponseTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/status/CountingInputStreamTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/status/DummyStatusReporter.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/status/StatusReporterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/transform/TransformSerialisationTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/transform/TransformTypeTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/usage/DummyUsageReporter.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/usage/UsageReporterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/lists/ListDocumentTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/modelsnapshots/GetModelSnapshotsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/modelsnapshots/PutModelSnapshotDescriptionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStartJobSchedulerActionTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractSerializingTestCase.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableTestCase.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractStreamableXContentTestCase.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/support/AbstractWireSerializingTestCase.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/ConcatTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/DependencySorterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterNumericTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/HighestRegisteredDomainTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/RegexExtractTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/RegexSplitTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/StringTransformTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/TransformFactoryTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/TransformTestUtils.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/date/DateFormatTransformTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/transforms/date/DoubleDateTransformTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelperNoBootstrapTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelperTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/PrelertStringsTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/SingleDocumentTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/time/DateTimeFormatterTimestampConverterTests.java create mode 100644 elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/utils/time/TimeUtilsTests.java create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.close_data.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.delete_job.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.delete_model_snapshot.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.flush_data.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_bucket.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_category.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_influencers.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_job.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_jobs.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_list.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_model_snapshots.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.get_records.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.pause_job.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.post_data.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.put_job.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.put_list.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.put_model_snapshot_description.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.resume_job.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.revert_model_snapshot.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.start_job_scheduler.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.stop_job_scheduler.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.validate_detector.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.validate_transform.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/api/xpack.prelert.validate_transforms.json create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/delete_model_snapshot.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/get_model_snapshots.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/job_get_stats.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/jobs_crud.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/jobs_get_result_buckets.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/jobs_get_result_categories.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/jobs_get_result_influencers.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/jobs_get_result_records.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/list_crud.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/post_data.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/put_model_snapshot_description.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/revert_model_snapshot.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/validate_detector.yaml create mode 100644 elasticsearch/src/test/resources/rest-api-spec/test/validate_transform.yaml create mode 100644 elasticsearch/src/test/resources/settings/all_comment_engine_api.yml create mode 100644 elasticsearch/src/test/resources/settings/empty_engine_api.yml create mode 100644 elasticsearch/src/test/resources/settings/invalid_engine_api.yml create mode 100644 elasticsearch/src/test/resources/settings/valid_engine_api.yml diff --git a/elasticsearch/.gitignore b/elasticsearch/.gitignore new file mode 100644 index 00000000000..12ec3667692 --- /dev/null +++ b/elasticsearch/.gitignore @@ -0,0 +1,44 @@ +# intellij files +.idea/ +*.iml +*.ipr +*.iws +build-idea/ + +# eclipse files +.project +.classpath +.settings +build-eclipse/ + +# netbeans files +nb-configuration.xml +nbactions.xml + +# gradle stuff +.gradle/ +build/ + +# gradle wrapper +/gradle/ +gradlew +gradlew.bat + +# maven stuff (to be removed when trunk becomes 4.x) +*-execution-hints.log +target/ +dependency-reduced-pom.xml + +# testing stuff +**/.local* +.vagrant/ + +# osx stuff +.DS_Store + +# needed in case docs build is run...maybe we can configure doc build to generate files under build? +html_docs + +# random old stuff that we should look at the necessity of... +/tmp/ +backwards/ \ No newline at end of file diff --git a/elasticsearch/README.asciidoc b/elasticsearch/README.asciidoc new file mode 100644 index 00000000000..4717b07f2a1 --- /dev/null +++ b/elasticsearch/README.asciidoc @@ -0,0 +1,3 @@ += Elasticsearch Prelert Plugin + +Behavioral Analytics for Elasticsearch diff --git a/elasticsearch/build.gradle b/elasticsearch/build.gradle new file mode 100644 index 00000000000..89292a16380 --- /dev/null +++ b/elasticsearch/build.gradle @@ -0,0 +1,91 @@ +import org.elasticsearch.gradle.precommit.PrecommitTasks + +import org.gradle.internal.os.OperatingSystem + +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +boolean isWindows = OperatingSystem.current().isWindows() +boolean isLinux = OperatingSystem.current().isLinux() +boolean isMacOsX = OperatingSystem.current().isMacOsX() + +project.ext.nasDirectory = isWindows ? "\\\\prelert-nas\\builds\\6.5.0\\" : + (isMacOsX ? "/Volumes/builds/6.5.0/" : "/export/builds/6.5.0/") +// norelease: replace with something else when we become part of x-plugins +project.ext.nasExtension = '_' + (System.getenv()['GIT_COMMIT'] ?: 'xxxxxxxxxxxxxx').substring(0, 14) + + (isWindows ? "_windows-x86_64.zip" : (isMacOsX ? "_darwin-x86_64.zip" : + (isLinux ? "_linux-x86_64.zip" : "_sunos-x86_64.zip"))) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'prelert' + description 'Prelert Plugin' + classname 'org.elasticsearch.xpack.prelert.PrelertPlugin' +} + +version = "${elasticsearchVersion}" + +// We need to enable this at some point +thirdPartyAudit.enabled = false + +dependencies { + compile group: 'net.sf.supercsv', name: 'super-csv', version:"${supercsvVersion}" + testCompile group: 'org.ini4j', name: 'ini4j', version:'0.5.2' +} + +test { + exclude '**/*NoBootstrapTests.class' +} + +task noBootstrapTest(type: Test, + dependsOn: test.dependsOn) { + classpath = project.test.classpath + testClassesDir = project.test.testClassesDir + include '**/*NoBootstrapTests.class' +} +check.dependsOn noBootstrapTest +noBootstrapTest.mustRunAfter test + +integTest { + cluster { + //setting 'useNativeProcess', 'true' + distribution = 'zip' + } +} + +integTest.mustRunAfter noBootstrapTest + +bundlePlugin { + from("${rootDir}/cppdistribution") { + into '.' + // Don't copy Windows import libraries + exclude "**/*.lib" + // Don't copy the test support library + exclude "**/libPreTest.*" + includeEmptyDirs = false + } +} + +// norelease: this won't be needed when the pluginAll task below is removed +class SimpleCopy extends DefaultTask { + String sourceFile; + String destFile; + + @TaskAction + def copy() { + Files.copy(Paths.get(sourceFile), Paths.get(destFile), StandardCopyOption.REPLACE_EXISTING) + } +} + +// norelease: by the time we move to x-plugins we cannot use the Prelert NAS at all +task pluginAll(type: SimpleCopy) { + // This doesn't use a Copy task because that builds hashes for a huge number of files on the NAS + String zipFile = "${esplugin.name}-${elasticsearchVersion}.zip" + sourceFile = "${projectDir}/build/distributions/" + zipFile + destFile = project.ext.nasDirectory + zipFile.replace('.zip', project.ext.nasExtension) +} + +pluginAll.dependsOn check + diff --git a/elasticsearch/gradle.properties b/elasticsearch/gradle.properties new file mode 100644 index 00000000000..f5e88724132 --- /dev/null +++ b/elasticsearch/gradle.properties @@ -0,0 +1 @@ +supercsvVersion=2.4.0 diff --git a/elasticsearch/licenses/super-csv-2.4.0.jar.sha1 b/elasticsearch/licenses/super-csv-2.4.0.jar.sha1 new file mode 100644 index 00000000000..a0b40213309 --- /dev/null +++ b/elasticsearch/licenses/super-csv-2.4.0.jar.sha1 @@ -0,0 +1 @@ +017f8708c929029dde48bc298deaf3c7ae2452d3 \ No newline at end of file diff --git a/elasticsearch/licenses/super-csv-LICENSE b/elasticsearch/licenses/super-csv-LICENSE new file mode 100644 index 00000000000..9e0ad072b25 --- /dev/null +++ b/elasticsearch/licenses/super-csv-LICENSE @@ -0,0 +1,203 @@ +/* + * Apache License + * Version 2.0, January 2004 + * http://www.apache.org/licenses/ + * + * TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + * + * 1. Definitions. + * + * "License" shall mean the terms and conditions for use, reproduction, + * and distribution as defined by Sections 1 through 9 of this document. + * + * "Licensor" shall mean the copyright owner or entity authorized by + * the copyright owner that is granting the License. + * + * "Legal Entity" shall mean the union of the acting entity and all + * other entities that control, are controlled by, or are under common + * control with that entity. For the purposes of this definition, + * "control" means (i) the power, direct or indirect, to cause the + * direction or management of such entity, whether by contract or + * otherwise, or (ii) ownership of fifty percent (50%) or more of the + * outstanding shares, or (iii) beneficial ownership of such entity. + * + * "You" (or "Your") shall mean an individual or Legal Entity + * exercising permissions granted by this License. + * + * "Source" form shall mean the preferred form for making modifications, + * including but not limited to software source code, documentation + * source, and configuration files. + * + * "Object" form shall mean any form resulting from mechanical + * transformation or translation of a Source form, including but + * not limited to compiled object code, generated documentation, + * and conversions to other media types. + * + * "Work" shall mean the work of authorship, whether in Source or + * Object form, made available under the License, as indicated by a + * copyright notice that is included in or attached to the work + * (an example is provided in the Appendix below). + * + * "Derivative Works" shall mean any work, whether in Source or Object + * form, that is based on (or derived from) the Work and for which the + * editorial revisions, annotations, elaborations, or other modifications + * represent, as a whole, an original work of authorship. For the purposes + * of this License, Derivative Works shall not include works that remain + * separable from, or merely link (or bind by name) to the interfaces of, + * the Work and Derivative Works thereof. + * + * "Contribution" shall mean any work of authorship, including + * the original version of the Work and any modifications or additions + * to that Work or Derivative Works thereof, that is intentionally + * submitted to Licensor for inclusion in the Work by the copyright owner + * or by an individual or Legal Entity authorized to submit on behalf of + * the copyright owner. For the purposes of this definition, "submitted" + * means any form of electronic, verbal, or written communication sent + * to the Licensor or its representatives, including but not limited to + * communication on electronic mailing lists, source code control systems, + * and issue tracking systems that are managed by, or on behalf of, the + * Licensor for the purpose of discussing and improving the Work, but + * excluding communication that is conspicuously marked or otherwise + * designated in writing by the copyright owner as "Not a Contribution." + * + * "Contributor" shall mean Licensor and any individual or Legal Entity + * on behalf of whom a Contribution has been received by Licensor and + * subsequently incorporated within the Work. + * + * 2. Grant of Copyright License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * copyright license to reproduce, prepare Derivative Works of, + * publicly display, publicly perform, sublicense, and distribute the + * Work and such Derivative Works in Source or Object form. + * + * 3. Grant of Patent License. Subject to the terms and conditions of + * this License, each Contributor hereby grants to You a perpetual, + * worldwide, non-exclusive, no-charge, royalty-free, irrevocable + * (except as stated in this section) patent license to make, have made, + * use, offer to sell, sell, import, and otherwise transfer the Work, + * where such license applies only to those patent claims licensable + * by such Contributor that are necessarily infringed by their + * Contribution(s) alone or by combination of their Contribution(s) + * with the Work to which such Contribution(s) was submitted. If You + * institute patent litigation against any entity (including a + * cross-claim or counterclaim in a lawsuit) alleging that the Work + * or a Contribution incorporated within the Work constitutes direct + * or contributory patent infringement, then any patent licenses + * granted to You under this License for that Work shall terminate + * as of the date such litigation is filed. + * + * 4. Redistribution. You may reproduce and distribute copies of the + * Work or Derivative Works thereof in any medium, with or without + * modifications, and in Source or Object form, provided that You + * meet the following conditions: + * + * (a) You must give any other recipients of the Work or + * Derivative Works a copy of this License; and + * + * (b) You must cause any modified files to carry prominent notices + * stating that You changed the files; and + * + * (c) You must retain, in the Source form of any Derivative Works + * that You distribute, all copyright, patent, trademark, and + * attribution notices from the Source form of the Work, + * excluding those notices that do not pertain to any part of + * the Derivative Works; and + * + * (d) If the Work includes a "NOTICE" text file as part of its + * distribution, then any Derivative Works that You distribute must + * include a readable copy of the attribution notices contained + * within such NOTICE file, excluding those notices that do not + * pertain to any part of the Derivative Works, in at least one + * of the following places: within a NOTICE text file distributed + * as part of the Derivative Works; within the Source form or + * documentation, if provided along with the Derivative Works; or, + * within a display generated by the Derivative Works, if and + * wherever such third-party notices normally appear. The contents + * of the NOTICE file are for informational purposes only and + * do not modify the License. You may add Your own attribution + * notices within Derivative Works that You distribute, alongside + * or as an addendum to the NOTICE text from the Work, provided + * that such additional attribution notices cannot be construed + * as modifying the License. + * + * You may add Your own copyright statement to Your modifications and + * may provide additional or different license terms and conditions + * for use, reproduction, or distribution of Your modifications, or + * for any such Derivative Works as a whole, provided Your use, + * reproduction, and distribution of the Work otherwise complies with + * the conditions stated in this License. + * + * 5. Submission of Contributions. Unless You explicitly state otherwise, + * any Contribution intentionally submitted for inclusion in the Work + * by You to the Licensor shall be under the terms and conditions of + * this License, without any additional terms or conditions. + * Notwithstanding the above, nothing herein shall supersede or modify + * the terms of any separate license agreement you may have executed + * with Licensor regarding such Contributions. + * + * 6. Trademarks. This License does not grant permission to use the trade + * names, trademarks, service marks, or product names of the Licensor, + * except as required for reasonable and customary use in describing the + * origin of the Work and reproducing the content of the NOTICE file. + * + * 7. Disclaimer of Warranty. Unless required by applicable law or + * agreed to in writing, Licensor provides the Work (and each + * Contributor provides its Contributions) on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied, including, without limitation, any warranties or conditions + * of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + * PARTICULAR PURPOSE. You are solely responsible for determining the + * appropriateness of using or redistributing the Work and assume any + * risks associated with Your exercise of permissions under this License. + * + * 8. Limitation of Liability. In no event and under no legal theory, + * whether in tort (including negligence), contract, or otherwise, + * unless required by applicable law (such as deliberate and grossly + * negligent acts) or agreed to in writing, shall any Contributor be + * liable to You for damages, including any direct, indirect, special, + * incidental, or consequential damages of any character arising as a + * result of this License or out of the use or inability to use the + * Work (including but not limited to damages for loss of goodwill, + * work stoppage, computer failure or malfunction, or any and all + * other commercial damages or losses), even if such Contributor + * has been advised of the possibility of such damages. + * + * 9. Accepting Warranty or Additional Liability. While redistributing + * the Work or Derivative Works thereof, You may choose to offer, + * and charge a fee for, acceptance of support, warranty, indemnity, + * or other liability obligations and/or rights consistent with this + * License. However, in accepting such obligations, You may act only + * on Your own behalf and on Your sole responsibility, not on behalf + * of any other Contributor, and only if You agree to indemnify, + * defend, and hold each Contributor harmless for any liability + * incurred by, or claims asserted against, such Contributor by reason + * of your accepting any such warranty or additional liability. + * + * END OF TERMS AND CONDITIONS + * + * APPENDIX: How to apply the Apache License to your work. + * + * To apply the Apache License to your work, attach the following + * boilerplate notice, with the fields enclosed by brackets "[]" + * replaced with your own identifying information. (Don't include + * the brackets!) The text should be enclosed in the appropriate + * comment syntax for the file format. We also recommend that a + * file or class name and description of purpose be included on the + * same "printed page" as the copyright notice for easier + * identification within third-party archives. + * + * Copyright 2007 Kasper B. Graversen + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/elasticsearch/licenses/super-csv-NOTICE b/elasticsearch/licenses/super-csv-NOTICE new file mode 100644 index 00000000000..e69de29bb2d diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/PrelertPlugin.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/PrelertPlugin.java new file mode 100644 index 00000000000..0d477f1b3cc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/PrelertPlugin.java @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchRequestParsers; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.prelert.action.PutListAction; +import org.elasticsearch.xpack.prelert.action.DeleteJobAction; +import org.elasticsearch.xpack.prelert.action.DeleteModelSnapshotAction; +import org.elasticsearch.xpack.prelert.action.GetBucketAction; +import org.elasticsearch.xpack.prelert.action.GetCategoryDefinitionAction; +import org.elasticsearch.xpack.prelert.action.GetInfluencersAction; +import org.elasticsearch.xpack.prelert.action.GetJobAction; +import org.elasticsearch.xpack.prelert.action.GetJobsAction; +import org.elasticsearch.xpack.prelert.action.GetListAction; +import org.elasticsearch.xpack.prelert.action.GetModelSnapshotsAction; +import org.elasticsearch.xpack.prelert.action.GetRecordsAction; +import org.elasticsearch.xpack.prelert.action.PauseJobAction; +import org.elasticsearch.xpack.prelert.action.PostDataAction; +import org.elasticsearch.xpack.prelert.action.PostDataCloseAction; +import org.elasticsearch.xpack.prelert.action.PostDataFlushAction; +import org.elasticsearch.xpack.prelert.action.PutJobAction; +import org.elasticsearch.xpack.prelert.action.PutModelSnapshotDescriptionAction; +import org.elasticsearch.xpack.prelert.action.ResumeJobAction; +import org.elasticsearch.xpack.prelert.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.prelert.action.StartJobSchedulerAction; +import org.elasticsearch.xpack.prelert.action.StopJobSchedulerAction; +import org.elasticsearch.xpack.prelert.action.UpdateJobSchedulerStatusAction; +import org.elasticsearch.xpack.prelert.action.UpdateJobStatusAction; +import org.elasticsearch.xpack.prelert.action.ValidateDetectorAction; +import org.elasticsearch.xpack.prelert.action.ValidateTransformAction; +import org.elasticsearch.xpack.prelert.action.ValidateTransformsAction; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.logs.JobLogs; +import org.elasticsearch.xpack.prelert.job.manager.AutodetectProcessManager; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchBulkDeleterFactory; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutoDetectResultProcessor; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutodetectResultsParser; +import org.elasticsearch.xpack.prelert.job.process.normalizer.noop.NoOpRenormaliser; +import org.elasticsearch.xpack.prelert.job.scheduler.ScheduledJobService; +import org.elasticsearch.xpack.prelert.job.metadata.JobAllocator; +import org.elasticsearch.xpack.prelert.job.metadata.JobLifeCycleService; +import org.elasticsearch.xpack.prelert.job.metadata.PrelertMetadata; +import org.elasticsearch.xpack.prelert.job.process.NativeController; +import org.elasticsearch.xpack.prelert.job.process.ProcessCtrl; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcessFactory; +import org.elasticsearch.xpack.prelert.job.process.autodetect.BlackHoleAutodetectProcess; +import org.elasticsearch.xpack.prelert.job.process.autodetect.NativeAutodetectProcessFactory; +import org.elasticsearch.xpack.prelert.job.scheduler.http.HttpDataExtractorFactory; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.usage.UsageReporter; +import org.elasticsearch.xpack.prelert.rest.data.RestPostDataAction; +import org.elasticsearch.xpack.prelert.rest.data.RestPostDataCloseAction; +import org.elasticsearch.xpack.prelert.rest.data.RestPostDataFlushAction; +import org.elasticsearch.xpack.prelert.rest.influencers.RestGetInfluencersAction; +import org.elasticsearch.xpack.prelert.rest.job.RestDeleteJobAction; +import org.elasticsearch.xpack.prelert.rest.job.RestGetJobAction; +import org.elasticsearch.xpack.prelert.rest.job.RestGetJobsAction; +import org.elasticsearch.xpack.prelert.rest.job.RestPauseJobAction; +import org.elasticsearch.xpack.prelert.rest.job.RestPutJobsAction; +import org.elasticsearch.xpack.prelert.rest.job.RestResumeJobAction; +import org.elasticsearch.xpack.prelert.rest.list.RestPutListAction; +import org.elasticsearch.xpack.prelert.rest.list.RestGetListAction; +import org.elasticsearch.xpack.prelert.rest.modelsnapshots.RestDeleteModelSnapshotAction; +import org.elasticsearch.xpack.prelert.rest.modelsnapshots.RestGetModelSnapshotsAction; +import org.elasticsearch.xpack.prelert.rest.modelsnapshots.RestPutModelSnapshotDescriptionAction; +import org.elasticsearch.xpack.prelert.rest.modelsnapshots.RestRevertModelSnapshotAction; +import org.elasticsearch.xpack.prelert.rest.results.RestGetBucketAction; +import org.elasticsearch.xpack.prelert.rest.results.RestGetCategoryAction; +import org.elasticsearch.xpack.prelert.rest.results.RestGetRecordsAction; +import org.elasticsearch.xpack.prelert.rest.schedulers.RestStartJobSchedulerAction; +import org.elasticsearch.xpack.prelert.rest.schedulers.RestStopJobSchedulerAction; +import org.elasticsearch.xpack.prelert.rest.validate.RestValidateDetectorAction; +import org.elasticsearch.xpack.prelert.rest.validate.RestValidateTransformAction; +import org.elasticsearch.xpack.prelert.rest.validate.RestValidateTransformsAction; +import org.elasticsearch.xpack.prelert.utils.NamedPipeHelper; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class PrelertPlugin extends Plugin implements ActionPlugin { + public static final String NAME = "prelert"; + public static final String BASE_PATH = "/_xpack/prelert/"; + public static final String THREAD_POOL_NAME = NAME; + + // NORELEASE - temporary solution + static final Setting USE_NATIVE_PROCESS_OPTION = Setting.boolSetting("useNativeProcess", false, Property.NodeScope, + Property.Deprecated); + + private final Settings settings; + private final Environment env; + + private final ParseFieldMatcherSupplier parseFieldMatcherSupplier; + + static { + MetaData.registerPrototype(PrelertMetadata.TYPE, PrelertMetadata.PROTO); + } + + public PrelertPlugin(Settings settings) { + this.settings = settings; + this.env = new Environment(settings); + ParseFieldMatcher matcher = new ParseFieldMatcher(settings); + parseFieldMatcherSupplier = () -> matcher; + } + + @Override + public List> getSettings() { + return Collections.unmodifiableList( + Arrays.asList(USE_NATIVE_PROCESS_OPTION, + JobLogs.DONT_DELETE_LOGS_SETTING, + ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING, + ProcessCtrl.MAX_ANOMALY_RECORDS_SETTING, + StatusReporter.ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING, + StatusReporter.ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING, + UsageReporter.UPDATE_INTERVAL_SETTING)); + } + + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + SearchRequestParsers searchRequestParsers) { + + // All components get binded in the guice context to the instances returned here + // and interfaces are not bound to their concrete classes. + // instead of `bind(Interface.class).to(Implementation.class);` this happens: + // `bind(Implementation.class).toInstance(INSTANCE);` + // For this reason we can't use interfaces in the constructor of transport actions. + // This ok for now as we will remove Guice soon + ElasticsearchJobProvider jobProvider = new ElasticsearchJobProvider(client, 0, parseFieldMatcherSupplier.getParseFieldMatcher()); + + JobManager jobManager = new JobManager(env, settings, jobProvider, clusterService); + AutodetectProcessFactory processFactory; + if (USE_NATIVE_PROCESS_OPTION.get(settings)) { + try { + NativeController nativeController = new NativeController(env, new NamedPipeHelper()); + nativeController.tailLogsInThread(); + processFactory = new NativeAutodetectProcessFactory(jobProvider, env, settings, nativeController); + } catch (IOException e) { + throw new ElasticsearchException("Failed to create native process factory", e); + } + } else { + processFactory = (JobDetails, ignoreDowntime) -> new BlackHoleAutodetectProcess(); + } + AutodetectResultsParser autodetectResultsParser = new AutodetectResultsParser(settings, parseFieldMatcherSupplier); + DataProcessor dataProcessor = + new AutodetectProcessManager(settings, client, env, threadPool, jobManager, autodetectResultsParser, processFactory); + ScheduledJobService scheduledJobService = new ScheduledJobService(threadPool, client, jobProvider, dataProcessor, + new HttpDataExtractorFactory(), System::currentTimeMillis); + return Arrays.asList( + jobProvider, + jobManager, + new JobAllocator(settings, clusterService, threadPool), + new JobLifeCycleService(settings, client, clusterService, scheduledJobService, dataProcessor, threadPool.generic()), + new ElasticsearchBulkDeleterFactory(client), //NORELEASE: this should use Delete-by-query + dataProcessor + ); + } + + @Override + public List> getRestHandlers() { + return Arrays.asList( + RestGetJobAction.class, + RestGetJobsAction.class, + RestPutJobsAction.class, + RestDeleteJobAction.class, + RestPauseJobAction.class, + RestResumeJobAction.class, + RestGetListAction.class, + RestPutListAction.class, + RestGetInfluencersAction.class, + RestGetRecordsAction.class, + RestGetBucketAction.class, + RestPostDataAction.class, + RestPostDataCloseAction.class, + RestPostDataFlushAction.class, + RestValidateDetectorAction.class, + RestValidateTransformAction.class, + RestValidateTransformsAction.class, + RestGetCategoryAction.class, + RestGetModelSnapshotsAction.class, + RestRevertModelSnapshotAction.class, + RestPutModelSnapshotDescriptionAction.class, + RestStartJobSchedulerAction.class, + RestStopJobSchedulerAction.class, + RestDeleteModelSnapshotAction.class + ); + } + + @Override + public List> getActions() { + return Arrays.asList( + new ActionHandler<>(GetJobAction.INSTANCE, GetJobAction.TransportAction.class), + new ActionHandler<>(GetJobsAction.INSTANCE, GetJobsAction.TransportAction.class), + new ActionHandler<>(PutJobAction.INSTANCE, PutJobAction.TransportAction.class), + new ActionHandler<>(DeleteJobAction.INSTANCE, DeleteJobAction.TransportAction.class), + new ActionHandler<>(PauseJobAction.INSTANCE, PauseJobAction.TransportAction.class), + new ActionHandler<>(ResumeJobAction.INSTANCE, ResumeJobAction.TransportAction.class), + new ActionHandler<>(UpdateJobStatusAction.INSTANCE, UpdateJobStatusAction.TransportAction.class), + new ActionHandler<>(UpdateJobSchedulerStatusAction.INSTANCE, UpdateJobSchedulerStatusAction.TransportAction.class), + new ActionHandler<>(GetListAction.INSTANCE, GetListAction.TransportAction.class), + new ActionHandler<>(PutListAction.INSTANCE, PutListAction.TransportAction.class), + new ActionHandler<>(GetBucketAction.INSTANCE, GetBucketAction.TransportAction.class), + new ActionHandler<>(GetInfluencersAction.INSTANCE, GetInfluencersAction.TransportAction.class), + new ActionHandler<>(GetRecordsAction.INSTANCE, GetRecordsAction.TransportAction.class), + new ActionHandler<>(PostDataAction.INSTANCE, PostDataAction.TransportAction.class), + new ActionHandler<>(PostDataCloseAction.INSTANCE, PostDataCloseAction.TransportAction.class), + new ActionHandler<>(PostDataFlushAction.INSTANCE, PostDataFlushAction.TransportAction.class), + new ActionHandler<>(ValidateDetectorAction.INSTANCE, ValidateDetectorAction.TransportAction.class), + new ActionHandler<>(ValidateTransformAction.INSTANCE, ValidateTransformAction.TransportAction.class), + new ActionHandler<>(ValidateTransformsAction.INSTANCE, ValidateTransformsAction.TransportAction.class), + new ActionHandler<>(GetCategoryDefinitionAction.INSTANCE, GetCategoryDefinitionAction.TransportAction.class), + new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, GetModelSnapshotsAction.TransportAction.class), + new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, RevertModelSnapshotAction.TransportAction.class), + new ActionHandler<>(PutModelSnapshotDescriptionAction.INSTANCE, PutModelSnapshotDescriptionAction.TransportAction.class), + new ActionHandler<>(StartJobSchedulerAction.INSTANCE, StartJobSchedulerAction.TransportAction.class), + new ActionHandler<>(StopJobSchedulerAction.INSTANCE, StopJobSchedulerAction.TransportAction.class), + new ActionHandler<>(DeleteModelSnapshotAction.INSTANCE, DeleteModelSnapshotAction.TransportAction.class) + ); + } + + public static Path resolveConfigFile(Environment env, String name) { + return env.configFile().resolve(NAME).resolve(name); + } + + public static Path resolveLogFile(Environment env, String name) { + return env.logsFile().resolve(NAME).resolve(name); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + final FixedExecutorBuilder builder = new FixedExecutorBuilder(settings, THREAD_POOL_NAME, + 5 * EsExecutors.boundedNumberOfProcessors(settings), 1000, "xpack.prelert.thread_pool"); + return Collections.singletonList(builder); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteJobAction.java new file mode 100644 index 00000000000..4adf4e3f1f5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteJobAction.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteJobAction extends Action { + + public static final DeleteJobAction INSTANCE = new DeleteJobAction(); + public static final String NAME = "cluster:admin/prelert/job/delete"; + + private DeleteJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + DeleteJobAction.Request other = (DeleteJobAction.Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, DeleteJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, DeleteJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.deleteJob(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteModelSnapshotAction.java new file mode 100644 index 00000000000..a6cf503186e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/DeleteModelSnapshotAction.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchBulkDeleter; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchBulkDeleterFactory; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; + +public class DeleteModelSnapshotAction extends Action { + + public static final DeleteModelSnapshotAction INSTANCE = new DeleteModelSnapshotAction(); + public static final String NAME = "cluster:admin/prelert/modelsnapshots/delete"; + + private DeleteModelSnapshotAction() { + super(NAME); + } + + @Override + public DeleteModelSnapshotAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public DeleteModelSnapshotAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + private String jobId; + private String snapshotId; + + private Request() { + } + + public Request(String jobId, String snapshotId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, "jobId"); + this.snapshotId = ExceptionsHelper.requireNonNull(snapshotId, "snapshotId"); + } + + public String getJobId() { + return jobId; + } + + public String getSnapshotId() { + return snapshotId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeString(snapshotId); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, DeleteModelSnapshotAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + private final JobManager jobManager; + private final ClusterService clusterService; + private final ElasticsearchBulkDeleterFactory bulkDeleterFactory; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ElasticsearchJobProvider jobProvider, JobManager jobManager, ClusterService clusterService, + ElasticsearchBulkDeleterFactory bulkDeleterFactory) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + this.jobManager = jobManager; + this.clusterService = clusterService; + this.bulkDeleterFactory = bulkDeleterFactory; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + + // Verify the snapshot exists + List deleteCandidates; + deleteCandidates = jobProvider.modelSnapshots( + request.getJobId(), 0, 1, null, null, null, true, request.getSnapshotId(), null + ).hits(); + + if (deleteCandidates.size() > 1) { + logger.warn("More than one model found for [jobId: " + request.getJobId() + + ", snapshotId: " + request.getSnapshotId() + "] tuple."); + } + + if (deleteCandidates.isEmpty()) { + throw new ResourceNotFoundException(Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId())); + } + ModelSnapshot deleteCandidate = deleteCandidates.get(0); + + // Verify the snapshot is not being used + // + // NORELEASE: technically, this could be stale and refuse a delete, but I think that's acceptable + // since it is non-destructive + Optional job = jobManager.getJob(request.getJobId(), clusterService.state()); + if (job.isPresent()) { + String currentModelInUse = job.get().getModelSnapshotId(); + if (currentModelInUse != null && currentModelInUse.equals(request.getSnapshotId())) { + throw new IllegalArgumentException(Messages.getMessage(Messages.REST_CANNOT_DELETE_HIGHEST_PRIORITY, + request.getSnapshotId(), request.getJobId())); + } + } + + // Delete the snapshot and any associated state files + ElasticsearchBulkDeleter deleter = bulkDeleterFactory.apply(request.getJobId()); + deleter.deleteModelSnapshot(deleteCandidate); + deleter.commit(new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + // We don't care about the bulk response, just that it succeeded + listener.onResponse(new DeleteModelSnapshotAction.Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + jobManager.audit(request.getJobId()).info(Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, + deleteCandidate.getDescription())); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetBucketAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetBucketAction.java new file mode 100644 index 00000000000..d64b7a00a1c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetBucketAction.java @@ -0,0 +1,414 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.persistence.BucketQueryBuilder; +import org.elasticsearch.xpack.prelert.job.persistence.BucketsQueryBuilder; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class GetBucketAction extends Action { + + public static final GetBucketAction INSTANCE = new GetBucketAction(); + public static final String NAME = "indices:admin/prelert/results/bucket/get"; + + private GetBucketAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField EXPAND = new ParseField("expand"); + public static final ParseField INCLUDE_INTERIM = new ParseField("includeInterim"); + public static final ParseField PARTITION_VALUE = new ParseField("partitionValue"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("maxNormalizedProbability"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setTimestamp, Bucket.TIMESTAMP); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + PARSER.declareBoolean(Request::setExpand, EXPAND); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareString(Request::setStart, START); + PARSER.declareString(Request::setEnd, END); + PARSER.declareBoolean(Request::setExpand, EXPAND); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Request::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + } + + public static Request parseRequest(String jobId, XContentParser parser, + ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String timestamp; + private boolean expand = false; + private boolean includeInterim = false; + private String partitionValue; + private String start; + private String end; + private PageParams pageParams = null; + private double anomalyScore = 0.0; + private double maxNormalizedProbability = 0.0; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public void setTimestamp(String timestamp) { + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, Bucket.TIMESTAMP.getPreferredName()); + } + + public String getTimestamp() { + return timestamp; + } + + public boolean isExpand() { + return expand; + } + + public void setExpand(boolean expand) { + this.expand = expand; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public String getPartitionValue() { + return partitionValue; + } + + public void setPartitionValue(String partitionValue) { + this.partitionValue = ExceptionsHelper.requireNonNull(partitionValue, PARTITION_VALUE.getPreferredName()); + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = ExceptionsHelper.requireNonNull(start, START.getPreferredName()); + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = ExceptionsHelper.requireNonNull(end, END.getPreferredName()); + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = ExceptionsHelper.requireNonNull(pageParams, PageParams.PAGE.getPreferredName()); + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + this.maxNormalizedProbability = maxNormalizedProbability; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if ((timestamp == null || timestamp.isEmpty()) + && (start == null || start.isEmpty() || end == null || end.isEmpty())) { + validationException = addValidationError("Either [timestamp] or [start, end] parameters must be set.", validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + timestamp = in.readOptionalString(); + expand = in.readBoolean(); + includeInterim = in.readBoolean(); + partitionValue = in.readOptionalString(); + start = in.readOptionalString(); + end = in.readOptionalString(); + anomalyScore = in.readDouble(); + maxNormalizedProbability = in.readDouble(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(timestamp); + out.writeBoolean(expand); + out.writeBoolean(includeInterim); + out.writeOptionalString(partitionValue); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeDouble(anomalyScore); + out.writeDouble(maxNormalizedProbability); + out.writeOptionalWriteable(pageParams); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(Bucket.TIMESTAMP.getPreferredName(), timestamp); + } + builder.field(EXPAND.getPreferredName(), expand); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + if (partitionValue != null) { + builder.field(PARTITION_VALUE.getPreferredName(), partitionValue); + } + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (pageParams != null) { + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + } + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, partitionValue, expand, includeInterim, + anomalyScore, maxNormalizedProbability, pageParams, start, end); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(timestamp, other.timestamp) && + Objects.equals(partitionValue, other.partitionValue) && + Objects.equals(expand, other.expand) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(anomalyScore, other.anomalyScore) && + Objects.equals(maxNormalizedProbability, other.maxNormalizedProbability) && + Objects.equals(pageParams, other.pageParams) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements StatusToXContent { + + private QueryPage buckets; + + Response() { + } + + Response(QueryPage buckets) { + this.buckets = buckets; + } + + public QueryPage getBuckets() { + return buckets; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + buckets = new QueryPage<>(in, Bucket::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + buckets.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return buckets.doXContentBody(builder, params); + } + + @Override + public RestStatus status() { + return buckets.hitCount() == 0 ? RestStatus.NOT_FOUND : RestStatus.OK; + } + + @Override + public int hashCode() { + return Objects.hash(buckets); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(buckets, other.buckets); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + QueryPage results; + // Single bucket + if (request.timestamp != null) { + BucketQueryBuilder.BucketQuery query = + new BucketQueryBuilder(request.timestamp).expand(request.expand) + .includeInterim(request.includeInterim) + .partitionValue(request.partitionValue) + .build(); + + results = jobProvider.bucket(request.jobId, query); + } else { + // Multiple buckets + BucketsQueryBuilder.BucketsQuery query = + new BucketsQueryBuilder().expand(request.expand) + .includeInterim(request.includeInterim) + .epochStart(request.start) + .epochEnd(request.end) + .from(request.pageParams.getFrom()) + .size(request.pageParams.getSize()) + .anomalyScoreThreshold(request.anomalyScore) + .normalizedProbabilityThreshold(request.maxNormalizedProbability) + .partitionValue(request.partitionValue) + .build(); + + results = jobProvider.buckets(request.jobId, query); + } + listener.onResponse(new Response(results)); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionAction.java new file mode 100644 index 00000000000..25a1dd03bcf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionAction.java @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetCategoryDefinitionAction extends +Action { + + public static final GetCategoryDefinitionAction INSTANCE = new GetCategoryDefinitionAction(); + private static final String NAME = "cluster:admin/prelert/categorydefinition/get"; + + private GetCategoryDefinitionAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + public static final ParseField CATEGORY_ID = new ParseField("categoryId"); + public static final ParseField FROM = new ParseField("from"); + public static final ParseField SIZE = new ParseField("size"); + + private String jobId; + private String categoryId; + private PageParams pageParams = null; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() { + } + + public String getCategoryId() { + return categoryId; + } + + public void setCategoryId(String categoryId) { + this.categoryId = ExceptionsHelper.requireNonNull(categoryId, CATEGORY_ID.getPreferredName()); + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + categoryId = in.readOptionalString(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(categoryId); + out.writeOptionalWriteable(pageParams); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Request request = (Request) o; + return Objects.equals(jobId, request.jobId) + && Objects.equals(categoryId, request.categoryId) + && Objects.equals(pageParams, request.pageParams); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, categoryId, pageParams); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetCategoryDefinitionAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContent { + + private QueryPage result; + + public Response(QueryPage result) { + this.result = result; + } + + Response() { + } + + public QueryPage getResult() { + return result; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + result = new QueryPage<>(in, CategoryDefinition::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + result.doXContentBody(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Response response = (Response) o; + return Objects.equals(result, response.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + QueryPage result; + if (request.categoryId != null ) { + result = jobProvider.categoryDefinition(request.jobId, request.categoryId); + } else { + result = jobProvider.categoryDefinitions(request.jobId, request.pageParams.getFrom(), + request.pageParams.getSize()); + } + listener.onResponse(new Response(result)); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetInfluencersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetInfluencersAction.java new file mode 100644 index 00000000000..c7e01923165 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetInfluencersAction.java @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.InfluencersQueryBuilder; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import java.io.IOException; +import java.util.Objects; + +public class GetInfluencersAction +extends Action { + + public static final GetInfluencersAction INSTANCE = new GetInfluencersAction(); + public static final String NAME = "indices:admin/prelert/results/influencers/get"; + + private GetInfluencersAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField INCLUDE_INTERIM = new ParseField("includeInterim"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + public static final ParseField SORT_FIELD = new ParseField("sort"); + public static final ParseField DESCENDING_SORT = new ParseField("desc"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, start) -> request.start = start, START); + PARSER.declareString((request, end) -> request.end = end, END); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareString(Request::setSort, SORT_FIELD); + PARSER.declareBoolean(Request::setDecending, DESCENDING_SORT); + } + + public static Request parseRequest(String jobId, String start, String end, XContentParser parser, + ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + if (start != null) { + request.start = start; + } + if (end != null) { + request.end = end; + } + return request; + } + + private String jobId; + private String start; + private String end; + private boolean includeInterim = false; + private PageParams pageParams = new PageParams(0, 100); + private double anomalyScoreFilter = 0.0; + private String sort = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean decending = false; + + Request() { + } + + public Request(String jobId, String start, String end) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.start = ExceptionsHelper.requireNonNull(start, START.getPreferredName()); + this.end = ExceptionsHelper.requireNonNull(end, END.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getStart() { + return start; + } + + public String getEnd() { + return end; + } + + public boolean isDecending() { + return decending; + } + + public void setDecending(boolean decending) { + this.decending = decending; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + public PageParams getPageParams() { + return pageParams; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public void setAnomalyScore(double anomalyScoreFilter) { + this.anomalyScoreFilter = anomalyScoreFilter; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = ExceptionsHelper.requireNonNull(sort, SORT_FIELD.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + includeInterim = in.readBoolean(); + pageParams = new PageParams(in); + start = in.readString(); + end = in.readString(); + sort = in.readOptionalString(); + decending = in.readBoolean(); + anomalyScoreFilter = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(includeInterim); + pageParams.writeTo(out); + out.writeString(start); + out.writeString(end); + out.writeOptionalString(sort); + out.writeBoolean(decending); + out.writeDouble(anomalyScoreFilter); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + builder.field(START.getPreferredName(), start); + builder.field(END.getPreferredName(), end); + builder.field(SORT_FIELD.getPreferredName(), sort); + builder.field(DESCENDING_SORT.getPreferredName(), decending); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScoreFilter); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, includeInterim, pageParams, start, end, sort, decending, anomalyScoreFilter); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(start, other.start) && Objects.equals(end, other.end) + && Objects.equals(includeInterim, other.includeInterim) && Objects.equals(pageParams, other.pageParams) + && Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && Objects.equals(decending, other.decending) + && Objects.equals(sort, other.sort); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContent { + + private QueryPage influencers; + + Response() { + } + + Response(QueryPage influencers) { + this.influencers = influencers; + } + + public QueryPage getInfluencers() { + return influencers; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + influencers = new QueryPage<>(in, Influencer::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + influencers.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return influencers.doXContentBody(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(influencers); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(influencers, other.influencers); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder().includeInterim(request.includeInterim) + .epochStart(request.start).epochEnd(request.end).from(request.pageParams.getFrom()).size(request.pageParams.getSize()) + .anomalyScoreThreshold(request.anomalyScoreFilter).sortField(request.sort).sortDescending(request.decending).build(); + + QueryPage page = jobProvider.influencers(request.jobId, query); + listener.onResponse(new Response(page)); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobAction.java new file mode 100644 index 00000000000..52a35346087 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobAction.java @@ -0,0 +1,376 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.manager.AutodetectProcessManager; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +public class GetJobAction extends Action { + + public static final GetJobAction INSTANCE = new GetJobAction(); + public static final String NAME = "cluster:admin/prelert/job/get"; + + private GetJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String jobId; + private boolean config; + private boolean dataCounts; + private boolean modelSizeStats; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public Request all() { + this.config = true; + this.dataCounts = true; + this.modelSizeStats = true; + return this; + } + + public boolean config() { + return config; + } + + public Request config(boolean config) { + this.config = config; + return this; + } + + public boolean dataCounts() { + return dataCounts; + } + + public Request dataCounts(boolean dataCounts) { + this.dataCounts = dataCounts; + return this; + } + + public boolean modelSizeStats() { + return modelSizeStats; + } + + public Request modelSizeStats(boolean modelSizeStats) { + this.modelSizeStats = modelSizeStats; + return this; + } + + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + config = in.readBoolean(); + dataCounts = in.readBoolean(); + modelSizeStats = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(config); + out.writeBoolean(dataCounts); + out.writeBoolean(modelSizeStats); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, config, dataCounts, modelSizeStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && this.config == other.config + && this.dataCounts == other.dataCounts && this.modelSizeStats == other.modelSizeStats; + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements StatusToXContent { + + static class JobInfo implements ToXContent, Writeable { + @Nullable + private Job jobConfig; + @Nullable + private DataCounts dataCounts; + @Nullable + private ModelSizeStats modelSizeStats; + + JobInfo(@Nullable Job job, @Nullable DataCounts dataCounts, @Nullable ModelSizeStats modelSizeStats) { + this.jobConfig = job; + this.dataCounts = dataCounts; + this.modelSizeStats = modelSizeStats; + } + + JobInfo(StreamInput in) throws IOException { + jobConfig = in.readOptionalWriteable(Job::new); + dataCounts = in.readOptionalWriteable(DataCounts::new); + modelSizeStats = in.readOptionalWriteable(ModelSizeStats::new); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (jobConfig != null) { + builder.field("config", jobConfig); + } + if (dataCounts != null) { + builder.field("data_counts", dataCounts); + } + if (modelSizeStats != null) { + builder.field("model_size_stats", modelSizeStats); + } + builder.endObject(); + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalWriteable(jobConfig); + out.writeOptionalWriteable(dataCounts); + out.writeOptionalWriteable(modelSizeStats); + } + + @Override + public int hashCode() { + return Objects.hash(jobConfig, dataCounts, modelSizeStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + JobInfo other = (JobInfo) obj; + return Objects.equals(jobConfig, other.jobConfig) + && Objects.equals(this.dataCounts, other.dataCounts) + && Objects.equals(this.modelSizeStats, other.modelSizeStats); + } + } + + private SingleDocument jobResponse; + + public Response() { + jobResponse = SingleDocument.empty(Job.TYPE); + } + + public Response(JobInfo jobResponse) { + this.jobResponse = new SingleDocument<>(Job.TYPE, jobResponse); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobResponse = new SingleDocument<>(in, JobInfo::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + jobResponse.writeTo(out); + } + + @Override + public RestStatus status() { + return jobResponse.status(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return jobResponse.toXContent(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(jobResponse); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(jobResponse, other.jobResponse); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + + public static class TransportAction extends TransportMasterNodeReadAction { + + private final JobManager jobManager; + private final AutodetectProcessManager processManager; + private final ElasticsearchJobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager, AutodetectProcessManager processManager, ElasticsearchJobProvider jobProvider) { + super(settings, GetJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + this.processManager = processManager; + this.jobProvider = jobProvider; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Get job '{}', config={}, data_counts={}, model_size_stats={}", + request.getJobId(), request.config(), request.dataCounts(), request.modelSizeStats()); + + // always get the job regardless of the request.config param because if the job + // can't be found a different response is returned. + Optional optionalJob = jobManager.getJob(request.getJobId(), state); + if (optionalJob.isPresent() == false) { + logger.debug(String.format(Locale.ROOT, "Cannot find job '%s'", request.getJobId())); + listener.onResponse(new Response()); + return; + } + + logger.debug("Returning job '" + optionalJob.get().getJobId() + "'"); + + Job job = request.config() && optionalJob.isPresent() ? optionalJob.get() : null; + DataCounts dataCounts = readDataCounts(request); + ModelSizeStats modelSizeStats = readModelSizeStats(request); + listener.onResponse(new Response(new Response.JobInfo(job, dataCounts, modelSizeStats))); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + private DataCounts readDataCounts(Request request) { + if (request.dataCounts()) { + Optional counts = processManager.getDataCounts(request.getJobId()); + return counts.orElseGet(() -> jobProvider.dataCounts(request.getJobId())); + } + + return null; + } + + private ModelSizeStats readModelSizeStats(Request request) { + if (request.modelSizeStats()) { + Optional sizeStats = processManager.getModelSizeStats(request.getJobId()); + return sizeStats.orElseGet(() -> jobProvider.modelSizeStats(request.getJobId()).orElse(null)); + } + + return null; + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobsAction.java new file mode 100644 index 00000000000..7caaefb554b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetJobsAction.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import java.io.IOException; +import java.util.Objects; + +public class GetJobsAction extends Action { + + public static final GetJobsAction INSTANCE = new GetJobsAction(); + public static final String NAME = "cluster:admin/prelert/jobs/get"; + + private GetJobsAction() { + super(NAME); + } + + @Override + public GetJobsAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public GetJobsAction.Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest implements ToXContent { + + public static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + } + + private PageParams pageParams = new PageParams(0, 100); + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + pageParams = new PageParams(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + pageParams.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(pageParams); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(pageParams, other.pageParams); + } + } + + + public static class Response extends ActionResponse implements ToXContent { + + private QueryPage jobs; + + public Response(QueryPage jobs) { + this.jobs = jobs; + } + + public Response() {} + + public QueryPage getResponse() { + return jobs; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobs = new QueryPage<>(in, Job::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + jobs.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return jobs.doXContentBody(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(jobs); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(jobs, other.jobs); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetJobsAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeReadAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager) { + super(settings, GetJobsAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + QueryPage jobsPage = jobManager.getJobs(request.pageParams.getFrom(), request.pageParams.getSize(), state); + listener.onResponse(new Response(jobsPage)); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetListAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetListAction.java new file mode 100644 index 00000000000..df1887fd4a8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetListAction.java @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.TransportGetAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + + +public class GetListAction extends Action { + + public static final GetListAction INSTANCE = new GetListAction(); + public static final String NAME = "cluster:admin/prelert/list/get"; + + private GetListAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String listId; + + Request() { + } + + public Request(String listId) { + this.listId = listId; + } + + public String getListId() { + return listId; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (listId == null) { + validationException = addValidationError("List ID is required for GetList API.", validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + listId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(listId); + } + + @Override + public int hashCode() { + return Objects.hash(listId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(listId, other.listId); + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetListAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements StatusToXContent { + + private SingleDocument response; + + public Response(SingleDocument document) { + this.response = document; + } + + Response() { + } + + public SingleDocument getResponse() { + return response; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + response = new SingleDocument<>(in, ListDocument::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + response.writeTo(out); + } + + @Override + public RestStatus status() { + return response.status(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return response.toXContent(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(response); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(response, other.response); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class TransportAction extends TransportMasterNodeReadAction { + + private final TransportGetAction transportGetAction; + + // TODO these need to be moved to a settings object later + // See #20 + private static final String PRELERT_INFO_INDEX = "prelert-int"; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + TransportGetAction transportGetAction) { + super(settings, GetListAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.transportGetAction = transportGetAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + final String listId = request.getListId(); + GetRequest getRequest = new GetRequest(PRELERT_INFO_INDEX, ListDocument.TYPE.getPreferredName(), listId); + transportGetAction.execute(getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getDocResponse) { + + try { + SingleDocument responseBody; + if (getDocResponse.isExists()) { + BytesReference docSource = getDocResponse.getSourceAsBytesRef(); + XContentParser parser = XContentFactory.xContent(docSource).createParser(docSource); + ListDocument listDocument = ListDocument.PARSER.apply(parser, () -> parseFieldMatcher); + responseBody = new SingleDocument<>(ListDocument.TYPE.getPreferredName(), listDocument); + } else { + responseBody = SingleDocument.empty(ListDocument.TYPE.getPreferredName()); + } + Response listResponse = new Response(responseBody); + listener.onResponse(listResponse); + } catch (Exception e) { + this.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } + +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsAction.java new file mode 100644 index 00000000000..04e0e389776 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsAction.java @@ -0,0 +1,349 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class GetModelSnapshotsAction +extends Action { + + public static final GetModelSnapshotsAction INSTANCE = new GetModelSnapshotsAction(); + public static final String NAME = "cluster:admin/prelert/modelsnapshots/get"; + + private GetModelSnapshotsAction() { + super(NAME); + } + + @Override + public GetModelSnapshotsAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public GetModelSnapshotsAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField SORT = new ParseField("sort"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField DESC = new ParseField("desc"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setDescriptionString, DESCRIPTION); + PARSER.declareString(Request::setStart, START); + PARSER.declareString(Request::setEnd, END); + PARSER.declareString(Request::setSort, SORT); + PARSER.declareBoolean(Request::setDescOrder, DESC); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + } + + public static Request parseRequest(String jobId, XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String sort; + private String description; + private String start; + private String end; + private boolean desc; + private PageParams pageParams = new PageParams(0, 100); + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + @Nullable + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = sort; + } + + public boolean getDescOrder() { + return desc; + } + + public void setDescOrder(boolean desc) { + this.desc = desc; + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = ExceptionsHelper.requireNonNull(pageParams, PageParams.PAGE.getPreferredName()); + } + + @Nullable + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = ExceptionsHelper.requireNonNull(start, START.getPreferredName()); + } + + @Nullable + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = ExceptionsHelper.requireNonNull(end, END.getPreferredName()); + } + + @Nullable + public String getDescriptionString() { + return description; + } + + public void setDescriptionString(String description) { + this.description = ExceptionsHelper.requireNonNull(description, DESCRIPTION.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + sort = in.readOptionalString(); + description = in.readOptionalString(); + start = in.readOptionalString(); + end = in.readOptionalString(); + desc = in.readBoolean(); + pageParams = new PageParams(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(sort); + out.writeOptionalString(description); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeBoolean(desc); + pageParams.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (sort != null) { + builder.field(SORT.getPreferredName(), sort); + } + builder.field(DESC.getPreferredName(), desc); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, description, start, end, sort, desc); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(description, other.description) + && Objects.equals(start, other.start) && Objects.equals(end, other.end) && Objects.equals(sort, other.sort) + && Objects.equals(desc, other.desc); + } + } + + public static class Response extends ActionResponse implements ToXContent { + + private QueryPage page; + + public Response(QueryPage page) { + this.page = page; + } + + Response() { + } + + public QueryPage getPage() { + return page; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + page = new QueryPage<>(in, ModelSnapshot::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + page.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return page.doXContentBody(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(page); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(page, other.page); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetModelSnapshotsAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + logger.debug(String.format(Locale.ROOT, + "Get model snapshots for job %s. from = %d, size = %d" + + " start = '%s', end='%s', sort=%s descending=%b, description filter=%s", + request.getJobId(), request.pageParams.getFrom(), request.pageParams.getSize(), request.getStart(), request.getEnd(), + request.getSort(), request.getDescOrder(), request.getDescriptionString())); + + QueryPage page = doGetPage(jobProvider, request); + + logger.debug(String.format(Locale.ROOT, "Return %d model snapshots for job %s", page.hitCount(), request.getJobId())); + listener.onResponse(new Response(page)); + } + + public static QueryPage doGetPage(JobProvider jobProvider, Request request) { + QueryPage page = jobProvider.modelSnapshots(request.getJobId(), request.pageParams.getFrom(), + request.pageParams.getSize(), request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), null, + request.getDescriptionString()); + + // The quantiles can be large, and totally dominate the output - + // it's + // clearer to remove them + if (page.hits() != null) { + for (ModelSnapshot modelSnapshot : page.hits()) { + modelSnapshot.setQuantiles(null); + } + } + return page; + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetRecordsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetRecordsAction.java new file mode 100644 index 00000000000..a42b9786afa --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/GetRecordsAction.java @@ -0,0 +1,373 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.persistence.RecordsQueryBuilder; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetRecordsAction extends Action { + + public static final GetRecordsAction INSTANCE = new GetRecordsAction(); + public static final String NAME = "indices:admin/prelert/results/records/get"; + + private GetRecordsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField INCLUDE_INTERIM = new ParseField("includeInterim"); + public static final ParseField ANOMALY_SCORE_FILTER = new ParseField("anomalyScore"); + public static final ParseField SORT = new ParseField("sort"); + public static final ParseField DESCENDING = new ParseField("desc"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("normalizedProbability"); + public static final ParseField PARTITION_VALUE = new ParseField("partitionValue"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, start) -> request.start = start, START); + PARSER.declareString((request, end) -> request.end = end, END); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + PARSER.declareString(Request::setSort, SORT); + PARSER.declareBoolean(Request::setDecending, DESCENDING); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE_FILTER); + PARSER.declareDouble(Request::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + } + + public static Request parseRequest(String jobId, XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String start; + private String end; + private boolean includeInterim = false; + private PageParams pageParams = new PageParams(0, 100); + private double anomalyScoreFilter = 0.0; + private String sort = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean decending = false; + private double maxNormalizedProbability = 0.0; + private String partitionValue; + + Request() { + } + + public Request(String jobId, String start, String end) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.start = ExceptionsHelper.requireNonNull(start, START.getPreferredName()); + this.end = ExceptionsHelper.requireNonNull(end, END.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getStart() { + return start; + } + + public String getEnd() { + return end; + } + + public boolean isDecending() { + return decending; + } + + public void setDecending(boolean decending) { + this.decending = decending; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + public PageParams getPageParams() { + return pageParams; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public void setAnomalyScore(double anomalyScoreFilter) { + this.anomalyScoreFilter = anomalyScoreFilter; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = ExceptionsHelper.requireNonNull(sort, SORT.getPreferredName()); + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + this.maxNormalizedProbability = maxNormalizedProbability; + } + + public String getPartitionValue() { + return partitionValue; + } + + public void setPartitionValue(String partitionValue) { + this.partitionValue = ExceptionsHelper.requireNonNull(partitionValue, PARTITION_VALUE.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + includeInterim = in.readBoolean(); + pageParams = new PageParams(in); + start = in.readString(); + end = in.readString(); + sort = in.readOptionalString(); + decending = in.readBoolean(); + anomalyScoreFilter = in.readDouble(); + maxNormalizedProbability = in.readDouble(); + partitionValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(includeInterim); + pageParams.writeTo(out); + out.writeString(start); + out.writeString(end); + out.writeOptionalString(sort); + out.writeBoolean(decending); + out.writeDouble(anomalyScoreFilter); + out.writeDouble(maxNormalizedProbability); + out.writeOptionalString(partitionValue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + builder.field(SORT.getPreferredName(), sort); + builder.field(DESCENDING.getPreferredName(), decending); + builder.field(ANOMALY_SCORE_FILTER.getPreferredName(), anomalyScoreFilter); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + if (partitionValue != null) { + builder.field(PARTITION_VALUE.getPreferredName(), partitionValue); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, start, end, sort, decending, anomalyScoreFilter, includeInterim, maxNormalizedProbability, + pageParams, partitionValue); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(sort, other.sort) && + Objects.equals(decending, other.decending) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(maxNormalizedProbability, other.maxNormalizedProbability) && + Objects.equals(pageParams, other.pageParams) && + Objects.equals(partitionValue, other.partitionValue); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContent { + + private QueryPage records; + + Response() { + } + + Response(QueryPage records) { + this.records = records; + } + + public QueryPage getRecords() { + return records; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + records = new QueryPage<>(in, AnomalyRecord::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + records.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return records.doXContentBody(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(records); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(records, other.records); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + RecordsQueryBuilder.RecordsQuery query = new RecordsQueryBuilder() + .includeInterim(request.includeInterim) + .epochStart(request.start) + .epochEnd(request.end) + .from(request.pageParams.getFrom()) + .size(request.pageParams.getSize()) + .anomalyScoreThreshold(request.anomalyScoreFilter) + .sortField(request.sort) + .sortDescending(request.decending) + .build(); + + QueryPage page = jobProvider.records(request.jobId, query); + listener.onResponse(new Response(page)); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PauseJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PauseJobAction.java new file mode 100644 index 00000000000..74ebfa24304 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PauseJobAction.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class PauseJobAction extends Action { + + public static final PauseJobAction INSTANCE = new PauseJobAction(); + public static final String NAME = "cluster:admin/prelert/job/pause"; + + private PauseJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PauseJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, PauseJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.info("Pausing job " + request.getJobId()); + jobManager.pauseJob(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataAction.java new file mode 100644 index 00000000000..a854cc6176f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataAction.java @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.manager.AutodetectProcessManager; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.TimeRange; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class PostDataAction extends Action { + + public static final PostDataAction INSTANCE = new PostDataAction(); + public static final String NAME = "cluster:admin/prelert/data/post"; + + private PostDataAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return null; + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, PostDataAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements StatusToXContent { + + private DataCounts dataCounts; + + Response(String jobId) { + dataCounts = new DataCounts(jobId); + } + + public Response(DataCounts counts) { + this.dataCounts = counts; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + dataCounts = new DataCounts(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + dataCounts.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.ACCEPTED; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return dataCounts.doXContentBody(builder, params); + } + + @Override + public int hashCode() { + return Objects.hashCode(dataCounts); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + + return Objects.equals(dataCounts, other.dataCounts); + + } + } + + public static class Request extends ActionRequest { + + public static final ParseField IGNORE_DOWNTIME = new ParseField("ignoreDowntime"); + public static final ParseField RESET_START = new ParseField("resetStart"); + public static final ParseField RESET_END = new ParseField("resetEnd"); + + private String jobId; + private boolean ignoreDowntime = false; + private String resetStart; + private String resetEnd; + private BytesReference content; + + Request() { + } + + public Request(String jobId) { + ExceptionsHelper.requireNonNull(jobId, "jobId"); + this.jobId = jobId; + } + + public String getJobId() { + return jobId; + } + + public boolean isIgnoreDowntime() { + return ignoreDowntime; + } + + public void setIgnoreDowntime(boolean ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + } + + public String getResetStart() { + return resetStart; + } + + public void setResetStart(String resetStart) { + this.resetStart = resetStart; + } + + public String getResetEnd() { + return resetEnd; + } + + public void setResetEnd(String resetEnd) { + this.resetEnd = resetEnd; + } + + public BytesReference getContent() { return content; } + + public void setContent(BytesReference content) { + this.content = content; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + ignoreDowntime = in.readBoolean(); + resetStart = in.readOptionalString(); + resetEnd = in.readOptionalString(); + content = in.readBytesReference(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(ignoreDowntime); + out.writeOptionalString(resetStart); + out.writeOptionalString(resetEnd); + out.writeBytesReference(content); + } + + @Override + public int hashCode() { + // content stream not included + return Objects.hash(jobId, ignoreDowntime, resetStart, resetEnd); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + + // content stream not included + return Objects.equals(jobId, other.jobId) && + Objects.equals(ignoreDowntime, other.ignoreDowntime) && + Objects.equals(resetStart, other.resetStart) && + Objects.equals(resetEnd, other.resetEnd); + } + } + + + public static class TransportAction extends HandledTransportAction { + + private final AutodetectProcessManager processManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, AutodetectProcessManager processManager) { + super(settings, PostDataAction.NAME, false, threadPool, transportService, actionFilters, + indexNameExpressionResolver, Request::new); + this.processManager = processManager; + } + + @Override + protected final void doExecute(Request request, ActionListener listener) { + + TimeRange timeRange = TimeRange.builder().startTime(request.getResetStart()).endTime(request.getResetEnd()).build(); + DataLoadParams params = new DataLoadParams(timeRange, request.isIgnoreDowntime()); + + // NORELEASE Make this all async so we don't need to pass off to another thread pool and block + threadPool.executor(PrelertPlugin.THREAD_POOL_NAME).execute(() -> { + try { + DataCounts dataCounts = processManager.processData(request.getJobId(), request.content.streamInput(), params); + listener.onResponse(new Response(dataCounts)); + } catch (IOException | ElasticsearchException e) { + listener.onFailure(e); + } + }); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataCloseAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataCloseAction.java new file mode 100644 index 00000000000..6847337f90d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataCloseAction.java @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.manager.AutodetectProcessManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class PostDataCloseAction extends Action { + + public static final PostDataCloseAction INSTANCE = new PostDataCloseAction(); + public static final String NAME = "cluster:admin/prelert/data/post/close"; + + private PostDataCloseAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + private String jobId; + + Request() {} + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, "jobId"); + } + + public String getJobId() { + return jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PostDataCloseAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + private Response() { + } + + private Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + // NORELEASE This should be a master node operation that updates the job's state + public static class TransportAction extends HandledTransportAction { + + private final AutodetectProcessManager processManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, AutodetectProcessManager processManager) { + super(settings, PostDataCloseAction.NAME, false, threadPool, transportService, actionFilters, + indexNameExpressionResolver, Request::new); + + this.processManager = processManager; + } + + @Override + protected final void doExecute(Request request, ActionListener listener) { + processManager.closeJob(request.getJobId()); + listener.onResponse(new Response(true)); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataFlushAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataFlushAction.java new file mode 100644 index 00000000000..99c2ca17556 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PostDataFlushAction.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.AutodetectProcessManager; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.TimeRange; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class PostDataFlushAction extends Action { + + public static final PostDataFlushAction INSTANCE = new PostDataFlushAction(); + public static final String NAME = "cluster:admin/prelert/data/post/flush"; + + private PostDataFlushAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest implements ToXContent { + + public static final ParseField CALC_INTERIM = new ParseField("calcInterim"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField ADVANCE_TIME = new ParseField("advanceTime"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareBoolean(Request::setCalcInterim, CALC_INTERIM); + PARSER.declareString(Request::setStart, START); + PARSER.declareString(Request::setEnd, END); + PARSER.declareString(Request::setAdvanceTime, ADVANCE_TIME); + } + + public static Request parseRequest(String jobId, XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private boolean calcInterim = false; + private String start; + private String end; + private String advanceTime; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public boolean getCalcInterim() { + return calcInterim; + } + + public void setCalcInterim(boolean calcInterim) { + this.calcInterim = calcInterim; + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public String getAdvanceTime() { return advanceTime; } + + public void setAdvanceTime(String advanceTime) { + this.advanceTime = advanceTime; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + calcInterim = in.readBoolean(); + start = in.readOptionalString(); + end = in.readOptionalString(); + advanceTime = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(calcInterim); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeOptionalString(advanceTime); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, calcInterim, start, end, advanceTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + calcInterim == other.calcInterim && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(advanceTime, other.advanceTime); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(CALC_INTERIM.getPreferredName(), calcInterim); + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (advanceTime != null) { + builder.field(ADVANCE_TIME.getPreferredName(), advanceTime); + } + builder.endObject(); + return builder; + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PostDataFlushAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + private Response() { + } + + private Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + // NORELEASE This should be a master node operation that updates the job's state + private final AutodetectProcessManager processManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, AutodetectProcessManager processManager) { + super(settings, PostDataFlushAction.NAME, false, threadPool, transportService, actionFilters, + indexNameExpressionResolver, PostDataFlushAction.Request::new); + + this.processManager = processManager; + } + + @Override + protected final void doExecute(PostDataFlushAction.Request request, ActionListener listener) { + + TimeRange timeRange = TimeRange.builder().startTime(request.getStart()).endTime(request.getEnd()).build(); + InterimResultsParams params = InterimResultsParams.builder() + .calcInterim(request.getCalcInterim()) + .forTimeRange(timeRange) + .advanceTime(request.getAdvanceTime()) + .build(); + + processManager.flushJob(request.getJobId(), params); + listener.onResponse(new Response(true)); + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutJobAction.java new file mode 100644 index 00000000000..6cbe3875ecb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutJobAction.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; + +import java.io.IOException; +import java.util.Objects; + +public class PutJobAction extends Action { + + public static final PutJobAction INSTANCE = new PutJobAction(); + public static final String NAME = "cluster:admin/prelert/job/put"; + + private PutJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static Request parseRequest(XContentParser parser, ParseFieldMatcherSupplier matcherSupplier) { + Job job = Job.PARSER.apply(parser, matcherSupplier).build(true); + return new Request(job); + } + + private Job job; + private boolean overwrite; + + public Request(Job job) { + this.job = job; + } + + Request() { + } + + public Job getJob() { + return job; + } + + public boolean isOverwrite() { + return overwrite; + } + + public void setOverwrite(boolean overwrite) { + this.overwrite = overwrite; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + job = new Job(in); + overwrite = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + job.writeTo(out); + out.writeBoolean(overwrite); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + job.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return overwrite == request.overwrite && + Objects.equals(job, request.job); + } + + @Override + public int hashCode() { + return Objects.hash(job, overwrite); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements ToXContent { + + private Job job; + + public Response(boolean acked, Job job) { + super(acked); + this.job = job; + } + + Response() { + } + + public Job getResponse() { + return job; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + job = new Job(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + job.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Don't serialize acknowledged because current api directly serializes the job details + job.doXContentBody(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(job, response.job); + } + + @Override + public int hashCode() { + return Objects.hash(job); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager) { + super(settings, PutJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.putJob(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutListAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutListAction.java new file mode 100644 index 00000000000..01a7237c035 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutListAction.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + + +public class PutListAction extends Action { + + public static final PutListAction INSTANCE = new PutListAction(); + public static final String NAME = "cluster:admin/prelert/list/put"; + + private PutListAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest implements ToXContent { + + public static Request parseRequest(XContentParser parser, ParseFieldMatcherSupplier matcherSupplier) { + ListDocument listDocument = ListDocument.PARSER.apply(parser, matcherSupplier); + return new Request(listDocument); + } + + private ListDocument listDocument; + + Request() { + + } + + public Request(ListDocument listDocument) { + this.listDocument = ExceptionsHelper.requireNonNull(listDocument, "listDocument"); + } + + public ListDocument getListDocument() { + return this.listDocument; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + listDocument = new ListDocument(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + listDocument.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + listDocument.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(listDocument); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(listDocument, other.listDocument); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutListAction action) { + super(client, action, new Request()); + } + } + public static class Response extends AcknowledgedResponse { + + public Response() { + super(true); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + + } + + // extends TransportMasterNodeAction, because we will store in cluster state. + public static class TransportAction extends TransportMasterNodeAction { + + private final TransportIndexAction transportIndexAction; + + // TODO these need to be moved to a settings object later. See #20 + private static final String PRELERT_INFO_INDEX = "prelert-int"; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + TransportIndexAction transportIndexAction) { + super(settings, PutListAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.transportIndexAction = transportIndexAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + ListDocument listDocument = request.getListDocument(); + final String listId = listDocument.getId(); + IndexRequest indexRequest = new IndexRequest(PRELERT_INFO_INDEX, ListDocument.TYPE.getPreferredName(), listId); + XContentBuilder builder = XContentFactory.jsonBuilder(); + indexRequest.source(listDocument.toXContent(builder, ToXContent.EMPTY_PARAMS)); + transportIndexAction.execute(indexRequest, new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + listener.onResponse(new Response()); + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not create list with ID [" + listId + "]", e); + throw new ResourceNotFoundException("Could not create list with ID [" + listId + "]", e); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionAction.java new file mode 100644 index 00000000000..007ea286ddc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionAction.java @@ -0,0 +1,302 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class PutModelSnapshotDescriptionAction extends +Action { + + public static final PutModelSnapshotDescriptionAction INSTANCE = new PutModelSnapshotDescriptionAction(); + public static final String NAME = "cluster:admin/prelert/modelsnapshot/put/description"; + + private PutModelSnapshotDescriptionAction() { + super(NAME); + } + + @Override + public PutModelSnapshotDescriptionAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PutModelSnapshotDescriptionAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, snapshotId) -> request.snapshotId = snapshotId, ModelSnapshot.SNAPSHOT_ID); + PARSER.declareString((request, description) -> request.description = description, ModelSnapshot.DESCRIPTION); + } + + public static Request parseRequest(String jobId, String snapshotId, XContentParser parser, + ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + if (snapshotId != null) { + request.snapshotId = snapshotId; + } + return request; + } + + private String jobId; + private String snapshotId; + private String description; + + Request() { + } + + public Request(String jobId, String snapshotId, String description) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.snapshotId = ExceptionsHelper.requireNonNull(snapshotId, ModelSnapshot.SNAPSHOT_ID.getPreferredName()); + this.description = ExceptionsHelper.requireNonNull(description, ModelSnapshot.DESCRIPTION.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getSnapshotId() { + return snapshotId; + } + + public String getDescriptionString() { + return description; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readString(); + description = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeString(snapshotId); + out.writeString(description); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(ModelSnapshot.SNAPSHOT_ID.getPreferredName(), snapshotId); + builder.field(ModelSnapshot.DESCRIPTION.getPreferredName(), description); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, snapshotId, description); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(description, other.description); + } + } + + public static class Response extends ActionResponse implements StatusToXContent { + + private SingleDocument response; + + public Response() { + response = SingleDocument.empty(ModelSnapshot.TYPE.getPreferredName()); + } + + public Response(ModelSnapshot modelSnapshot) { + response = new SingleDocument<>(ModelSnapshot.TYPE.getPreferredName(), modelSnapshot); + } + + public SingleDocument getResponse() { + return response; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + response = new SingleDocument<>(in, ModelSnapshot::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + response.writeTo(out); + } + + @Override + public RestStatus status() { + return response.status(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return response.toXContent(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(response); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(response, other.response); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutModelSnapshotDescriptionAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final ElasticsearchJobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, ElasticsearchJobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + + logger.debug("Received request to change model snapshot description using '" + request.getDescriptionString() + + "' for snapshot ID '" + request.getSnapshotId() + "' for job '" + request.getJobId() + "'"); + + List changeCandidates = getChangeCandidates(request); + checkForClashes(request); + + if (changeCandidates.size() > 1) { + logger.warn("More than one model found for [jobId: " + request.getJobId() + ", snapshotId: " + request.getSnapshotId() + + "] tuple."); + } + ModelSnapshot modelSnapshot = changeCandidates.get(0); + modelSnapshot.setDescription(request.getDescriptionString()); + jobProvider.updateModelSnapshot(request.getJobId(), modelSnapshot, false); + + modelSnapshot.setDescription(request.getDescriptionString()); + + // The quantiles can be large, and totally dominate the output - + // it's + // clearer to remove them + modelSnapshot.setQuantiles(null); + + listener.onResponse(new Response(modelSnapshot)); + + } + + private List getChangeCandidates(Request request) { + List changeCandidates = getModelSnapshots(request.getJobId(), request.getSnapshotId(), null); + if (changeCandidates == null || changeCandidates.isEmpty()) { + throw new ResourceNotFoundException(Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId())); + } + return changeCandidates; + } + + private void checkForClashes(Request request) { + List clashCandidates = getModelSnapshots(request.getJobId(), null, request.getDescriptionString()); + if (clashCandidates != null && !clashCandidates.isEmpty()) { + throw new IllegalArgumentException(Messages.getMessage( + Messages.REST_DESCRIPTION_ALREADY_USED, request.getDescriptionString(), request.getJobId())); + } + } + + private List getModelSnapshots(String jobId, String snapshotId, String description) { + return jobProvider.modelSnapshots(jobId, 0, 1, null, null, null, true, snapshotId, description).hits(); + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ResumeJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ResumeJobAction.java new file mode 100644 index 00000000000..a40a878dd89 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ResumeJobAction.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class ResumeJobAction extends Action { + + public static final ResumeJobAction INSTANCE = new ResumeJobAction(); + public static final String NAME = "cluster:admin/prelert/job/resume"; + + private ResumeJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + ResumeJobAction.Request other = (ResumeJobAction.Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, ResumeJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, ResumeJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.info("Resuming job " + request.getJobId()); + jobManager.resumeJob(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotAction.java new file mode 100644 index 00000000000..85a9c3a2194 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotAction.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchBulkDeleterFactory; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.OldDataRemover; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class RevertModelSnapshotAction +extends Action { + + public static final RevertModelSnapshotAction INSTANCE = new RevertModelSnapshotAction(); + public static final String NAME = "indices:admin/prelert/modelsnapshots/revert"; + + private RevertModelSnapshotAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static final ParseField TIME = new ParseField("time"); + public static final ParseField SNAPSHOT_ID = new ParseField("snapshotId"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField DELETE_INTERVENING = new ParseField("deleteInterveningResults"); + + private static ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setTime, TIME); + PARSER.declareString(Request::setSnapshotId, SNAPSHOT_ID); + PARSER.declareString(Request::setDescription, DESCRIPTION); + PARSER.declareBoolean(Request::setDeleteInterveningResults, DELETE_INTERVENING); + } + + public static Request parseRequest(String jobId, XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String time; + private String snapshotId; + private String description; + private boolean deleteInterveningResults; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getTime() { + return time; + } + + public void setTime(String time) { + this.time = time; + } + + public String getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(String snapshotId) { + this.snapshotId = snapshotId; + } + + @Override + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public boolean getDeleteInterveningResults() { + return deleteInterveningResults; + } + + public void setDeleteInterveningResults(boolean deleteInterveningResults) { + this.deleteInterveningResults = deleteInterveningResults; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (time == null && snapshotId == null && description == null) { + validationException = addValidationError(Messages.getMessage(Messages.REST_INVALID_REVERT_PARAMS), validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + time = in.readOptionalString(); + snapshotId = in.readOptionalString(); + description = in.readOptionalString(); + deleteInterveningResults = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(time); + out.writeOptionalString(snapshotId); + out.writeOptionalString(description); + out.writeBoolean(deleteInterveningResults); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (time != null) { + builder.field(TIME.getPreferredName(), time); + } + if (snapshotId != null) { + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + builder.field(DELETE_INTERVENING.getPreferredName(), deleteInterveningResults); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, time, snapshotId, description, deleteInterveningResults); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(time, other.time) && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(description, other.description) + && Objects.equals(deleteInterveningResults, other.deleteInterveningResults); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements StatusToXContent { + + private SingleDocument response; + + public Response() { + super(false); + response = SingleDocument.empty(ModelSnapshot.TYPE.getPreferredName()); + } + + public Response(ModelSnapshot modelSnapshot) { + super(true); + response = new SingleDocument<>(ModelSnapshot.TYPE.getPreferredName(), modelSnapshot); + } + + public SingleDocument getResponse() { + return response; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + response = new SingleDocument<>(in, ModelSnapshot::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + response.writeTo(out); + } + + @Override + public RestStatus status() { + return response.status(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return response.toXContent(builder, params); + } + + @Override + public int hashCode() { + return Objects.hash(response); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(response, other.response); + } + + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + builder.startObject(); + toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + private final JobProvider jobProvider; + private final ElasticsearchBulkDeleterFactory bulkDeleterFactory; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, ElasticsearchJobProvider jobProvider, + ClusterService clusterService, ElasticsearchBulkDeleterFactory bulkDeleterFactory) { + super(settings, NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + this.jobProvider = jobProvider; + this.bulkDeleterFactory = bulkDeleterFactory; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Received request to revert to time '" + request.getTime() + "' description '" + request.getDescription() + + "' snapshot id '" + request.getSnapshotId() + "' for job '" + request.getJobId() + "', deleting intervening " + + " results: " + request.getDeleteInterveningResults()); + + if (request.getTime() == null && request.getSnapshotId() == null && request.getDescription() == null) { + throw new IllegalStateException(Messages.getMessage(Messages.REST_INVALID_REVERT_PARAMS)); + } + + Optional job = jobManager.getJob(request.getJobId(), clusterService.state()); + Allocation allocation = jobManager.getJobAllocation(request.getJobId()); + if (job.isPresent() && allocation.getStatus().equals(JobStatus.RUNNING)) { + throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT)); + } + + ModelSnapshot modelSnapshot = getModelSnapshot(request, jobProvider); + if (request.getDeleteInterveningResults()) { + listener = wrapListener(listener, modelSnapshot, request.getJobId()); + } + jobManager.revertSnapshot(request, listener, modelSnapshot); + } + + private ModelSnapshot getModelSnapshot(Request request, JobProvider provider) { + logger.info("Reverting to snapshot '" + request.getSnapshotId() + "' for time '" + request.getTime() + "'"); + + List revertCandidates; + revertCandidates = provider.modelSnapshots(request.getJobId(), 0, 1, null, request.getTime(), + ModelSnapshot.TIMESTAMP.getPreferredName(), true, request.getSnapshotId(), request.getDescription()).hits(); + + if (revertCandidates == null || revertCandidates.isEmpty()) { + throw new ResourceNotFoundException(Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId())); + } + ModelSnapshot modelSnapshot = revertCandidates.get(0); + + // The quantiles can be large, and totally dominate the output - + // it's + // clearer to remove them + modelSnapshot.setQuantiles(null); + return modelSnapshot; + } + + private ActionListener wrapListener(ActionListener listener, + ModelSnapshot modelSnapshot, String jobId) { + + // If we need to delete buckets that occurred after the snapshot, we + // wrap + // the listener with one that invokes the OldDataRemover on + // acknowledged responses + return ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + Date deleteAfter = modelSnapshot.getLatestResultTimeStamp(); + logger.debug("Removing intervening records: last record: " + deleteAfter + ", last result: " + + modelSnapshot.getLatestResultTimeStamp()); + + logger.info("Deleting buckets after '" + deleteAfter + "'"); + + // NORELEASE: OldDataRemover is basically delete-by-query. + // We should replace this + // whole abstraction with DBQ eventually + OldDataRemover remover = new OldDataRemover(jobProvider, bulkDeleterFactory); + remover.deleteResultsAfter(new ActionListener() { + @Override + public void onResponse(BulkResponse bulkItemResponses) { + listener.onResponse(response); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, jobId, deleteAfter.getTime() + 1); + } + }, listener::onFailure); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerAction.java new file mode 100644 index 00000000000..1f7ee21c138 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerAction.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class StartJobSchedulerAction +extends Action { + + public static final StartJobSchedulerAction INSTANCE = new StartJobSchedulerAction(); + public static final String NAME = "cluster:admin/prelert/job/scheduler/start"; + + private StartJobSchedulerAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareObject((request, schedulerState) -> request.schedulerState = schedulerState, SchedulerState.PARSER, + SchedulerState.TYPE_FIELD); + } + + public static Request parseRequest(String jobId, XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Request request = PARSER.apply(parser, parseFieldMatcherSupplier); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + // TODO (norelease): instead of providing a scheduler state, the user should just provide: startTimeMillis and endTimeMillis + // the state is useless here as it should always be STARTING + private SchedulerState schedulerState; + + public Request(String jobId, SchedulerState schedulerState) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.schedulerState = ExceptionsHelper.requireNonNull(schedulerState, SchedulerState.TYPE_FIELD.getPreferredName()); + if (schedulerState.getStatus() != JobSchedulerStatus.STARTING) { + throw new IllegalStateException( + "Start job scheduler action requires the scheduler status to be [" + JobSchedulerStatus.STARTING + "]"); + } + } + + Request() { + } + + public String getJobId() { + return jobId; + } + + public SchedulerState getSchedulerState() { + return schedulerState; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + schedulerState = new SchedulerState(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + schedulerState.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(SchedulerState.TYPE_FIELD.getPreferredName(), schedulerState); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, schedulerState); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(schedulerState, other.schedulerState); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, StartJobSchedulerAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() { + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager) { + super(settings, StartJobSchedulerAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.startJobScheduler(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerAction.java new file mode 100644 index 00000000000..4678ca7df07 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerAction.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; + +import java.io.IOException; +import java.util.Objects; + +public class StopJobSchedulerAction +extends Action { + + public static final StopJobSchedulerAction INSTANCE = new StopJobSchedulerAction(); + public static final String NAME = "cluster:admin/prelert/job/scheduler/stop"; + + private StopJobSchedulerAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = Objects.requireNonNull(jobId); + } + + Request() { + } + + public String getJobId() { + return jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, StopJobSchedulerAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() { + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager) { + super(settings, StopJobSchedulerAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.stopJobScheduler(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusAction.java new file mode 100644 index 00000000000..a05c2d3f1e3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusAction.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class UpdateJobSchedulerStatusAction extends Action { + + public static final UpdateJobSchedulerStatusAction INSTANCE = new UpdateJobSchedulerStatusAction(); + public static final String NAME = "cluster:admin/prelert/job/scheduler/status/update"; + + private UpdateJobSchedulerStatusAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + private JobSchedulerStatus schedulerStatus; + + public Request(String jobId, JobSchedulerStatus schedulerStatus) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.schedulerStatus = ExceptionsHelper.requireNonNull(schedulerStatus, SchedulerState.STATUS.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public JobSchedulerStatus getSchedulerStatus() { + return schedulerStatus; + } + + public void setSchedulerStatus(JobSchedulerStatus schedulerStatus) { + this.schedulerStatus = schedulerStatus; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + schedulerStatus = JobSchedulerStatus.fromStream(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + schedulerStatus.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, schedulerStatus); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + UpdateJobSchedulerStatusAction.Request other = (UpdateJobSchedulerStatusAction.Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(schedulerStatus, other.schedulerStatus); + } + + @Override + public String toString() { + return "Request{" + + "jobId='" + jobId + '\'' + + ", schedulerStatus=" + schedulerStatus + + '}'; + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, UpdateJobSchedulerStatusAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, UpdateJobSchedulerStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.updateSchedulerStatus(request.getJobId(), request.getSchedulerStatus()); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusAction.java new file mode 100644 index 00000000000..fb12cc71a8f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusAction.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.manager.JobManager; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class UpdateJobStatusAction + extends Action { + + public static final UpdateJobStatusAction INSTANCE = new UpdateJobStatusAction(); + public static final String NAME = "cluster:admin/prelert/job/status/update"; + + private UpdateJobStatusAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + private JobStatus status; + + public Request(String jobId, JobStatus status) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.status = ExceptionsHelper.requireNonNull(status, Allocation.STATUS.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public JobStatus getStatus() { + return status; + } + + public void setStatus(JobStatus status) { + this.status = status; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + status = JobStatus.fromStream(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + status.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, status); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + UpdateJobStatusAction.Request other = (UpdateJobStatusAction.Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(status, other.status); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, UpdateJobStatusAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, UpdateJobStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.setJobStatus(request.getJobId(), request.getStatus()); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorAction.java new file mode 100644 index 00000000000..b108a92f31e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import java.io.IOException; +import java.util.Objects; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.Detector; + +public class ValidateDetectorAction +extends Action { + + public static final ValidateDetectorAction INSTANCE = new ValidateDetectorAction(); + public static final String NAME = "cluster:admin/prelert/validate/detector"; + + protected ValidateDetectorAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, ValidateDetectorAction action) { + super(client, action, new Request()); + } + + } + + public static class Request extends ActionRequest implements ToXContent { + + private Detector detector; + + // NORELEASE this needs to change so the body is not directly the + // detector but and object that contains a field for the detector + public static Request parseRequest(XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + Detector detector = Detector.PARSER.apply(parser, parseFieldMatcherSupplier).build(); + return new Request(detector); + } + + Request() { + this.detector = null; + } + + public Request(Detector detector) { + this.detector = detector; + } + + public Detector getDetector() { + return detector; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + detector.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + detector = new Detector(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + detector.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(detector); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(detector, other.detector); + } + + } + + public static class Response extends AcknowledgedResponse { + + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, ValidateDetectorAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + listener.onResponse(new Response(true)); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformAction.java new file mode 100644 index 00000000000..077e68ff756 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import java.io.IOException; +import java.util.Objects; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigVerifier; + +public class ValidateTransformAction +extends Action { + + public static final ValidateTransformAction INSTANCE = new ValidateTransformAction(); + public static final String NAME = "cluster:admin/prelert/validate/transform"; + + protected ValidateTransformAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, ValidateTransformAction action) { + super(client, action, new Request()); + } + + } + + public static class Request extends ActionRequest implements ToXContent { + + private TransformConfig transform; + + public static Request parseRequest(XContentParser parser, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + TransformConfig transform = TransformConfig.PARSER.apply(parser, parseFieldMatcherSupplier); + return new Request(transform); + } + + Request() { + this.transform = null; + } + + public Request(TransformConfig transform) { + this.transform = transform; + } + + public TransformConfig getTransform() { + return transform; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + transform.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + transform = new TransformConfig(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + transform.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(transform); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(transform, other.transform); + } + + } + + public static class Response extends AcknowledgedResponse { + + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, ValidateTransformAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + TransformConfigVerifier.verify(request.getTransform()); + listener.onResponse(new Response(true)); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsAction.java new file mode 100644 index 00000000000..cba74b85fe6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsAction.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigsVerifier; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class ValidateTransformsAction +extends Action { + + public static final ValidateTransformsAction INSTANCE = new ValidateTransformsAction(); + public static final String NAME = "cluster:admin/prelert/validate/transforms"; + + protected ValidateTransformsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, ValidateTransformsAction action) { + super(client, action, new Request()); + } + + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField TRANSFORMS = new ParseField("transforms"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + a -> new Request((List) a[0])); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), TransformConfig.PARSER, TRANSFORMS); + } + + private List transforms; + + Request() { + this.transforms = null; + } + + public Request(List transforms) { + this.transforms = transforms; + } + + public List getTransforms() { + return transforms; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeList(transforms); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + transforms = in.readList(TransformConfig::new); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.array(TRANSFORMS.getPreferredName(), transforms.toArray(new Object[transforms.size()])); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(transforms); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(transforms, other.transforms); + } + + } + + public static class Response extends AcknowledgedResponse { + + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, ValidateTransformsAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + TransformConfigsVerifier.verify(request.getTransforms()); + listener.onResponse(new Response(true)); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisConfig.java new file mode 100644 index 00000000000..01ed4727d0f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisConfig.java @@ -0,0 +1,714 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; + + +/** + * Autodetect analysis configuration options describes which fields are + * analysed and the functions to use. + *

+ * The configuration can contain multiple detectors, a new anomaly detector will + * be created for each detector configuration. The fields + * bucketSpan, batchSpan, summaryCountFieldName and categorizationFieldName + * apply to all detectors. + *

+ * If a value has not been set it will be null + * Object wrappers are used around integral types & booleans so they can take + * null values. + */ +public class AnalysisConfig extends ToXContentToBytes implements Writeable { + /** + * Serialisation names + */ + private static final ParseField ANALYSIS_CONFIG = new ParseField("analysisConfig"); + private static final ParseField BUCKET_SPAN = new ParseField("bucketSpan"); + private static final ParseField BATCH_SPAN = new ParseField("batchSpan"); + private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorizationFieldName"); + private static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorizationFilters"); + private static final ParseField LATENCY = new ParseField("latency"); + private static final ParseField PERIOD = new ParseField("period"); + private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summaryCountFieldName"); + private static final ParseField DETECTORS = new ParseField("detectors"); + private static final ParseField INFLUENCERS = new ParseField("influencers"); + private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlappingBuckets"); + private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("resultFinalizationWindow"); + private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariateByFields"); + private static final ParseField MULTIPLE_BUCKET_SPANS = new ParseField("multipleBucketSpans"); + private static final ParseField USER_PER_PARTITION_NORMALIZATION = new ParseField("usePerPartitionNormalization"); + + private static final String PRELERT_CATEGORY_FIELD = "prelertcategory"; + public static final Set AUTO_CREATED_FIELDS = new HashSet<>(Arrays.asList(PRELERT_CATEGORY_FIELD)); + + public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L; + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(), a -> new AnalysisConfig.Builder((List) a[0])); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Detector.PARSER.apply(p, c).build(), DETECTORS); + PARSER.declareLong(Builder::setBucketSpan, BUCKET_SPAN); + PARSER.declareLong(Builder::setBatchSpan, BATCH_SPAN); + PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME); + PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS); + PARSER.declareLong(Builder::setLatency, LATENCY); + PARSER.declareLong(Builder::setPeriod, PERIOD); + PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME); + PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS); + PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); + PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW); + PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS); + PARSER.declareLongArray(Builder::setMultipleBucketSpans, MULTIPLE_BUCKET_SPANS); + PARSER.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION); + } + + /** + * These values apply to all detectors + */ + private final long bucketSpan; + private final Long batchSpan; + private final String categorizationFieldName; + private final List categorizationFilters; + private final long latency; + private final Long period; + private final String summaryCountFieldName; + private final List detectors; + private final List influencers; + private final Boolean overlappingBuckets; + private final Long resultFinalizationWindow; + private final Boolean multivariateByFields; + private final List multipleBucketSpans; + private final boolean usePerPartitionNormalization; + + private AnalysisConfig(Long bucketSpan, Long batchSpan, String categorizationFieldName, List categorizationFilters, + long latency, Long period, String summaryCountFieldName, List detectors, List influencers, + Boolean overlappingBuckets, Long resultFinalizationWindow, Boolean multivariateByFields, + List multipleBucketSpans, boolean usePerPartitionNormalization) { + this.detectors = detectors; + this.bucketSpan = bucketSpan; + this.batchSpan = batchSpan; + this.latency = latency; + this.period = period; + this.categorizationFieldName = categorizationFieldName; + this.categorizationFilters = categorizationFilters; + this.summaryCountFieldName = summaryCountFieldName; + this.influencers = influencers; + this.overlappingBuckets = overlappingBuckets; + this.resultFinalizationWindow = resultFinalizationWindow; + this.multivariateByFields = multivariateByFields; + this.multipleBucketSpans = multipleBucketSpans; + this.usePerPartitionNormalization = usePerPartitionNormalization; + } + + public AnalysisConfig(StreamInput in) throws IOException { + bucketSpan = in.readLong(); + batchSpan = in.readOptionalLong(); + categorizationFieldName = in.readOptionalString(); + categorizationFilters = in.readBoolean() ? in.readList(StreamInput::readString) : null; + latency = in.readLong(); + period = in.readOptionalLong(); + summaryCountFieldName = in.readOptionalString(); + detectors = in.readList(Detector::new); + influencers = in.readList(StreamInput::readString); + overlappingBuckets = in.readOptionalBoolean(); + resultFinalizationWindow = in.readOptionalLong(); + multivariateByFields = in.readOptionalBoolean(); + multipleBucketSpans = in.readBoolean() ? in.readList(StreamInput::readLong) : null; + usePerPartitionNormalization = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(bucketSpan); + out.writeOptionalLong(batchSpan); + out.writeOptionalString(categorizationFieldName); + if (categorizationFilters != null) { + out.writeBoolean(true); + out.writeStringList(categorizationFilters); + } else { + out.writeBoolean(false); + } + out.writeLong(latency); + out.writeOptionalLong(period); + out.writeOptionalString(summaryCountFieldName); + out.writeList(detectors); + out.writeStringList(influencers); + out.writeOptionalBoolean(overlappingBuckets); + out.writeOptionalLong(resultFinalizationWindow); + out.writeOptionalBoolean(multivariateByFields); + if (multipleBucketSpans != null) { + out.writeBoolean(true); + out.writeVInt(multipleBucketSpans.size()); + for (Long bucketSpan : multipleBucketSpans) { + out.writeLong(bucketSpan); + } + } else { + out.writeBoolean(false); + } + out.writeBoolean(usePerPartitionNormalization); + } + + /** + * The size of the interval the analysis is aggregated into measured in + * seconds + * + * @return The bucketspan or null if not set + */ + public Long getBucketSpan() { + return bucketSpan; + } + + public long getBucketSpanOrDefault() { + return bucketSpan; + } + + /** + * Interval into which to batch seasonal data measured in seconds + * + * @return The batchspan or null if not set + */ + public Long getBatchSpan() { + return batchSpan; + } + + public String getCategorizationFieldName() { + return categorizationFieldName; + } + + public List getCategorizationFilters() { + return categorizationFilters; + } + + /** + * The latency interval (seconds) during which out-of-order records should be handled. + * + * @return The latency interval (seconds) or null if not set + */ + public Long getLatency() { + return latency; + } + + /** + * The repeat interval for periodic data in multiples of + * {@linkplain #getBatchSpan()} + * + * @return The period or null if not set + */ + public Long getPeriod() { + return period; + } + + /** + * The name of the field that contains counts for pre-summarised input + * + * @return The field name or null if not set + */ + public String getSummaryCountFieldName() { + return summaryCountFieldName; + } + + /** + * The list of analysis detectors. In a valid configuration the list should + * contain at least 1 {@link Detector} + * + * @return The Detectors used in this job + */ + public List getDetectors() { + return detectors; + } + + /** + * The list of influence field names + */ + public List getInfluencers() { + return influencers; + } + + /** + * Return the list of term fields. + * These are the influencer fields, partition field, + * by field and over field of each detector. + * null and empty strings are filtered from the + * config. + * + * @return Set of term fields - never null + */ + public Set termFields() { + Set termFields = new TreeSet<>(); + + for (Detector d : getDetectors()) { + addIfNotNull(termFields, d.getByFieldName()); + addIfNotNull(termFields, d.getOverFieldName()); + addIfNotNull(termFields, d.getPartitionFieldName()); + } + + for (String i : getInfluencers()) { + addIfNotNull(termFields, i); + } + + // remove empty strings + termFields.remove(""); + + return termFields; + } + + public Set extractReferencedLists() { + return detectors.stream().map(Detector::extractReferencedLists) + .flatMap(Set::stream).collect(Collectors.toSet()); + } + + public Boolean getOverlappingBuckets() { + return overlappingBuckets; + } + + public Long getResultFinalizationWindow() { + return resultFinalizationWindow; + } + + public Boolean getMultivariateByFields() { + return multivariateByFields; + } + + public List getMultipleBucketSpans() { + return multipleBucketSpans; + } + + public boolean getUsePerPartitionNormalization() { + return usePerPartitionNormalization; + } + + /** + * Return the list of fields required by the analysis. + * These are the influencer fields, metric field, partition field, + * by field and over field of each detector, plus the summary count + * field and the categorization field name of the job. + * null and empty strings are filtered from the + * config. + * + * @return List of required analysis fields - never null + */ + public List analysisFields() { + Set analysisFields = termFields(); + + addIfNotNull(analysisFields, categorizationFieldName); + addIfNotNull(analysisFields, summaryCountFieldName); + + for (Detector d : getDetectors()) { + addIfNotNull(analysisFields, d.getFieldName()); + } + + // remove empty strings + analysisFields.remove(""); + + return new ArrayList<>(analysisFields); + } + + private static void addIfNotNull(Set fields, String field) { + if (field != null) { + fields.add(field); + } + } + + public List fields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getFieldName()); + } + + private List collectNonNullAndNonEmptyDetectorFields( + Function fieldGetter) { + Set fields = new HashSet<>(); + + for (Detector d : getDetectors()) { + addIfNotNull(fields, fieldGetter.apply(d)); + } + + // remove empty strings + fields.remove(""); + + return new ArrayList<>(fields); + } + + public List byFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getByFieldName()); + } + + public List overFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getOverFieldName()); + } + + + public List partitionFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getPartitionFieldName()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + if (batchSpan != null) { + builder.field(BATCH_SPAN.getPreferredName(), batchSpan); + } + if (categorizationFieldName != null) { + builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName); + } + if (categorizationFilters != null) { + builder.field(CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); + } + builder.field(LATENCY.getPreferredName(), latency); + if (period != null) { + builder.field(PERIOD.getPreferredName(), period); + } + if (summaryCountFieldName != null) { + builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName); + } + builder.field(DETECTORS.getPreferredName(), detectors); + builder.field(INFLUENCERS.getPreferredName(), influencers); + if (overlappingBuckets != null) { + builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets); + } + if (resultFinalizationWindow != null) { + builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow); + } + if (multivariateByFields != null) { + builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields); + } + if (multipleBucketSpans != null) { + builder.field(MULTIPLE_BUCKET_SPANS.getPreferredName(), multipleBucketSpans); + } + builder.field(USER_PER_PARTITION_NORMALIZATION.getPreferredName(), usePerPartitionNormalization); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AnalysisConfig that = (AnalysisConfig) o; + return latency == that.latency && + usePerPartitionNormalization == that.usePerPartitionNormalization && + Objects.equals(bucketSpan, that.bucketSpan) && + Objects.equals(batchSpan, that.batchSpan) && + Objects.equals(categorizationFieldName, that.categorizationFieldName) && + Objects.equals(categorizationFilters, that.categorizationFilters) && + Objects.equals(period, that.period) && + Objects.equals(summaryCountFieldName, that.summaryCountFieldName) && + Objects.equals(detectors, that.detectors) && + Objects.equals(influencers, that.influencers) && + Objects.equals(overlappingBuckets, that.overlappingBuckets) && + Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) && + Objects.equals(multivariateByFields, that.multivariateByFields) && + Objects.equals(multipleBucketSpans, that.multipleBucketSpans); + } + + @Override + public int hashCode() { + return Objects.hash( + bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, latency, period, + summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, + multivariateByFields, multipleBucketSpans, usePerPartitionNormalization + ); + } + + public static class Builder { + + public static final long DEFAULT_BUCKET_SPAN = 300L; + + private List detectors; + private long bucketSpan = DEFAULT_BUCKET_SPAN; + private Long batchSpan; + private long latency = 0L; + private Long period; + private String categorizationFieldName; + private List categorizationFilters; + private String summaryCountFieldName; + private List influencers = new ArrayList<>(); + private Boolean overlappingBuckets; + private Long resultFinalizationWindow; + private Boolean multivariateByFields; + private List multipleBucketSpans; + private boolean usePerPartitionNormalization = false; + + public Builder(List detectors) { + this.detectors = detectors; + } + + Builder(AnalysisConfig analysisConfig) { + this.detectors = analysisConfig.detectors; + this.bucketSpan = analysisConfig.bucketSpan; + this.batchSpan = analysisConfig.batchSpan; + this.latency = analysisConfig.latency; + this.period = analysisConfig.period; + this.categorizationFieldName = analysisConfig.categorizationFieldName; + this.categorizationFilters = analysisConfig.categorizationFilters; + this.summaryCountFieldName = analysisConfig.summaryCountFieldName; + this.influencers = analysisConfig.influencers; + this.overlappingBuckets = analysisConfig.overlappingBuckets; + this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; + this.multivariateByFields = analysisConfig.multivariateByFields; + this.multipleBucketSpans = analysisConfig.multipleBucketSpans; + this.usePerPartitionNormalization = analysisConfig.usePerPartitionNormalization; + } + + public void setDetectors(List detectors) { + this.detectors = detectors; + } + + public void setBucketSpan(long bucketSpan) { + this.bucketSpan = bucketSpan; + } + + public void setBatchSpan(long batchSpan) { + this.batchSpan = batchSpan; + } + + public void setLatency(long latency) { + this.latency = latency; + } + + public void setPeriod(long period) { + this.period = period; + } + + public void setCategorizationFieldName(String categorizationFieldName) { + this.categorizationFieldName = categorizationFieldName; + } + + public void setCategorizationFilters(List categorizationFilters) { + this.categorizationFilters = categorizationFilters; + } + + public void setSummaryCountFieldName(String summaryCountFieldName) { + this.summaryCountFieldName = summaryCountFieldName; + } + + public void setInfluencers(List influencers) { + this.influencers = influencers; + } + + public void setOverlappingBuckets(Boolean overlappingBuckets) { + this.overlappingBuckets = overlappingBuckets; + } + + public void setResultFinalizationWindow(Long resultFinalizationWindow) { + this.resultFinalizationWindow = resultFinalizationWindow; + } + + public void setMultivariateByFields(Boolean multivariateByFields) { + this.multivariateByFields = multivariateByFields; + } + + public void setMultipleBucketSpans(List multipleBucketSpans) { + this.multipleBucketSpans = multipleBucketSpans; + } + + public void setUsePerPartitionNormalization(boolean usePerPartitionNormalization) { + this.usePerPartitionNormalization = usePerPartitionNormalization; + } + + /** + * Checks the configuration is valid + *

    + *
  1. Check that if non-null BucketSpan, BatchSpan, Latency and Period are + * >= 0
    2. + *
  2. Check that if non-null Latency is <= MAX_LATENCY
    3. + *
  3. Check there is at least one detector configured
    4. + *
  4. Check all the detectors are configured correctly
    5. + *
  5. Check that OVERLAPPING_BUCKETS is set appropriately
    6. + *
  6. Check that MULTIPLE_BUCKETSPANS are set appropriately
    7. + *
  7. If Per Partition normalization is configured at least one detector + * must have a partition field and no influences can be used
    8. + *
+ */ + public AnalysisConfig build() { + checkFieldIsNotNegativeIfSpecified(BUCKET_SPAN.getPreferredName(), bucketSpan); + checkFieldIsNotNegativeIfSpecified(BATCH_SPAN.getPreferredName(), batchSpan); + checkFieldIsNotNegativeIfSpecified(LATENCY.getPreferredName(), latency); + checkFieldIsNotNegativeIfSpecified(PERIOD.getPreferredName(), period); + + verifyDetectorAreDefined(detectors); + verifyFieldName(summaryCountFieldName); + verifyFieldName(categorizationFieldName); + + verifyCategorizationFilters(categorizationFilters, categorizationFieldName); + verifyMultipleBucketSpans(multipleBucketSpans, bucketSpan); + + overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors); + + if (usePerPartitionNormalization) { + checkDetectorsHavePartitionFields(detectors); + checkNoInfluencersAreSet(influencers); + } + + return new AnalysisConfig(bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, + latency, period, summaryCountFieldName, detectors, influencers, overlappingBuckets, + resultFinalizationWindow, multivariateByFields, multipleBucketSpans, usePerPartitionNormalization); + } + + private static void checkFieldIsNotNegativeIfSpecified(String fieldName, Long value) { + if (value != null && value < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, fieldName, 0, value); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyDetectorAreDefined(List detectors) { + if (detectors == null || detectors.isEmpty()) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS)); + } + } + + private static void verifyCategorizationFilters(List filters, String categorizationFieldName) { + if (filters == null || filters.isEmpty()) { + return; + } + + verifyCategorizationFieldNameSetIfFiltersAreSet(categorizationFieldName); + verifyCategorizationFiltersAreDistinct(filters); + verifyCategorizationFiltersContainNoneEmpty(filters); + verifyCategorizationFiltersAreValidRegex(filters); + } + + private static void verifyCategorizationFieldNameSetIfFiltersAreSet(String categorizationFieldName) { + if (categorizationFieldName == null) { + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME)); + } + } + + private static void verifyCategorizationFiltersAreDistinct(List filters) { + if (filters.stream().distinct().count() != filters.size()) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES)); + } + } + + private static void verifyCategorizationFiltersContainNoneEmpty(List filters) { + if (filters.stream().anyMatch(f -> f.isEmpty())) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY)); + } + } + + private static void verifyCategorizationFiltersAreValidRegex(List filters) { + for (String filter : filters) { + if (!isValidRegex(filter)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX, filter)); + } + } + } + + private static void verifyMultipleBucketSpans(List multipleBucketSpans, Long bucketSpan) { + if (multipleBucketSpans == null) { + return; + } + + if (bucketSpan == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_REQUIRE_BUCKETSPAN)); + } + for (Long span : multipleBucketSpans) { + if ((span % bucketSpan != 0L) || (span <= bucketSpan)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, span, bucketSpan)); + } + } + } + + private static boolean checkDetectorsHavePartitionFields(List detectors) { + for (Detector detector : detectors) { + if (!Strings.isNullOrEmpty(detector.getPartitionFieldName())) { + return true; + } + } + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD)); + } + + private static boolean checkNoInfluencersAreSet(List influencers) { + if (!influencers.isEmpty()) { + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS)); + } + + return true; + } + + /** + * Check that the characters used in a field name will not cause problems. + * + * @param field The field name to be validated + * @return true + */ + public static boolean verifyFieldName(String field) throws ElasticsearchParseException { + if (field != null && containsInvalidChar(field)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_INVALID_FIELDNAME_CHARS, field, Detector.PROHIBITED)); + } + return true; + } + + private static boolean containsInvalidChar(String field) { + for (Character ch : Detector.PROHIBITED_FIELDNAME_CHARACTERS) { + if (field.indexOf(ch) >= 0) { + return true; + } + } + return field.chars().anyMatch(ch -> Character.isISOControl(ch)); + } + + private static boolean isValidRegex(String exp) { + try { + Pattern.compile(exp); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } + + private static Boolean verifyOverlappingBucketsConfig(Boolean overlappingBuckets, List detectors) { + // If any detector function is rare/freq_rare, mustn't use overlapping buckets + boolean mustNotUse = false; + + List illegalFunctions = new ArrayList<>(); + for (Detector d : detectors) { + if (Detector.NO_OVERLAPPING_BUCKETS_FUNCTIONS.contains(d.getFunction())) { + illegalFunctions.add(d.getFunction()); + mustNotUse = true; + } + } + + if (Boolean.TRUE.equals(overlappingBuckets) && mustNotUse) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION, illegalFunctions.toString())); + } + + return overlappingBuckets; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisLimits.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisLimits.java new file mode 100644 index 00000000000..556761b6a92 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/AnalysisLimits.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.Objects; + +/** + * Analysis limits for autodetect + *

+ * If an option has not been set it shouldn't be used so the default value is picked up instead. + */ +public class AnalysisLimits extends ToXContentToBytes implements Writeable { + /** + * Serialisation field names + */ + public static final ParseField MODEL_MEMORY_LIMIT = new ParseField("modelMemoryLimit"); + public static final ParseField CATEGORIZATION_EXAMPLES_LIMIT = new ParseField("categorizationExamplesLimit"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "analysis_limits", a -> new AnalysisLimits((Long) a[0], (Long) a[1])); + + static { + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MODEL_MEMORY_LIMIT); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CATEGORIZATION_EXAMPLES_LIMIT); + } + + /** + * It is initialised to null. + * A value of null or 0 will result to the default being used. + */ + private final Long modelMemoryLimit; + + /** + * It is initialised to null. + * A value of null will result to the default being used. + */ + private final Long categorizationExamplesLimit; + + public AnalysisLimits(Long modelMemoryLimit, Long categorizationExamplesLimit) { + this.modelMemoryLimit = modelMemoryLimit; + if (categorizationExamplesLimit != null && categorizationExamplesLimit < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, CATEGORIZATION_EXAMPLES_LIMIT, 0, + categorizationExamplesLimit); + throw new IllegalArgumentException(msg); + } + this.categorizationExamplesLimit = categorizationExamplesLimit; + } + + public AnalysisLimits(StreamInput in) throws IOException { + this(in.readOptionalLong(), in.readOptionalLong()); + } + + /** + * Maximum size of the model in MB before the anomaly detector + * will drop new samples to prevent the model using any more + * memory + * + * @return The set memory limit or null if not set + */ + @Nullable + public Long getModelMemoryLimit() { + return modelMemoryLimit; + } + + /** + * Gets the limit to the number of examples that are stored per category + * + * @return the limit or null if not set + */ + @Nullable + public Long getCategorizationExamplesLimit() { + return categorizationExamplesLimit; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalLong(modelMemoryLimit); + out.writeOptionalLong(categorizationExamplesLimit); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (modelMemoryLimit != null) { + builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), modelMemoryLimit); + } + if (categorizationExamplesLimit != null) { + builder.field(CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), categorizationExamplesLimit); + } + builder.endObject(); + return builder; + } + + /** + * Overridden equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof AnalysisLimits == false) { + return false; + } + + AnalysisLimits that = (AnalysisLimits) other; + return Objects.equals(this.modelMemoryLimit, that.modelMemoryLimit) && + Objects.equals(this.categorizationExamplesLimit, that.categorizationExamplesLimit); + } + + @Override + public int hashCode() { + return Objects.hash(modelMemoryLimit, categorizationExamplesLimit); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java new file mode 100644 index 00000000000..b5353c0bbbb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/CategorizerState.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + + +/** + * The categorizer state does not need to be loaded on the Java side. + * However, the Java process DOES set up a mapping on the Elasticsearch + * index to tell Elasticsearch not to analyse the categorizer state documents + * in any way. + */ +public class CategorizerState { + /** + * The type of this class used when persisting the data + */ + public static final String TYPE = "categorizerState"; + + private CategorizerState() { + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataCounts.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataCounts.java new file mode 100644 index 00000000000..a0c8622c6f8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataCounts.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Job processed record counts. + *

+ * The getInput... methods return the actual number of + * fields/records sent the the API including invalid records. + * The getProcessed... methods are the number sent to the + * Engine. + *

+ * The inputRecordCount field is calculated so it + * should not be set in deserialisation but it should be serialised + * so the field is visible. + */ + +public class DataCounts extends ToXContentToBytes implements Writeable { + + public static final String DOCUMENT_SUFFIX = "-data-counts"; + public static final String PROCESSED_RECORD_COUNT_STR = "processed_record_count"; + public static final String PROCESSED_FIELD_COUNT_STR = "processed_field_count"; + public static final String INPUT_BYTES_STR = "input_bytes"; + public static final String INPUT_RECORD_COUNT_STR = "input_record_count"; + public static final String INPUT_FIELD_COUNT_STR = "input_field_count"; + public static final String INVALID_DATE_COUNT_STR = "invalid_date_count"; + public static final String MISSING_FIELD_COUNT_STR = "missing_field_count"; + public static final String OUT_OF_ORDER_TIME_COUNT_STR = "out_of_order_timestamp_count"; + public static final String EARLIEST_RECORD_TIME_STR = "earliest_record_timestamp"; + public static final String LATEST_RECORD_TIME_STR = "latest_record_timestamp"; + + public static final ParseField PROCESSED_RECORD_COUNT = new ParseField(PROCESSED_RECORD_COUNT_STR); + public static final ParseField PROCESSED_FIELD_COUNT = new ParseField(PROCESSED_FIELD_COUNT_STR); + public static final ParseField INPUT_BYTES = new ParseField(INPUT_BYTES_STR); + public static final ParseField INPUT_RECORD_COUNT = new ParseField(INPUT_RECORD_COUNT_STR); + public static final ParseField INPUT_FIELD_COUNT = new ParseField(INPUT_FIELD_COUNT_STR); + public static final ParseField INVALID_DATE_COUNT = new ParseField(INVALID_DATE_COUNT_STR); + public static final ParseField MISSING_FIELD_COUNT = new ParseField(MISSING_FIELD_COUNT_STR); + public static final ParseField OUT_OF_ORDER_TIME_COUNT = new ParseField(OUT_OF_ORDER_TIME_COUNT_STR); + public static final ParseField EARLIEST_RECORD_TIME = new ParseField(EARLIEST_RECORD_TIME_STR); + public static final ParseField LATEST_RECORD_TIME = new ParseField(LATEST_RECORD_TIME_STR); + + public static final ParseField TYPE = new ParseField("dataCounts"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("data_counts", a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], + (long) a[4], (long) a[5], (long) a[6], (long) a[7], (Date) a[8], (Date) a[9])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_RECORD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_BYTES); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INVALID_DATE_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MISSING_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_OF_ORDER_TIME_COUNT); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + EARLIEST_RECORD_TIME.getPreferredName() + "]"); + }, EARLIEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RECORD_TIME.getPreferredName() + "]"); + }, LATEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareLong((t, u) -> {;}, INPUT_RECORD_COUNT); + } + + private final String jobId; + private long processedRecordCount; + private long processedFieldCount; + private long inputBytes; + private long inputFieldCount; + private long invalidDateCount; + private long missingFieldCount; + private long outOfOrderTimeStampCount; + // NORELEASE: Use Jodatime instead + private Date earliestRecordTimeStamp; + private Date latestRecordTimeStamp; + + public DataCounts(String jobId, long processedRecordCount, long processedFieldCount, long inputBytes, + long inputFieldCount, long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, + Date earliestRecordTimeStamp, Date latestRecordTimeStamp) { + this.jobId = jobId; + this.processedRecordCount = processedRecordCount; + this.processedFieldCount = processedFieldCount; + this.inputBytes = inputBytes; + this.inputFieldCount = inputFieldCount; + this.invalidDateCount = invalidDateCount; + this.missingFieldCount = missingFieldCount; + this.outOfOrderTimeStampCount = outOfOrderTimeStampCount; + this.latestRecordTimeStamp = latestRecordTimeStamp; + this.earliestRecordTimeStamp = earliestRecordTimeStamp; + } + + public DataCounts(String jobId) { + this.jobId = jobId; + } + + public DataCounts(DataCounts lhs) { + jobId = lhs.jobId; + processedRecordCount = lhs.processedRecordCount; + processedFieldCount = lhs.processedFieldCount; + inputBytes = lhs.inputBytes; + inputFieldCount = lhs.inputFieldCount; + invalidDateCount = lhs.invalidDateCount; + missingFieldCount = lhs.missingFieldCount; + outOfOrderTimeStampCount = lhs.outOfOrderTimeStampCount; + latestRecordTimeStamp = lhs.latestRecordTimeStamp; + earliestRecordTimeStamp = lhs.earliestRecordTimeStamp; + } + + public DataCounts(StreamInput in) throws IOException { + jobId = in.readString(); + processedRecordCount = in.readVLong(); + processedFieldCount = in.readVLong(); + inputBytes = in.readVLong(); + inputFieldCount = in.readVLong(); + invalidDateCount = in.readVLong(); + missingFieldCount = in.readVLong(); + outOfOrderTimeStampCount = in.readVLong(); + if (in.readBoolean()) { + latestRecordTimeStamp = new Date(in.readVLong()); + } + if (in.readBoolean()) { + earliestRecordTimeStamp = new Date(in.readVLong()); + } + in.readVLong(); // throw away inputRecordCount + } + + public String getJobid() { + return jobId; + } + + /** + * Number of records processed by this job. + * This value is the number of records sent passed on to + * the engine i.e. {@linkplain #getInputRecordCount()} minus + * records with bad dates or out of order + * + * @return Number of records processed by this job {@code long} + */ + public long getProcessedRecordCount() { + return processedRecordCount; + } + + public void incrementProcessedRecordCount(long additional) { + processedRecordCount += additional; + } + + /** + * Number of data points (processed record count * the number + * of analysed fields) processed by this job. This count does + * not include the time field. + * + * @return Number of data points processed by this job {@code long} + */ + public long getProcessedFieldCount() { + return processedFieldCount; + } + + public void calcProcessedFieldCount(long analysisFieldsPerRecord) { + processedFieldCount = + (processedRecordCount * analysisFieldsPerRecord) + - missingFieldCount; + + // processedFieldCount could be a -ve value if no + // records have been written in which case it should be 0 + processedFieldCount = (processedFieldCount < 0) ? 0 : processedFieldCount; + } + + /** + * Total number of input records read. + * This = processed record count + date parse error records count + * + out of order record count. + *

+ * Records with missing fields are counted as they are still written. + * + * @return Total number of input records read {@code long} + */ + public long getInputRecordCount() { + return processedRecordCount + outOfOrderTimeStampCount + + invalidDateCount; + } + + /** + * The total number of bytes sent to this job. + * This value includes the bytes from any records + * that have been discarded for any reason + * e.g. because the date cannot be read + * + * @return Volume in bytes + */ + public long getInputBytes() { + return inputBytes; + } + + public void incrementInputBytes(long additional) { + inputBytes += additional; + } + + /** + * The total number of fields sent to the job + * including fields that aren't analysed. + * + * @return The total number of fields sent to the job + */ + public long getInputFieldCount() { + return inputFieldCount; + } + + public void incrementInputFieldCount(long additional) { + inputFieldCount += additional; + } + + /** + * The number of records with an invalid date field that could + * not be parsed or converted to epoch time. + * + * @return The number of records with an invalid date field + */ + public long getInvalidDateCount() { + return invalidDateCount; + } + + public void incrementInvalidDateCount(long additional) { + invalidDateCount += additional; + } + + + /** + * The number of missing fields that had been + * configured for analysis. + * + * @return The number of missing fields + */ + public long getMissingFieldCount() { + return missingFieldCount; + } + + public void incrementMissingFieldCount(long additional) { + missingFieldCount += additional; + } + + /** + * The number of records with a timestamp that is + * before the time of the latest record. Records should + * be in ascending chronological order + * + * @return The number of records with a timestamp that is before the time of the latest record + */ + public long getOutOfOrderTimeStampCount() { + return outOfOrderTimeStampCount; + } + + public void incrementOutOfOrderTimeStampCount(long additional) { + outOfOrderTimeStampCount += additional; + } + + /** + * The time of the first record seen. + * + * @return The first record time + */ + public Date getEarliestRecordTimeStamp() { + return earliestRecordTimeStamp; + } + + /** + * If {@code earliestRecordTimeStamp} has not been set (i.e. is {@code null}) + * then set it to {@code timeStamp} + * + * @param timeStamp Candidate time + * @throws IllegalStateException if {@code earliestRecordTimeStamp} is already set + */ + public void setEarliestRecordTimeStamp(Date timeStamp) { + if (earliestRecordTimeStamp != null) { + throw new IllegalStateException("earliestRecordTimeStamp can only be set once"); + } + earliestRecordTimeStamp = timeStamp; + } + + + /** + * The time of the latest record seen. + * + * @return Latest record time + */ + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + public void setLatestRecordTimeStamp(Date latestRecordTimeStamp) { + this.latestRecordTimeStamp = latestRecordTimeStamp; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeVLong(processedRecordCount); + out.writeVLong(processedFieldCount); + out.writeVLong(inputBytes); + out.writeVLong(inputFieldCount); + out.writeVLong(invalidDateCount); + out.writeVLong(missingFieldCount); + out.writeVLong(outOfOrderTimeStampCount); + if (latestRecordTimeStamp != null) { + out.writeBoolean(true); + out.writeVLong(latestRecordTimeStamp.getTime()); + } else { + out.writeBoolean(false); + } + if (earliestRecordTimeStamp != null) { + out.writeBoolean(true); + out.writeVLong(earliestRecordTimeStamp.getTime()); + } else { + out.writeBoolean(false); + } + out.writeVLong(getInputRecordCount()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(PROCESSED_RECORD_COUNT.getPreferredName(), processedRecordCount); + builder.field(PROCESSED_FIELD_COUNT.getPreferredName(), processedFieldCount); + builder.field(INPUT_BYTES.getPreferredName(), inputBytes); + builder.field(INPUT_FIELD_COUNT.getPreferredName(), inputFieldCount); + builder.field(INVALID_DATE_COUNT.getPreferredName(), invalidDateCount); + builder.field(MISSING_FIELD_COUNT.getPreferredName(), missingFieldCount); + builder.field(OUT_OF_ORDER_TIME_COUNT.getPreferredName(), outOfOrderTimeStampCount); + if (earliestRecordTimeStamp != null) { + builder.field(EARLIEST_RECORD_TIME.getPreferredName(), earliestRecordTimeStamp.getTime()); + } + if (latestRecordTimeStamp != null) { + builder.field(LATEST_RECORD_TIME.getPreferredName(), latestRecordTimeStamp.getTime()); + } + builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount()); + + return builder; + } + + /** + * Equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof DataCounts == false) { + return false; + } + + DataCounts that = (DataCounts) other; + + return Objects.equals(this.jobId, that.jobId) && + this.processedRecordCount == that.processedRecordCount && + this.processedFieldCount == that.processedFieldCount && + this.inputBytes == that.inputBytes && + this.inputFieldCount == that.inputFieldCount && + this.invalidDateCount == that.invalidDateCount && + this.missingFieldCount == that.missingFieldCount && + this.outOfOrderTimeStampCount == that.outOfOrderTimeStampCount && + Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) && + Objects.equals(this.earliestRecordTimeStamp, that.earliestRecordTimeStamp); + + } + + @Override + public int hashCode() { + return Objects.hash(jobId, processedRecordCount, processedFieldCount, + inputBytes, inputFieldCount, invalidDateCount, missingFieldCount, + outOfOrderTimeStampCount, latestRecordTimeStamp, earliestRecordTimeStamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataDescription.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataDescription.java new file mode 100644 index 00000000000..5119e6dda15 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/DataDescription.java @@ -0,0 +1,358 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.time.DateTimeFormatterTimestampConverter; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +/** + * Describes the format of the data used in the job and how it should + * be interpreted by autodetect. + *

+ * Data must either be in a textual delineated format (e.g. csv, tsv) or JSON + * the {@linkplain DataFormat} enum indicates which. {@link #getTimeField()} + * is the name of the field containing the timestamp and {@link #getTimeFormat()} + * is the format code for the date string in as described by + * {@link java.time.format.DateTimeFormatter}. The default quote character for + * delineated formats is {@value #DEFAULT_QUOTE_CHAR} but any other character can be + * used. + */ +public class DataDescription extends ToXContentToBytes implements Writeable { + /** + * Enum of the acceptable data formats. + */ + public enum DataFormat implements Writeable { + JSON("json"), + DELIMITED("delimited"), + SINGLE_LINE("single_line"), + ELASTICSEARCH("elasticsearch"); + + /** + * Delimited used to be called delineated. We keep supporting that for backwards + * compatibility. + */ + private static final String DEPRECATED_DELINEATED = "DELINEATED"; + private String name; + + private DataFormat(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Case-insensitive from string method. + * Works with either JSON, json, etc. + * + * @param value String representation + * @return The data format + */ + public static DataFormat forString(String value) { + String valueUpperCase = value.toUpperCase(Locale.ROOT); + return DEPRECATED_DELINEATED.equals(valueUpperCase) ? DELIMITED : DataFormat + .valueOf(valueUpperCase); + } + + public static DataFormat readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown DataFormat ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + } + + private static final ParseField DATA_DESCRIPTION_FIELD = new ParseField("dataDescription"); + private static final ParseField FORMAT_FIELD = new ParseField("format"); + private static final ParseField TIME_FIELD_NAME_FIELD = new ParseField("timeField"); + private static final ParseField TIME_FORMAT_FIELD = new ParseField("timeFormat"); + private static final ParseField FIELD_DELIMITER_FIELD = new ParseField("fieldDelimiter"); + private static final ParseField QUOTE_CHARACTER_FIELD = new ParseField("quoteCharacter"); + + /** + * Special time format string for epoch times (seconds) + */ + public static final String EPOCH = "epoch"; + + /** + * Special time format string for epoch times (milli-seconds) + */ + public static final String EPOCH_MS = "epoch_ms"; + + /** + * By default autodetect expects the timestamp in a field with this name + */ + public static final String DEFAULT_TIME_FIELD = "time"; + + /** + * The default field delimiter expected by the native autodetect + * program. + */ + public static final char DEFAULT_DELIMITER = '\t'; + + /** + * Csv data must have this line ending + */ + public static final char LINE_ENDING = '\n'; + + /** + * The default quote character used to escape text in + * delineated data formats + */ + public static final char DEFAULT_QUOTE_CHAR = '"'; + + private final DataFormat dataFormat; + private final String timeFieldName; + private final String timeFormat; + private final char fieldDelimiter; + private final char quoteCharacter; + + public static final ObjectParser PARSER = + new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), Builder::new); + + static { + PARSER.declareString(Builder::setFormat, FORMAT_FIELD); + PARSER.declareString(Builder::setTimeField, TIME_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setTimeFormat, TIME_FORMAT_FIELD); + PARSER.declareField(Builder::setFieldDelimiter, DataDescription::extractChar, FIELD_DELIMITER_FIELD, ValueType.STRING); + PARSER.declareField(Builder::setQuoteCharacter, DataDescription::extractChar, QUOTE_CHARACTER_FIELD, ValueType.STRING); + } + + public DataDescription(DataFormat dataFormat, String timeFieldName, String timeFormat, char fieldDelimiter, char quoteCharacter) { + this.dataFormat = dataFormat; + this.timeFieldName = timeFieldName; + this.timeFormat = timeFormat; + this.fieldDelimiter = fieldDelimiter; + this.quoteCharacter = quoteCharacter; + } + + public DataDescription(StreamInput in) throws IOException { + dataFormat = DataFormat.readFromStream(in); + timeFieldName = in.readString(); + timeFormat = in.readString(); + fieldDelimiter = (char) in.read(); + quoteCharacter = (char) in.read(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + dataFormat.writeTo(out); + out.writeString(timeFieldName); + out.writeString(timeFormat); + out.write(fieldDelimiter); + out.write(quoteCharacter); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FORMAT_FIELD.getPreferredName(), dataFormat); + builder.field(TIME_FIELD_NAME_FIELD.getPreferredName(), timeFieldName); + builder.field(TIME_FORMAT_FIELD.getPreferredName(), timeFormat); + builder.field(FIELD_DELIMITER_FIELD.getPreferredName(), String.valueOf(fieldDelimiter)); + builder.field(QUOTE_CHARACTER_FIELD.getPreferredName(), String.valueOf(quoteCharacter)); + builder.endObject(); + return builder; + } + + /** + * The format of the data to be processed. + * Defaults to {@link DataDescription.DataFormat#DELIMITED} + * + * @return The data format + */ + public DataFormat getFormat() { + return dataFormat; + } + + /** + * The name of the field containing the timestamp + * + * @return A String if set or null + */ + public String getTimeField() { + return timeFieldName; + } + + /** + * Either {@value #EPOCH}, {@value #EPOCH_MS} or a SimpleDateTime format string. + * If not set (is null or an empty string) or set to + * {@value #EPOCH} (the default) then the date is assumed to be in + * seconds from the epoch. + * + * @return A String if set or null + */ + public String getTimeFormat() { + return timeFormat; + } + + /** + * If the data is in a delineated format with a header e.g. csv or tsv + * this is the delimiter character used. This is only applicable if + * {@linkplain #getFormat()} is {@link DataDescription.DataFormat#DELIMITED}. + * The default value is {@value #DEFAULT_DELIMITER} + * + * @return A char + */ + public char getFieldDelimiter() { + return fieldDelimiter; + } + + /** + * The quote character used in delineated formats. + * Defaults to {@value #DEFAULT_QUOTE_CHAR} + * + * @return The delineated format quote character + */ + public char getQuoteCharacter() { + return quoteCharacter; + } + + /** + * Returns true if the data described by this object needs + * transforming before processing by autodetect. + * A transformation must be applied if either a timeformat is + * not in seconds since the epoch or the data is in Json format. + * + * @return True if the data should be transformed. + */ + public boolean transform() { + return dataFormat == DataFormat.JSON || + isTransformTime(); + } + + /** + * Return true if the time is in a format that needs transforming. + * Anytime format this isn't {@value #EPOCH} or null + * needs transforming. + * + * @return True if the time field needs to be transformed. + */ + public boolean isTransformTime() { + return timeFormat != null && !EPOCH.equals(timeFormat); + } + + /** + * Return true if the time format is {@value #EPOCH_MS} + * + * @return True if the date is in milli-seconds since the epoch. + */ + public boolean isEpochMs() { + return EPOCH_MS.equals(timeFormat); + } + + private static char extractChar(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + String charStr = parser.text(); + if (charStr.length() != 1) { + throw new IllegalArgumentException("String must be a single character, found [" + charStr + "]"); + } + return charStr.charAt(0); + } + throw new IllegalArgumentException("Unsupported token [" + parser.currentToken() + "]"); + } + + /** + * Overridden equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof DataDescription == false) { + return false; + } + + DataDescription that = (DataDescription) other; + + return this.dataFormat == that.dataFormat && + this.quoteCharacter == that.quoteCharacter && + Objects.equals(this.timeFieldName, that.timeFieldName) && + Objects.equals(this.timeFormat, that.timeFormat) && + Objects.equals(this.fieldDelimiter, that.fieldDelimiter); + } + + @Override + public int hashCode() { + return Objects.hash(dataFormat, quoteCharacter, timeFieldName, + timeFormat, fieldDelimiter); + } + + public static class Builder { + + private DataFormat dataFormat = DataFormat.DELIMITED; + private String timeFieldName = DEFAULT_TIME_FIELD; + private String timeFormat = EPOCH; + private char fieldDelimiter = DEFAULT_DELIMITER; + private char quoteCharacter = DEFAULT_QUOTE_CHAR; + + public void setFormat(DataFormat format) { + dataFormat = ExceptionsHelper.requireNonNull(format, FORMAT_FIELD.getPreferredName() + " must not be null"); + } + + private void setFormat(String format) { + setFormat(DataFormat.forString(format)); + } + + public void setTimeField(String fieldName) { + timeFieldName = ExceptionsHelper.requireNonNull(fieldName, TIME_FIELD_NAME_FIELD.getPreferredName() + " must not be null"); + } + + public void setTimeFormat(String format) { + ExceptionsHelper.requireNonNull(format, TIME_FORMAT_FIELD.getPreferredName() + " must not be null"); + switch (format) { + case EPOCH: + case EPOCH_MS: + break; + default: + try { + DateTimeFormatterTimestampConverter.ofPattern(format); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, format)); + } + } + timeFormat = format; + } + + public void setFieldDelimiter(char delimiter) { + fieldDelimiter = delimiter; + } + + public void setQuoteCharacter(char value) { + quoteCharacter = value; + } + + public DataDescription build() { + return new DataDescription(dataFormat, timeFieldName, timeFormat, fieldDelimiter,quoteCharacter); + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Detector.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Detector.java new file mode 100644 index 00000000000..68ed874a799 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Detector.java @@ -0,0 +1,780 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.config.DefaultDetectorDescription; +import org.elasticsearch.xpack.prelert.job.detectionrules.DetectionRule; +import org.elasticsearch.xpack.prelert.job.detectionrules.RuleCondition; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Defines the fields to be used in the analysis. + * fieldname must be set and only one of byFieldName + * and overFieldName should be set. + */ +public class Detector extends ToXContentToBytes implements Writeable { + + public enum ExcludeFrequent implements Writeable { + ALL("all"), + NONE("none"), + BY("by"), + OVER("over"); + + private final String token; + + ExcludeFrequent(String token) { + this.token = token; + } + + public String getToken() { + return token; + } + + /** + * Case-insensitive from string method. + * Works with either JSON, json, etc. + * + * @param value String representation + * @return The data format + */ + public static ExcludeFrequent forString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static ExcludeFrequent readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown ExcludeFrequent ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + } + + public static final ParseField DETECTOR_FIELD = new ParseField("detector"); + public static final ParseField DETECTOR_DESCRIPTION_FIELD = new ParseField("detectorDescription"); + public static final ParseField FUNCTION_FIELD = new ParseField("function"); + public static final ParseField FIELD_NAME_FIELD = new ParseField("fieldName"); + public static final ParseField BY_FIELD_NAME_FIELD = new ParseField("byFieldName"); + public static final ParseField OVER_FIELD_NAME_FIELD = new ParseField("overFieldName"); + public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partitionFieldName"); + public static final ParseField USE_NULL_FIELD = new ParseField("useNull"); + public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("excludeFrequent"); + public static final ParseField DETECTOR_RULES_FIELD = new ParseField("detectorRules"); + + public static final ObjectParser PARSER = new ObjectParser<>("detector", Builder::new); + + static { + PARSER.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD); + PARSER.declareString(Builder::setFunction, FUNCTION_FIELD); + PARSER.declareString(Builder::setFieldName, FIELD_NAME_FIELD); + PARSER.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD); + PARSER.declareBoolean(Builder::setUseNull, USE_NULL_FIELD); + PARSER.declareField(Builder::setExcludeFrequent, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return ExcludeFrequent.forString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareObjectArray(Builder::setDetectorRules, DetectionRule.PARSER, DETECTOR_RULES_FIELD); + } + + public static final String COUNT = "count"; + public static final String HIGH_COUNT = "high_count"; + public static final String LOW_COUNT = "low_count"; + public static final String NON_ZERO_COUNT = "non_zero_count"; + public static final String LOW_NON_ZERO_COUNT = "low_non_zero_count"; + public static final String HIGH_NON_ZERO_COUNT = "high_non_zero_count"; + public static final String NZC = "nzc"; + public static final String LOW_NZC = "low_nzc"; + public static final String HIGH_NZC = "high_nzc"; + public static final String DISTINCT_COUNT = "distinct_count"; + public static final String LOW_DISTINCT_COUNT = "low_distinct_count"; + public static final String HIGH_DISTINCT_COUNT = "high_distinct_count"; + public static final String DC = "dc"; + public static final String LOW_DC = "low_dc"; + public static final String HIGH_DC = "high_dc"; + public static final String RARE = "rare"; + public static final String FREQ_RARE = "freq_rare"; + public static final String INFO_CONTENT = "info_content"; + public static final String LOW_INFO_CONTENT = "low_info_content"; + public static final String HIGH_INFO_CONTENT = "high_info_content"; + public static final String METRIC = "metric"; + public static final String MEAN = "mean"; + public static final String MEDIAN = "median"; + public static final String HIGH_MEAN = "high_mean"; + public static final String LOW_MEAN = "low_mean"; + public static final String AVG = "avg"; + public static final String HIGH_AVG = "high_avg"; + public static final String LOW_AVG = "low_avg"; + public static final String MIN = "min"; + public static final String MAX = "max"; + public static final String SUM = "sum"; + public static final String LOW_SUM = "low_sum"; + public static final String HIGH_SUM = "high_sum"; + public static final String NON_NULL_SUM = "non_null_sum"; + public static final String LOW_NON_NULL_SUM = "low_non_null_sum"; + public static final String HIGH_NON_NULL_SUM = "high_non_null_sum"; + /** + * Population variance is called varp to match Splunk + */ + public static final String POPULATION_VARIANCE = "varp"; + public static final String LOW_POPULATION_VARIANCE = "low_varp"; + public static final String HIGH_POPULATION_VARIANCE = "high_varp"; + public static final String TIME_OF_DAY = "time_of_day"; + public static final String TIME_OF_WEEK = "time_of_week"; + public static final String LAT_LONG = "lat_long"; + + + /** + * The set of valid function names. + */ + public static final Set ANALYSIS_FUNCTIONS = + new HashSet<>(Arrays.asList( + // The convention here is that synonyms (only) go on the same line + COUNT, + HIGH_COUNT, + LOW_COUNT, + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC, + DISTINCT_COUNT, DC, + LOW_DISTINCT_COUNT, LOW_DC, + HIGH_DISTINCT_COUNT, HIGH_DC, + RARE, + FREQ_RARE, + INFO_CONTENT, + LOW_INFO_CONTENT, + HIGH_INFO_CONTENT, + METRIC, + MEAN, AVG, + HIGH_MEAN, HIGH_AVG, + LOW_MEAN, LOW_AVG, + MEDIAN, + MIN, + MAX, + SUM, + LOW_SUM, + HIGH_SUM, + NON_NULL_SUM, + LOW_NON_NULL_SUM, + HIGH_NON_NULL_SUM, + POPULATION_VARIANCE, + LOW_POPULATION_VARIANCE, + HIGH_POPULATION_VARIANCE, + TIME_OF_DAY, + TIME_OF_WEEK, + LAT_LONG + )); + + /** + * The set of functions that do not require a field, by field or over field + */ + public static final Set COUNT_WITHOUT_FIELD_FUNCTIONS = + new HashSet<>(Arrays.asList( + COUNT, + HIGH_COUNT, + LOW_COUNT, + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC, + TIME_OF_DAY, + TIME_OF_WEEK + )); + + /** + * The set of functions that require a fieldname + */ + public static final Set FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + DISTINCT_COUNT, DC, + LOW_DISTINCT_COUNT, LOW_DC, + HIGH_DISTINCT_COUNT, HIGH_DC, + INFO_CONTENT, + LOW_INFO_CONTENT, + HIGH_INFO_CONTENT, + METRIC, + MEAN, AVG, + HIGH_MEAN, HIGH_AVG, + LOW_MEAN, LOW_AVG, + MEDIAN, + MIN, + MAX, + SUM, + LOW_SUM, + HIGH_SUM, + NON_NULL_SUM, + LOW_NON_NULL_SUM, + HIGH_NON_NULL_SUM, + POPULATION_VARIANCE, + LOW_POPULATION_VARIANCE, + HIGH_POPULATION_VARIANCE, + LAT_LONG + )); + + /** + * The set of functions that require a by fieldname + */ + public static final Set BY_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + RARE, + FREQ_RARE + )); + + /** + * The set of functions that require a over fieldname + */ + public static final Set OVER_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + FREQ_RARE + )); + + /** + * The set of functions that cannot have a by fieldname + */ + public static final Set NO_BY_FIELD_NAME_FUNCTIONS = + new HashSet<>(); + + /** + * The set of functions that cannot have an over fieldname + */ + public static final Set NO_OVER_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC + )); + + /** + * The set of functions that must not be used with overlapping buckets + */ + public static final Set NO_OVERLAPPING_BUCKETS_FUNCTIONS = + new HashSet<>(Arrays.asList( + RARE, + FREQ_RARE + )); + + /** + * The set of functions that should not be used with overlapping buckets + * as they gain no benefit but have overhead + */ + public static final Set OVERLAPPING_BUCKETS_FUNCTIONS_NOT_NEEDED = + new HashSet<>(Arrays.asList( + MIN, + MAX, + TIME_OF_DAY, + TIME_OF_WEEK + )); + + /** + * field names cannot contain any of these characters + * ", \ + */ + public static final Character[] PROHIBITED_FIELDNAME_CHARACTERS = {'"', '\\'}; + public static final String PROHIBITED = String.join(",", + Arrays.stream(PROHIBITED_FIELDNAME_CHARACTERS).map( + c -> Character.toString(c)).collect(Collectors.toList())); + + + private final String detectorDescription; + private final String function; + private final String fieldName; + private final String byFieldName; + private final String overFieldName; + private final String partitionFieldName; + private final boolean useNull; + private final ExcludeFrequent excludeFrequent; + private final List detectorRules; + + public Detector(StreamInput in) throws IOException { + detectorDescription = in.readString(); + function = in.readString(); + fieldName = in.readOptionalString(); + byFieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + useNull = in.readBoolean(); + excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null; + detectorRules = in.readList(DetectionRule::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(detectorDescription); + out.writeString(function); + out.writeOptionalString(fieldName); + out.writeOptionalString(byFieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(partitionFieldName); + out.writeBoolean(useNull); + if (excludeFrequent != null) { + out.writeBoolean(true); + excludeFrequent.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeList(detectorRules); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DETECTOR_DESCRIPTION_FIELD.getPreferredName(), detectorDescription); + builder.field(FUNCTION_FIELD.getPreferredName(), function); + if (fieldName != null) { + builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName); + } + if (useNull) { + builder.field(USE_NULL_FIELD.getPreferredName(), useNull); + } + if (excludeFrequent != null) { + builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); + } + builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules); + builder.endObject(); + return builder; + } + + private Detector(String detectorDescription, String function, String fieldName, String byFieldName, String overFieldName, + String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List detectorRules) { + this.function = function; + this.fieldName = fieldName; + this.byFieldName = byFieldName; + this.overFieldName = overFieldName; + this.partitionFieldName = partitionFieldName; + this.useNull = useNull; + this.excludeFrequent = excludeFrequent; + // REMOVE THIS LINE WHEN REMOVING JACKSON_DATABIND: + detectorRules = detectorRules != null ? detectorRules : Collections.emptyList(); + this.detectorRules = Collections.unmodifiableList(detectorRules); + this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this); + } + + public String getDetectorDescription() { + return detectorDescription; + } + + /** + * The analysis function used e.g. count, rare, min etc. There is no + * validation to check this value is one a predefined set + * + * @return The function or null if not set + */ + public String getFunction() { + return function; + } + + /** + * The Analysis field + * + * @return The field to analyse + */ + public String getFieldName() { + return fieldName; + } + + /** + * The 'by' field or null if not set. + * + * @return The 'by' field + */ + public String getByFieldName() { + return byFieldName; + } + + /** + * The 'over' field or null if not set. + * + * @return The 'over' field + */ + public String getOverFieldName() { + return overFieldName; + } + + /** + * Segments the analysis along another field to have completely + * independent baselines for each instance of partitionfield + * + * @return The Partition Field + */ + public String getPartitionFieldName() { + return partitionFieldName; + } + + /** + * Where there isn't a value for the 'by' or 'over' field should a new + * series be used as the 'null' series. + * + * @return true if the 'null' series should be created + */ + public boolean isUseNull() { + return useNull; + } + + /** + * Excludes frequently-occuring metrics from the analysis; + * can apply to 'by' field, 'over' field, or both + * + * @return the value that the user set + */ + public ExcludeFrequent getExcludeFrequent() { + return excludeFrequent; + } + + public List getDetectorRules() { + return detectorRules; + } + + /** + * Returns a list with the byFieldName, overFieldName and partitionFieldName that are not null + * + * @return a list with the byFieldName, overFieldName and partitionFieldName that are not null + */ + public List extractAnalysisFields() { + List analysisFields = Arrays.asList(getByFieldName(), + getOverFieldName(), getPartitionFieldName()); + return analysisFields.stream().filter(item -> item != null).collect(Collectors.toList()); + } + + public Set extractReferencedLists() { + return detectorRules == null ? Collections.emptySet() + : detectorRules.stream().map(DetectionRule::extractReferencedLists) + .flatMap(Set::stream).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Detector == false) { + return false; + } + + Detector that = (Detector) other; + + return Objects.equals(this.detectorDescription, that.detectorDescription) && + Objects.equals(this.function, that.function) && + Objects.equals(this.fieldName, that.fieldName) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.useNull, that.useNull) && + Objects.equals(this.excludeFrequent, that.excludeFrequent) && + Objects.equals(this.detectorRules, that.detectorRules); + } + + @Override + public int hashCode() { + return Objects.hash(detectorDescription, function, fieldName, byFieldName, + overFieldName, partitionFieldName, useNull, excludeFrequent, + detectorRules); + } + + public static class Builder { + + /** + * Functions that do not support rules: + *

    + *
  • lat_long - because it is a multivariate feature + *
  • metric - because having the same conditions on min,max,mean is + * error-prone + *
+ */ + static final Set FUNCTIONS_WITHOUT_RULE_SUPPORT = new HashSet<>(Arrays.asList(Detector.LAT_LONG, Detector.METRIC)); + + private String detectorDescription; + private String function; + private String fieldName; + private String byFieldName; + private String overFieldName; + private String partitionFieldName; + private boolean useNull = false; + private ExcludeFrequent excludeFrequent; + private List detectorRules = Collections.emptyList(); + + public Builder() { + } + + public Builder(Detector detector) { + detectorDescription = detector.detectorDescription; + function = detector.function; + fieldName = detector.fieldName; + byFieldName = detector.byFieldName; + overFieldName = detector.overFieldName; + partitionFieldName = detector.partitionFieldName; + useNull = detector.useNull; + excludeFrequent = detector.excludeFrequent; + detectorRules = new ArrayList<>(detector.detectorRules.size()); + for (DetectionRule rule : detector.getDetectorRules()) { + detectorRules.add(rule); + } + } + + public Builder(String function, String fieldName) { + this.function = function; + this.fieldName = fieldName; + } + + public void setDetectorDescription(String detectorDescription) { + this.detectorDescription = detectorDescription; + } + + public void setFunction(String function) { + this.function = function; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public void setByFieldName(String byFieldName) { + this.byFieldName = byFieldName; + } + + public void setOverFieldName(String overFieldName) { + this.overFieldName = overFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public void setUseNull(boolean useNull) { + this.useNull = useNull; + } + + public void setExcludeFrequent(ExcludeFrequent excludeFrequent) { + this.excludeFrequent = excludeFrequent; + } + + public void setDetectorRules(List detectorRules) { + this.detectorRules = detectorRules; + } + + public List getDetectorRules() { + return detectorRules; + } + + public Detector build() { + return build(false); + } + + public Detector build(boolean isSummarised) { + boolean emptyField = Strings.isEmpty(fieldName); + boolean emptyByField = Strings.isEmpty(byFieldName); + boolean emptyOverField = Strings.isEmpty(overFieldName); + if (Detector.ANALYSIS_FUNCTIONS.contains(function) == false) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_UNKNOWN_FUNCTION, function)); + } + + if (emptyField && emptyByField && emptyOverField) { + if (!Detector.COUNT_WITHOUT_FIELD_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_NO_ANALYSIS_FIELD_NOT_COUNT)); + } + } + + if (isSummarised && Detector.METRIC.equals(function)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_INCOMPATIBLE_PRESUMMARIZED, Detector.METRIC)); + } + + // check functions have required fields + + if (emptyField && Detector.FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_FIELDNAME, function)); + } + + if (!emptyField && (Detector.FIELD_NAME_FUNCTIONS.contains(function) == false)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION, function)); + } + + if (emptyByField && Detector.BY_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_BYFIELD, function)); + } + + if (!emptyByField && Detector.NO_BY_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION, function)); + } + + if (emptyOverField && Detector.OVER_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_OVERFIELD, function)); + } + + if (!emptyOverField && Detector.NO_OVER_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION, function)); + } + + // field names cannot contain certain characters + String[] fields = { fieldName, byFieldName, overFieldName, partitionFieldName }; + for (String field : fields) { + verifyFieldName(field); + } + + String function = this.function == null ? Detector.METRIC : this.function; + if (detectorRules.isEmpty() == false) { + if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); + throw new IllegalArgumentException(msg); + } + for (DetectionRule rule : detectorRules) { + checkScoping(rule); + } + } + + return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, + useNull, excludeFrequent, detectorRules); + } + + public List extractAnalysisFields() { + List analysisFields = Arrays.asList(byFieldName, + overFieldName, partitionFieldName); + return analysisFields.stream().filter(item -> item != null).collect(Collectors.toList()); + } + + /** + * Check that the characters used in a field name will not cause problems. + * + * @param field + * The field name to be validated + * @return true + */ + public static boolean verifyFieldName(String field) throws ElasticsearchParseException { + if (field != null && containsInvalidChar(field)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_INVALID_FIELDNAME_CHARS, field, Detector.PROHIBITED)); + + } + return true; + } + + private static boolean containsInvalidChar(String field) { + for (Character ch : Detector.PROHIBITED_FIELDNAME_CHARACTERS) { + if (field.indexOf(ch) >= 0) { + return true; + } + } + return field.chars().anyMatch(ch -> Character.isISOControl(ch)); + } + + private void checkScoping(DetectionRule rule) throws ElasticsearchParseException { + String targetFieldName = rule.getTargetFieldName(); + checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName); + List validOptions = getValidFieldNameOptions(rule); + for (RuleCondition condition : rule.getRuleConditions()) { + if (!validOptions.contains(condition.getFieldName())) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions, + condition.getFieldName()); + throw new IllegalArgumentException(msg); + } + } + } + + private void checkTargetFieldNameIsValid(List analysisFields, String targetFieldName) + throws ElasticsearchParseException { + if (targetFieldName != null && !analysisFields.contains(targetFieldName)) { + String msg = + Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, analysisFields, targetFieldName); + throw new IllegalArgumentException(msg); + } + } + + private List getValidFieldNameOptions(DetectionRule rule) { + List result = new ArrayList<>(); + if (overFieldName != null) { + result.add(byFieldName == null ? overFieldName : byFieldName); + } else if (byFieldName != null) { + result.add(byFieldName); + } + + if (rule.getTargetFieldName() != null) { + ScopingLevel targetLevel = ScopingLevel.from(this, rule.getTargetFieldName()); + result = result.stream().filter(field -> targetLevel.isHigherThan(ScopingLevel.from(this, field))) + .collect(Collectors.toList()); + } + + if (isEmptyFieldNameAllowed(rule)) { + result.add(null); + } + return result; + } + + private boolean isEmptyFieldNameAllowed(DetectionRule rule) { + List analysisFields = extractAnalysisFields(); + return analysisFields.isEmpty() || (rule.getTargetFieldName() != null && analysisFields.size() == 1); + } + + enum ScopingLevel { + PARTITION(3), + OVER(2), + BY(1); + + int level; + + ScopingLevel(int level) { + this.level = level; + } + + boolean isHigherThan(ScopingLevel other) { + return level > other.level; + } + + static ScopingLevel from(Detector.Builder detector, String fieldName) { + if (fieldName.equals(detector.partitionFieldName)) { + return ScopingLevel.PARTITION; + } + if (fieldName.equals(detector.overFieldName)) { + return ScopingLevel.OVER; + } + if (fieldName.equals(detector.byFieldName)) { + return ScopingLevel.BY; + } + throw new IllegalArgumentException( + "fieldName '" + fieldName + "' does not match an analysis field"); + } + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntime.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntime.java new file mode 100644 index 00000000000..3cf11df6ff3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntime.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum IgnoreDowntime implements Writeable { + + NEVER, ONCE, ALWAYS; + + /** + *

+ * Parses a string and returns the corresponding enum value. + *

+ *

+ * The method differs from {@link #valueOf(String)} by being + * able to handle leading/trailing whitespace and being case + * insensitive. + *

+ *

+ * If there is no match {@link IllegalArgumentException} is thrown. + *

+ * + * @param value A String that should match one of the enum values + * @return the matching enum value + */ + public static IgnoreDowntime fromString(String value) { + return valueOf(value.trim().toUpperCase(Locale.ROOT)); + } + + public static IgnoreDowntime fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum JobSchedulerStatus {\n ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Job.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Job.java new file mode 100644 index 00000000000..f818ddb7808 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/Job.java @@ -0,0 +1,884 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; +import org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigsVerifier; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; + +/** + * This class represents a configured and created Job. The creation time is set + * to the time the object was constructed, Status is set to + * {@link JobStatus#RUNNING} and the finished time and last data time fields are + * {@code null} until the job has seen some data or it is finished respectively. + * If the job was created to read data from a list of files FileUrls will be a + * non-empty list else the expects data to be streamed to it. + */ +public class Job extends AbstractDiffable implements Writeable, ToXContent { + + public static final Job PROTO = new Job(null, null, null, null, null, 0L, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null); + + public static final long DEFAULT_BUCKETSPAN = 300; + + public static final String TYPE = "job"; + + /* + * Field names used in serialization + */ + public static final ParseField ID = new ParseField("jobId"); + public static final ParseField ANALYSIS_CONFIG = new ParseField("analysisConfig"); + public static final ParseField ANALYSIS_LIMITS = new ParseField("analysisLimits"); + public static final ParseField COUNTS = new ParseField("counts"); + public static final ParseField CREATE_TIME = new ParseField("createTime"); + public static final ParseField CUSTOM_SETTINGS = new ParseField("customSettings"); + public static final ParseField DATA_DESCRIPTION = new ParseField("dataDescription"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField FINISHED_TIME = new ParseField("finishedTime"); + public static final ParseField IGNORE_DOWNTIME = new ParseField("ignoreDowntime"); + public static final ParseField LAST_DATA_TIME = new ParseField("lastDataTime"); + public static final ParseField MODEL_DEBUG_CONFIG = new ParseField("modelDebugConfig"); + public static final ParseField SCHEDULER_CONFIG = new ParseField("schedulerConfig"); + public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalizationWindowDays"); + public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("backgroundPersistInterval"); + public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("modelSnapshotRetentionDays"); + public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("resultsRetentionDays"); + public static final ParseField TIMEOUT = new ParseField("timeout"); + public static final ParseField TRANSFORMS = new ParseField("transforms"); + public static final ParseField MODEL_SIZE_STATS = new ParseField("modelSizeStats"); + public static final ParseField AVERAGE_BUCKET_PROCESSING_TIME = new ParseField("averageBucketProcessingTimeMs"); + public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("modelSnapshotId"); + + public static final ObjectParser PARSER = new ObjectParser<>("job_details", Builder::new); + + static { + PARSER.declareString(Builder::setId, ID); + PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION); + PARSER.declareField(Builder::setCreateTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + CREATE_TIME.getPreferredName() + "]"); + }, CREATE_TIME, ValueType.VALUE); + PARSER.declareField(Builder::setFinishedTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + FINISHED_TIME.getPreferredName() + "]"); + }, FINISHED_TIME, ValueType.VALUE); + PARSER.declareField(Builder::setLastDataTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LAST_DATA_TIME.getPreferredName() + "]"); + }, LAST_DATA_TIME, ValueType.VALUE); + PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG); + PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS); + PARSER.declareObject(Builder::setSchedulerConfig, SchedulerConfig.PARSER, SCHEDULER_CONFIG); + PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION); + PARSER.declareObject(Builder::setModelSizeStats, ModelSizeStats.PARSER, MODEL_SIZE_STATS); + PARSER.declareObjectArray(Builder::setTransforms, TransformConfig.PARSER, TRANSFORMS); + PARSER.declareObject(Builder::setModelDebugConfig, ModelDebugConfig.PARSER, MODEL_DEBUG_CONFIG); + PARSER.declareObject(Builder::setCounts, DataCounts.PARSER, COUNTS); + PARSER.declareField(Builder::setIgnoreDowntime, (p, c) -> IgnoreDowntime.fromString(p.text()), IGNORE_DOWNTIME, ValueType.STRING); + PARSER.declareLong(Builder::setTimeout, TIMEOUT); + PARSER.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS); + PARSER.declareLong(Builder::setBackgroundPersistInterval, BACKGROUND_PERSIST_INTERVAL); + PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); + PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT); + PARSER.declareDouble(Builder::setAverageBucketProcessingTimeMs, AVERAGE_BUCKET_PROCESSING_TIME); + PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID); + } + + private final String jobId; + private final String description; + // NORELEASE: Use Jodatime instead + private final Date createTime; + private final Date finishedTime; + private final Date lastDataTime; + private final long timeout; + private final AnalysisConfig analysisConfig; + private final AnalysisLimits analysisLimits; + private final SchedulerConfig schedulerConfig; + private final DataDescription dataDescription; + private final ModelSizeStats modelSizeStats; + private final List transforms; + private final ModelDebugConfig modelDebugConfig; + private final DataCounts counts; + private final IgnoreDowntime ignoreDowntime; + private final Long renormalizationWindowDays; + private final Long backgroundPersistInterval; + private final Long modelSnapshotRetentionDays; + private final Long resultsRetentionDays; + private final Map customSettings; + private final Double averageBucketProcessingTimeMs; + private final String modelSnapshotId; + + public Job(String jobId, String description, Date createTime, Date finishedTime, Date lastDataTime, long timeout, + AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, SchedulerConfig schedulerConfig, + DataDescription dataDescription, ModelSizeStats modelSizeStats, List transforms, + ModelDebugConfig modelDebugConfig, DataCounts counts, IgnoreDowntime ignoreDowntime, Long renormalizationWindowDays, + Long backgroundPersistInterval, Long modelSnapshotRetentionDays, Long resultsRetentionDays, + Map customSettings, Double averageBucketProcessingTimeMs, String modelSnapshotId) { + this.jobId = jobId; + this.description = description; + this.createTime = createTime; + this.finishedTime = finishedTime; + this.lastDataTime = lastDataTime; + this.timeout = timeout; + this.analysisConfig = analysisConfig; + this.analysisLimits = analysisLimits; + this.schedulerConfig = schedulerConfig; + this.dataDescription = dataDescription; + this.modelSizeStats = modelSizeStats; + this.transforms = transforms; + this.modelDebugConfig = modelDebugConfig; + this.counts = counts; + this.ignoreDowntime = ignoreDowntime; + this.renormalizationWindowDays = renormalizationWindowDays; + this.backgroundPersistInterval = backgroundPersistInterval; + this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.resultsRetentionDays = resultsRetentionDays; + this.customSettings = customSettings; + this.averageBucketProcessingTimeMs = averageBucketProcessingTimeMs; + this.modelSnapshotId = modelSnapshotId; + } + + public Job(StreamInput in) throws IOException { + jobId = in.readString(); + description = in.readOptionalString(); + createTime = new Date(in.readVLong()); + finishedTime = in.readBoolean() ? new Date(in.readVLong()) : null; + lastDataTime = in.readBoolean() ? new Date(in.readVLong()) : null; + timeout = in.readVLong(); + analysisConfig = new AnalysisConfig(in); + analysisLimits = in.readOptionalWriteable(AnalysisLimits::new); + schedulerConfig = in.readOptionalWriteable(SchedulerConfig::new); + dataDescription = in.readOptionalWriteable(DataDescription::new); + modelSizeStats = in.readOptionalWriteable(ModelSizeStats::new); + transforms = in.readList(TransformConfig::new); + modelDebugConfig = in.readOptionalWriteable(ModelDebugConfig::new); + counts = in.readOptionalWriteable(DataCounts::new); + ignoreDowntime = in.readOptionalWriteable(IgnoreDowntime::fromStream); + renormalizationWindowDays = in.readOptionalLong(); + backgroundPersistInterval = in.readOptionalLong(); + modelSnapshotRetentionDays = in.readOptionalLong(); + resultsRetentionDays = in.readOptionalLong(); + customSettings = in.readMap(); + averageBucketProcessingTimeMs = in.readOptionalDouble(); + modelSnapshotId = in.readOptionalString(); + } + + @Override + public Job readFrom(StreamInput in) throws IOException { + return new Job(in); + } + + /** + * Return the Job Id. This name is preferred when serialising to the REST + * API. + * + * @return The job Id string + */ + public String getId() { + return jobId; + } + + /** + * Return the Job Id. This name is preferred when serialising to the data + * store. + * + * @return The job Id string + */ + public String getJobId() { + return jobId; + } + + /** + * The job description + * + * @return job description + */ + public String getDescription() { + return description; + } + + /** + * The Job creation time. This name is preferred when serialising to the + * REST API. + * + * @return The date the job was created + */ + public Date getCreateTime() { + return createTime; + } + + /** + * The Job creation time. This name is preferred when serialising to the + * data store. + * + * @return The date the job was created + */ + public Date getAtTimestamp() { + return createTime; + } + + /** + * The time the job was finished or null if not finished. + * + * @return The date the job was last retired or null + */ + public Date getFinishedTime() { + return finishedTime; + } + + /** + * The last time data was uploaded to the job or null if no + * data has been seen. + * + * @return The date at which the last data was processed + */ + public Date getLastDataTime() { + return lastDataTime; + } + + /** + * The job timeout setting in seconds. Jobs are retired if they do not + * receive data for this period of time. The default is 600 seconds + * + * @return The timeout period in seconds + */ + public long getTimeout() { + return timeout; + } + + /** + * The analysis configuration object + * + * @return The AnalysisConfig + */ + public AnalysisConfig getAnalysisConfig() { + return analysisConfig; + } + + /** + * The analysis options object + * + * @return The AnalysisLimits + */ + public AnalysisLimits getAnalysisLimits() { + return analysisLimits; + } + + public IgnoreDowntime getIgnoreDowntime() { + return ignoreDowntime; + } + + public SchedulerConfig getSchedulerConfig() { + return schedulerConfig; + } + + public ModelDebugConfig getModelDebugConfig() { + return modelDebugConfig; + } + + /** + * The memory usage object + * + * @return The ModelSizeStats + */ + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + /** + * If not set the input data is assumed to be csv with a '_time' field in + * epoch format. + * + * @return A DataDescription or null + * @see DataDescription + */ + public DataDescription getDataDescription() { + return dataDescription; + } + + public List getTransforms() { + return transforms; + } + + /** + * Processed records count + * + * @return the processed records counts + */ + public DataCounts getCounts() { + return counts; + } + + /** + * The duration of the renormalization window in days + * + * @return renormalization window in days + */ + public Long getRenormalizationWindowDays() { + return renormalizationWindowDays; + } + + /** + * The background persistence interval in seconds + * + * @return background persistence interval in seconds + */ + public Long getBackgroundPersistInterval() { + return backgroundPersistInterval; + } + + public Long getModelSnapshotRetentionDays() { + return modelSnapshotRetentionDays; + } + + public Long getResultsRetentionDays() { + return resultsRetentionDays; + } + + public Map getCustomSettings() { + return customSettings; + } + + public Double getAverageBucketProcessingTimeMs() { + return averageBucketProcessingTimeMs; + } + + public String getModelSnapshotId() { + return modelSnapshotId; + } + + /** + * Get a list of all input data fields mentioned in the job configuration, + * namely analysis fields, time field and transform input fields. + * + * @return the list of fields - never null + */ + public List allFields() { + Set allFields = new TreeSet<>(); + + // analysis fields + if (analysisConfig != null) { + allFields.addAll(analysisConfig.analysisFields()); + } + + // transform input fields + if (transforms != null) { + for (TransformConfig tc : transforms) { + List inputFields = tc.getInputs(); + if (inputFields != null) { + allFields.addAll(inputFields); + } + } + } + + // time field + if (dataDescription != null) { + String timeField = dataDescription.getTimeField(); + if (timeField != null) { + allFields.add(timeField); + } + } + + // remove empty strings + allFields.remove(""); + + return new ArrayList<>(allFields); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(description); + out.writeVLong(createTime.getTime()); + if (finishedTime != null) { + out.writeBoolean(true); + out.writeVLong(finishedTime.getTime()); + } else { + out.writeBoolean(false); + } + if (lastDataTime != null) { + out.writeBoolean(true); + out.writeVLong(lastDataTime.getTime()); + } else { + out.writeBoolean(false); + } + out.writeVLong(timeout); + analysisConfig.writeTo(out); + out.writeOptionalWriteable(analysisLimits); + out.writeOptionalWriteable(schedulerConfig); + out.writeOptionalWriteable(dataDescription); + out.writeOptionalWriteable(modelSizeStats); + out.writeList(transforms); + out.writeOptionalWriteable(modelDebugConfig); + out.writeOptionalWriteable(counts); + out.writeOptionalWriteable(ignoreDowntime); + out.writeOptionalLong(renormalizationWindowDays); + out.writeOptionalLong(backgroundPersistInterval); + out.writeOptionalLong(modelSnapshotRetentionDays); + out.writeOptionalLong(resultsRetentionDays); + out.writeMap(customSettings); + out.writeOptionalDouble(averageBucketProcessingTimeMs); + out.writeOptionalString(modelSnapshotId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(ID.getPreferredName(), jobId); + builder.field(DESCRIPTION.getPreferredName(), description); + builder.field(CREATE_TIME.getPreferredName(), createTime.getTime()); + if (finishedTime != null) { + builder.field(FINISHED_TIME.getPreferredName(), finishedTime.getTime()); + } + if (lastDataTime != null) { + builder.field(LAST_DATA_TIME.getPreferredName(), lastDataTime.getTime()); + } + builder.field(TIMEOUT.getPreferredName(), timeout); + builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); + if (analysisLimits != null) { + builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); + } + if (schedulerConfig != null) { + builder.field(SCHEDULER_CONFIG.getPreferredName(), schedulerConfig, params); + } + if (dataDescription != null) { + builder.field(DATA_DESCRIPTION.getPreferredName(), dataDescription, params); + } + if (modelSizeStats != null) { + builder.field(MODEL_SIZE_STATS.getPreferredName(), modelSizeStats, params); + } + if (transforms != null) { + builder.field(TRANSFORMS.getPreferredName(), transforms); + } + if (modelDebugConfig != null) { + builder.field(MODEL_DEBUG_CONFIG.getPreferredName(), modelDebugConfig, params); + } + if (counts != null) { + builder.field(COUNTS.getPreferredName(), counts, params); + } + if (ignoreDowntime != null) { + builder.field(IGNORE_DOWNTIME.getPreferredName(), ignoreDowntime); + } + if (renormalizationWindowDays != null) { + builder.field(RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); + } + if (backgroundPersistInterval != null) { + builder.field(BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval); + } + if (modelSnapshotRetentionDays != null) { + builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); + } + if (resultsRetentionDays != null) { + builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + if (averageBucketProcessingTimeMs != null) { + builder.field(AVERAGE_BUCKET_PROCESSING_TIME.getPreferredName(), averageBucketProcessingTimeMs); + } + if (modelSnapshotId != null){ + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Job == false) { + return false; + } + + Job that = (Job) other; + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.description, that.description) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.finishedTime, that.finishedTime) && Objects.equals(this.lastDataTime, that.lastDataTime) + && (this.timeout == that.timeout) && Objects.equals(this.analysisConfig, that.analysisConfig) + && Objects.equals(this.analysisLimits, that.analysisLimits) && Objects.equals(this.dataDescription, that.dataDescription) + && Objects.equals(this.modelDebugConfig, that.modelDebugConfig) && Objects.equals(this.modelSizeStats, that.modelSizeStats) + && Objects.equals(this.transforms, that.transforms) && Objects.equals(this.counts, that.counts) + && Objects.equals(this.ignoreDowntime, that.ignoreDowntime) + && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) + && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) + && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) + && Objects.equals(this.customSettings, that.customSettings) + && Objects.equals(this.modelSnapshotId, that.modelSnapshotId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, description, createTime, finishedTime, lastDataTime, timeout, analysisConfig, + analysisLimits, dataDescription, modelDebugConfig, modelSizeStats, transforms, counts, renormalizationWindowDays, + backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, ignoreDowntime, customSettings, + modelSnapshotId); + } + + // Class alreadt extends from AbstractDiffable, so copied from ToXContentToBytes#toString() + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + + public static class Builder { + + /** + * Valid jobId characters. Note that '.' is allowed but not documented. + */ + private static final Pattern VALID_JOB_ID_CHAR_PATTERN = Pattern.compile("[a-z0-9_\\-\\.]+"); + public static final int MAX_JOB_ID_LENGTH = 64; + public static final long MIN_BACKGROUND_PERSIST_INTERVAL = 3600; + public static final long DEFAULT_TIMEOUT = 600; + + private String id; + private String description; + + private AnalysisConfig analysisConfig; + private AnalysisLimits analysisLimits; + private SchedulerConfig schedulerConfig; + private List transforms = new ArrayList<>(); + private ModelSizeStats modelSizeStats; + private DataDescription dataDescription; + private Date createTime; + private Date finishedTime; + private Date lastDataTime; + private Long timeout = DEFAULT_TIMEOUT; + private ModelDebugConfig modelDebugConfig; + private Long renormalizationWindowDays; + private Long backgroundPersistInterval; + private Long modelSnapshotRetentionDays; + private Long resultsRetentionDays; + private DataCounts counts; + private IgnoreDowntime ignoreDowntime; + private Map customSettings; + private Double averageBucketProcessingTimeMs; + private String modelSnapshotId; + + public Builder() { + } + + public Builder(String id) { + this.id = id; + } + + public Builder(Job job) { + this.id = job.getId(); + this.description = job.getDescription(); + this.analysisConfig = job.getAnalysisConfig(); + this.schedulerConfig = job.getSchedulerConfig(); + this.transforms = job.getTransforms(); + this.modelSizeStats = job.getModelSizeStats(); + this.dataDescription = job.getDataDescription(); + this.createTime = job.getCreateTime(); + this.finishedTime = job.getFinishedTime(); + this.lastDataTime = job.getLastDataTime(); + this.timeout = job.getTimeout(); + this.modelDebugConfig = job.getModelDebugConfig(); + this.renormalizationWindowDays = job.getRenormalizationWindowDays(); + this.backgroundPersistInterval = job.getBackgroundPersistInterval(); + this.resultsRetentionDays = job.getResultsRetentionDays(); + this.counts = job.getCounts(); + this.ignoreDowntime = job.getIgnoreDowntime(); + this.customSettings = job.getCustomSettings(); + this.averageBucketProcessingTimeMs = job.getAverageBucketProcessingTimeMs(); + this.modelSnapshotId = job.getModelSnapshotId(); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setCustomSettings(Map customSettings) { + this.customSettings = customSettings; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setAnalysisConfig(AnalysisConfig.Builder configBuilder) { + analysisConfig = configBuilder.build(); + } + + public void setAnalysisLimits(AnalysisLimits analysisLimits) { + if (this.analysisLimits != null) { + long oldMemoryLimit = this.analysisLimits.getModelMemoryLimit(); + long newMemoryLimit = analysisLimits.getModelMemoryLimit(); + if (newMemoryLimit < oldMemoryLimit) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED, + oldMemoryLimit, newMemoryLimit)); + } + } + this.analysisLimits = analysisLimits; + } + + public void setSchedulerConfig(SchedulerConfig.Builder config) { + schedulerConfig = config.build(); + } + + public void setTimeout(Long timeout) { + this.timeout = timeout; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public void setFinishedTime(Date finishedTime) { + this.finishedTime = finishedTime; + } + + public void setLastDataTime(Date lastDataTime) { + this.lastDataTime = lastDataTime; + } + + public void setTransforms(List transforms) { + this.transforms = transforms; + } + + public void setModelSizeStats(ModelSizeStats.Builder modelSizeStats) { + this.modelSizeStats = modelSizeStats.build(); + } + + public void setDataDescription(DataDescription.Builder description) { + dataDescription = description.build(); + } + + public void setModelDebugConfig(ModelDebugConfig modelDebugConfig) { + this.modelDebugConfig = modelDebugConfig; + } + + public void setBackgroundPersistInterval(Long backgroundPersistInterval) { + this.backgroundPersistInterval = backgroundPersistInterval; + } + + public void setRenormalizationWindowDays(Long renormalizationWindowDays) { + this.renormalizationWindowDays = renormalizationWindowDays; + } + + public void setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { + this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + } + + public void setResultsRetentionDays(Long resultsRetentionDays) { + this.resultsRetentionDays = resultsRetentionDays; + } + + public void setIgnoreDowntime(IgnoreDowntime ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + } + + public void setCounts(DataCounts counts) { + this.counts = counts; + } + + public void setAverageBucketProcessingTimeMs(Double averageBucketProcessingTimeMs) { + this.averageBucketProcessingTimeMs = averageBucketProcessingTimeMs; + } + + public void setModelSnapshotId(String modelSnapshotId) { + this.modelSnapshotId = modelSnapshotId; + } + + public Job build() { + return build(false); + } + + public Job build(boolean fromApi) { + if (analysisConfig == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_MISSING_ANALYSISCONFIG)); + } + + if (schedulerConfig != null) { + if (analysisConfig.getBucketSpan() == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_REQUIRES_BUCKET_SPAN)); + } + if (schedulerConfig.getDataSource() == SchedulerConfig.DataSource.ELASTICSEARCH) { + if (analysisConfig.getLatency() != null && analysisConfig.getLatency() > 0) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_ELASTICSEARCH_DOES_NOT_SUPPORT_LATENCY)); + } + if (schedulerConfig.getAggregationsOrAggs() != null + && !SchedulerConfig.DOC_COUNT.equals(analysisConfig.getSummaryCountFieldName())) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_AGGREGATIONS_REQUIRES_SUMMARY_COUNT_FIELD, + SchedulerConfig.DataSource.ELASTICSEARCH.toString(), SchedulerConfig.DOC_COUNT)); + } + if (dataDescription == null || dataDescription.getFormat() != DataDescription.DataFormat.ELASTICSEARCH) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_ELASTICSEARCH_REQUIRES_DATAFORMAT_ELASTICSEARCH)); + } + } + } + if (transforms != null && transforms.isEmpty() == false) { + TransformConfigsVerifier.verify(transforms); + checkTransformOutputIsUsed(); + } else { + if (dataDescription != null && dataDescription.getFormat() == DataDescription.DataFormat.SINGLE_LINE) { + String msg = Messages.getMessage( + Messages.JOB_CONFIG_DATAFORMAT_REQUIRES_TRANSFORM, + DataDescription.DataFormat.SINGLE_LINE); + + throw new IllegalArgumentException(msg); + } + } + + checkValueNotLessThan(0, "timeout", timeout); + checkValueNotLessThan(0, "renormalizationWindowDays", renormalizationWindowDays); + checkValueNotLessThan(MIN_BACKGROUND_PERSIST_INTERVAL, "backgroundPersistInterval", backgroundPersistInterval); + checkValueNotLessThan(0, "modelSnapshotRetentionDays", modelSnapshotRetentionDays); + checkValueNotLessThan(0, "resultsRetentionDays", resultsRetentionDays); + + String id; + Date createTime; + Date finishedTime; + Date lastDataTime; + DataCounts counts; + ModelSizeStats modelSizeStats; + Double averageBucketProcessingTimeMs; + String modelSnapshotId; + if (fromApi) { + id = this.id == null ? UUIDs.base64UUID().toLowerCase(Locale.ROOT): this.id; + createTime = this.createTime == null ? new Date() : this.createTime; + finishedTime = null; + lastDataTime = null; + counts = new DataCounts(id); + modelSizeStats = null; + averageBucketProcessingTimeMs = null; + modelSnapshotId = null; + } else { + id = this.id; + createTime = this.createTime; + finishedTime = this.finishedTime; + lastDataTime = this.lastDataTime; + counts = this.counts; + modelSizeStats = this.modelSizeStats; + averageBucketProcessingTimeMs = this.averageBucketProcessingTimeMs; + modelSnapshotId = this.modelSnapshotId; + } + if (id.length() > MAX_JOB_ID_LENGTH) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_ID_TOO_LONG, MAX_JOB_ID_LENGTH)); + } + if (!VALID_JOB_ID_CHAR_PATTERN.matcher(id).matches()) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_INVALID_JOBID_CHARS)); + } + return new Job( + id, description, createTime, finishedTime, lastDataTime, timeout, analysisConfig, analysisLimits, + schedulerConfig, dataDescription, modelSizeStats, transforms, modelDebugConfig, counts, + ignoreDowntime, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, + resultsRetentionDays, customSettings, averageBucketProcessingTimeMs, modelSnapshotId + ); + } + + private static void checkValueNotLessThan(long minVal, String name, Long value) { + if (value != null && value < minVal) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, name, minVal, value)); + } + } + + /** + * Transform outputs should be used in either the date field, + * as an analysis field or input to another transform + */ + private boolean checkTransformOutputIsUsed() { + Set usedFields = new TransformConfigs(transforms).inputFieldNames(); + usedFields.addAll(analysisConfig.analysisFields()); + String summaryCountFieldName = analysisConfig.getSummaryCountFieldName(); + boolean isSummarised = !Strings.isNullOrEmpty(summaryCountFieldName); + if (isSummarised) { + usedFields.remove(summaryCountFieldName); + } + + String timeField = dataDescription == null ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField(); + usedFields.add(timeField); + + for (TransformConfig tc : transforms) { + // if the type has no default outputs it doesn't need an output + boolean usesAnOutput = tc.type().defaultOutputNames().isEmpty() + || tc.getOutputs().stream().anyMatch(outputName -> usedFields.contains(outputName)); + + if (isSummarised && tc.getOutputs().contains(summaryCountFieldName)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_DUPLICATED_OUTPUT_NAME, tc.type().prettyName()); + throw new IllegalArgumentException(msg); + } + + if (!usesAnOutput) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_OUTPUTS_UNUSED, + tc.type().prettyName()); + throw new IllegalArgumentException(msg); + } + } + + return false; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatus.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatus.java new file mode 100644 index 00000000000..947f2a81c99 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatus.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum JobSchedulerStatus implements Writeable { + + STARTING, STARTED, STOPPING, STOPPED; + + public static JobSchedulerStatus fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static JobSchedulerStatus fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum JobSchedulerStatus {\n ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobStatus.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobStatus.java new file mode 100644 index 00000000000..0de9c984e58 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/JobStatus.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; + +/** + * Jobs whether running or complete are in one of these states. + * When a job is created it is initialised in to the status closed + * i.e. it is not running. + */ +public enum JobStatus implements Writeable { + + RUNNING, CLOSING, CLOSED, FAILED, PAUSING, PAUSED; + + public static JobStatus fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static JobStatus fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum JobStatus {\n ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + /** + * @return {@code true} if status matches any of the given {@code candidates} + */ + public boolean isAnyOf(JobStatus... candidates) { + return Arrays.stream(candidates).anyMatch(candidate -> this == candidate); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfig.java new file mode 100644 index 00000000000..36deb981cdd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfig.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class ModelDebugConfig extends ToXContentToBytes implements Writeable { + /** + * Enum of the acceptable output destinations. + */ + public enum DebugDestination implements Writeable { + FILE("file"), + DATA_STORE("data_store"); + + private String name; + + DebugDestination(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Case-insensitive from string method. Works with FILE, File, file, + * etc. + * + * @param value + * String representation + * @return The output destination + */ + public static DebugDestination forString(String value) { + String valueUpperCase = value.toUpperCase(Locale.ROOT); + return DebugDestination.valueOf(valueUpperCase); + } + + public static DebugDestination readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown DebugDestination ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + } + + private static final double MAX_PERCENTILE = 100.0; + + private static final ParseField TYPE_FIELD = new ParseField("modelDebugConfig"); + private static final ParseField WRITE_TO_FIELD = new ParseField("writeTo"); + private static final ParseField BOUNDS_PERCENTILE_FIELD = new ParseField("boundsPercentile"); + private static final ParseField TERMS_FIELD = new ParseField("terms"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE_FIELD.getPreferredName(), a -> { + if (a[0] == null) { + return new ModelDebugConfig((Double) a[1], (String) a[2]); + } else { + return new ModelDebugConfig((DebugDestination) a[0], (Double) a[1], (String) a[2]); + } + }); + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> DebugDestination.forString(p.text()), WRITE_TO_FIELD, + ValueType.STRING); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), BOUNDS_PERCENTILE_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD); + } + + private final DebugDestination writeTo; + private final double boundsPercentile; + private final String terms; + + public ModelDebugConfig(double boundsPercentile, String terms) { + this(DebugDestination.FILE, boundsPercentile, terms); + } + + public ModelDebugConfig(DebugDestination writeTo, double boundsPercentile, String terms) { + if (boundsPercentile < 0.0 || boundsPercentile > MAX_PERCENTILE) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE); + throw new IllegalArgumentException(msg); + } + this.writeTo = writeTo; + this.boundsPercentile = boundsPercentile; + this.terms = terms; + } + + public ModelDebugConfig(StreamInput in) throws IOException { + writeTo = in.readOptionalWriteable(DebugDestination::readFromStream); + boundsPercentile = in.readDouble(); + terms = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalWriteable(writeTo); + out.writeDouble(boundsPercentile); + out.writeOptionalString(terms); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (writeTo != null) { + builder.field(WRITE_TO_FIELD.getPreferredName(), writeTo.getName()); + } + builder.field(BOUNDS_PERCENTILE_FIELD.getPreferredName(), boundsPercentile); + if (terms != null) { + builder.field(TERMS_FIELD.getPreferredName(), terms); + } + builder.endObject(); + return builder; + } + + public DebugDestination getWriteTo() { + return this.writeTo; + } + + public double getBoundsPercentile() { + return this.boundsPercentile; + } + + public String getTerms() { + return this.terms; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelDebugConfig == false) { + return false; + } + + ModelDebugConfig that = (ModelDebugConfig) other; + return Objects.equals(this.writeTo, that.writeTo) && Objects.equals(this.boundsPercentile, that.boundsPercentile) + && Objects.equals(this.terms, that.terms); + } + + @Override + public int hashCode() { + return Objects.hash(this.writeTo, boundsPercentile, terms); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSizeStats.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSizeStats.java new file mode 100644 index 00000000000..d69d0ed27c9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSizeStats.java @@ -0,0 +1,334 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Provide access to the C++ model memory usage numbers for the Java process. + */ +public class ModelSizeStats extends ToXContentToBytes implements Writeable { + + /** + * Field Names + */ + private static final ParseField MODEL_SIZE_STATS_FIELD = new ParseField("modelSizeStats"); + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField MODEL_BYTES_FIELD = new ParseField("modelBytes"); + public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("totalByFieldCount"); + public static final ParseField TOTAL_OVER_FIELD_COUNT_FIELD = new ParseField("totalOverFieldCount"); + public static final ParseField TOTAL_PARTITION_FIELD_COUNT_FIELD = new ParseField("totalPartitionFieldCount"); + public static final ParseField BUCKET_ALLOCATION_FAILURES_COUNT_FIELD = new ParseField("bucketAllocationFailuresCount"); + public static final ParseField MEMORY_STATUS_FIELD = new ParseField("memoryStatus"); + public static final ParseField LOG_TIME_FIELD = new ParseField("logTime"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + MODEL_SIZE_STATS_FIELD.getPreferredName(), a -> new Builder((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD); + PARSER.declareField(Builder::setLogTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LOG_TIME_FIELD.getPreferredName() + "]"); + }, LOG_TIME_FIELD, ValueType.VALUE); + PARSER.declareField(Builder::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP_FIELD.getPreferredName() + "]"); + }, TIMESTAMP_FIELD, ValueType.VALUE); + PARSER.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING); + } + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("modelSizeStats"); + + /** + * The status of the memory monitored by the ResourceMonitor. OK is default, + * SOFT_LIMIT means that the models have done some aggressive pruning to + * keep the memory below the limit, and HARD_LIMIT means that samples have + * been dropped + */ + public enum MemoryStatus implements Writeable { + OK("ok"), SOFT_LIMIT("soft_limit"), HARD_LIMIT("hard_limit"); + + private String name; + + private MemoryStatus(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static MemoryStatus fromString(String statusName) { + for (MemoryStatus status : values()) { + if (status.name.equals(statusName)) { + return status; + } + } + throw new IllegalArgumentException("Unknown MemoryStatus [" + statusName + "]"); + } + + public static MemoryStatus readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown MemoryStatus ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + } + + private final String jobId; + private final String id; + private final long modelBytes; + private final long totalByFieldCount; + private final long totalOverFieldCount; + private final long totalPartitionFieldCount; + private final long bucketAllocationFailuresCount; + private final MemoryStatus memoryStatus; + private final Date timestamp; + private final Date logTime; + + private ModelSizeStats(String jobId, String id, long modelBytes, long totalByFieldCount, long totalOverFieldCount, + long totalPartitionFieldCount, long bucketAllocationFailuresCount, MemoryStatus memoryStatus, + Date timestamp, Date logTime) { + this.jobId = jobId; + this.id = id; + this.modelBytes = modelBytes; + this.totalByFieldCount = totalByFieldCount; + this.totalOverFieldCount = totalOverFieldCount; + this.totalPartitionFieldCount = totalPartitionFieldCount; + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + this.memoryStatus = memoryStatus; + this.timestamp = timestamp; + this.logTime = logTime; + } + + public ModelSizeStats(StreamInput in) throws IOException { + jobId = in.readString(); + id = null; + modelBytes = in.readVLong(); + totalByFieldCount = in.readVLong(); + totalOverFieldCount = in.readVLong(); + totalPartitionFieldCount = in.readVLong(); + bucketAllocationFailuresCount = in.readVLong(); + memoryStatus = MemoryStatus.readFromStream(in); + logTime = new Date(in.readLong()); + timestamp = in.readBoolean() ? new Date(in.readLong()) : null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeVLong(modelBytes); + out.writeVLong(totalByFieldCount); + out.writeVLong(totalOverFieldCount); + out.writeVLong(totalPartitionFieldCount); + out.writeVLong(bucketAllocationFailuresCount); + memoryStatus.writeTo(out); + out.writeLong(logTime.getTime()); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder) throws IOException { + builder.field(JOB_ID.getPreferredName(), jobId); + builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes); + builder.field(TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), totalByFieldCount); + builder.field(TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), totalOverFieldCount); + builder.field(TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), totalPartitionFieldCount); + builder.field(BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), bucketAllocationFailuresCount); + builder.field(MEMORY_STATUS_FIELD.getPreferredName(), memoryStatus.getName()); + builder.field(LOG_TIME_FIELD.getPreferredName(), logTime.getTime()); + if (timestamp != null) { + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestamp.getTime()); + } + + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return id; + } + + public long getModelBytes() { + return modelBytes; + } + + public long getTotalByFieldCount() { + return totalByFieldCount; + } + + public long getTotalPartitionFieldCount() { + return totalPartitionFieldCount; + } + + public long getTotalOverFieldCount() { + return totalOverFieldCount; + } + + public long getBucketAllocationFailuresCount() { + return bucketAllocationFailuresCount; + } + + public MemoryStatus getMemoryStatus() { + return memoryStatus; + } + + public Date getTimestamp() { + return timestamp; + } + + public Date getLogTime() { + return logTime; + } + + @Override + public int hashCode() { + // this.id excluded here as it is generated by the datastore + return Objects.hash(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + this.bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelSizeStats == false) { + return false; + } + + ModelSizeStats that = (ModelSizeStats) other; + + return this.modelBytes == that.modelBytes && this.totalByFieldCount == that.totalByFieldCount + && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount + && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount + && Objects.equals(this.memoryStatus, that.memoryStatus) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.logTime, that.logTime) + && Objects.equals(this.jobId, that.jobId); + } + + // NORELEASE This will not be needed once we are able to parse ModelSizeStats all at once. + public static class Builder { + + private final String jobId; + private String id; + private long modelBytes; + private long totalByFieldCount; + private long totalOverFieldCount; + private long totalPartitionFieldCount; + private long bucketAllocationFailuresCount; + private MemoryStatus memoryStatus; + private Date timestamp; + private Date logTime; + + public Builder(String jobId) { + this.jobId = jobId; + id = TYPE.getPreferredName(); + memoryStatus = MemoryStatus.OK; + logTime = new Date(); + } + + public void setId(String id) { + this.id = Objects.requireNonNull(id); + } + + public void setModelBytes(long modelBytes) { + this.modelBytes = modelBytes; + } + + public void setTotalByFieldCount(long totalByFieldCount) { + this.totalByFieldCount = totalByFieldCount; + } + + public void setTotalPartitionFieldCount(long totalPartitionFieldCount) { + this.totalPartitionFieldCount = totalPartitionFieldCount; + } + + public void setTotalOverFieldCount(long totalOverFieldCount) { + this.totalOverFieldCount = totalOverFieldCount; + } + + public void setBucketAllocationFailuresCount(long bucketAllocationFailuresCount) { + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + } + + public void setMemoryStatus(MemoryStatus memoryStatus) { + Objects.requireNonNull(memoryStatus, "[" + MEMORY_STATUS_FIELD.getPreferredName() + "] must not be null"); + this.memoryStatus = memoryStatus; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public void setLogTime(Date logTime) { + this.logTime = logTime; + } + + public ModelSizeStats build() { + return new ModelSizeStats(jobId, id, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSnapshot.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSnapshot.java new file mode 100644 index 00000000000..f875f7ddcd5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelSnapshot.java @@ -0,0 +1,297 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + + +/** + * ModelSnapshot Result POJO + */ +public class ModelSnapshot extends ToXContentToBytes implements Writeable { + /** + * Field Names + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField RESTORE_PRIORITY = new ParseField("restorePriority"); + public static final ParseField SNAPSHOT_ID = new ParseField("snapshotId"); + public static final ParseField SNAPSHOT_DOC_COUNT = new ParseField("snapshotDocCount"); + public static final ParseField LATEST_RECORD_TIME = new ParseField("latestRecordTimeStamp"); + public static final ParseField LATEST_RESULT_TIME = new ParseField("latestResultTimeStamp"); + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("modelSnapshot"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new ModelSnapshot((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareField(ModelSnapshot::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareString(ModelSnapshot::setDescription, DESCRIPTION); + PARSER.declareLong(ModelSnapshot::setRestorePriority, RESTORE_PRIORITY); + PARSER.declareString(ModelSnapshot::setSnapshotId, SNAPSHOT_ID); + PARSER.declareInt(ModelSnapshot::setSnapshotDocCount, SNAPSHOT_DOC_COUNT); + PARSER.declareObject(ModelSnapshot::setModelSizeStats, ModelSizeStats.PARSER, ModelSizeStats.TYPE); + PARSER.declareField(ModelSnapshot::setLatestRecordTimeStamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RECORD_TIME.getPreferredName() + "]"); + }, LATEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareField(ModelSnapshot::setLatestResultTimeStamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RESULT_TIME.getPreferredName() + "]"); + }, LATEST_RESULT_TIME, ValueType.VALUE); + PARSER.declareObject(ModelSnapshot::setQuantiles, Quantiles.PARSER, Quantiles.TYPE); + } + + private final String jobId; + private Date timestamp; + private String description; + private long restorePriority; + private String snapshotId; + private int snapshotDocCount; + private ModelSizeStats modelSizeStats; + private Date latestRecordTimeStamp; + private Date latestResultTimeStamp; + private Quantiles quantiles; + + public ModelSnapshot(String jobId) { + this.jobId = jobId; + } + + public ModelSnapshot(StreamInput in) throws IOException { + jobId = in.readString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + description = in.readOptionalString(); + restorePriority = in.readLong(); + snapshotId = in.readOptionalString(); + snapshotDocCount = in.readInt(); + if (in.readBoolean()) { + modelSizeStats = new ModelSizeStats(in); + } + if (in.readBoolean()) { + latestRecordTimeStamp = new Date(in.readLong()); + } + if (in.readBoolean()) { + latestResultTimeStamp = new Date(in.readLong()); + } + if (in.readBoolean()) { + quantiles = new Quantiles(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeOptionalString(description); + out.writeLong(restorePriority); + out.writeOptionalString(snapshotId); + out.writeInt(snapshotDocCount); + boolean hasModelSizeStats = modelSizeStats != null; + out.writeBoolean(hasModelSizeStats); + if (hasModelSizeStats) { + modelSizeStats.writeTo(out); + } + boolean hasLatestRecordTimeStamp = latestRecordTimeStamp != null; + out.writeBoolean(hasLatestRecordTimeStamp); + if (hasLatestRecordTimeStamp) { + out.writeLong(latestRecordTimeStamp.getTime()); + } + boolean hasLatestResultTimeStamp = latestResultTimeStamp != null; + out.writeBoolean(hasLatestResultTimeStamp); + if (hasLatestResultTimeStamp) { + out.writeLong(latestResultTimeStamp.getTime()); + } + boolean hasQuantiles = quantiles != null; + out.writeBoolean(hasQuantiles); + if (hasQuantiles) { + quantiles.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + builder.field(RESTORE_PRIORITY.getPreferredName(), restorePriority); + if (snapshotId != null) { + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + } + builder.field(SNAPSHOT_DOC_COUNT.getPreferredName(), snapshotDocCount); + if (modelSizeStats != null) { + builder.field(ModelSizeStats.TYPE.getPreferredName(), modelSizeStats); + } + if (latestRecordTimeStamp != null) { + builder.field(LATEST_RECORD_TIME.getPreferredName(), latestRecordTimeStamp.getTime()); + } + if (latestResultTimeStamp != null) { + builder.field(LATEST_RESULT_TIME.getPreferredName(), latestResultTimeStamp.getTime()); + } + if (quantiles != null) { + builder.field(Quantiles.TYPE.getPreferredName(), quantiles); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public long getRestorePriority() { + return restorePriority; + } + + public void setRestorePriority(long restorePriority) { + this.restorePriority = restorePriority; + } + + public String getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(String snapshotId) { + this.snapshotId = snapshotId; + } + + public int getSnapshotDocCount() { + return snapshotDocCount; + } + + public void setSnapshotDocCount(int snapshotDocCount) { + this.snapshotDocCount = snapshotDocCount; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public void setModelSizeStats(ModelSizeStats.Builder modelSizeStats) { + this.modelSizeStats = modelSizeStats.build(); + } + + public Quantiles getQuantiles() { + return quantiles; + } + + public void setQuantiles(Quantiles q) { + quantiles = q; + } + + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + public void setLatestRecordTimeStamp(Date latestRecordTimeStamp) { + this.latestRecordTimeStamp = latestRecordTimeStamp; + } + + public Date getLatestResultTimeStamp() { + return latestResultTimeStamp; + } + + public void setLatestResultTimeStamp(Date latestResultTimeStamp) { + this.latestResultTimeStamp = latestResultTimeStamp; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, description, restorePriority, snapshotId, quantiles, + snapshotDocCount, modelSizeStats, latestRecordTimeStamp, latestResultTimeStamp); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelSnapshot == false) { + return false; + } + + ModelSnapshot that = (ModelSnapshot) other; + + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.description, that.description) + && this.restorePriority == that.restorePriority + && Objects.equals(this.snapshotId, that.snapshotId) + && this.snapshotDocCount == that.snapshotDocCount + && Objects.equals(this.modelSizeStats, that.modelSizeStats) + && Objects.equals(this.quantiles, that.quantiles) + && Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) + && Objects.equals(this.latestResultTimeStamp, that.latestResultTimeStamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelState.java new file mode 100644 index 00000000000..3e069f3239b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/ModelState.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + + +/** + * The serialised models can get very large and only the C++ code + * understands how to decode them, hence there is no reason to load + * them into the Java process. + * + * However, the Java process DOES set up a mapping on the Elasticsearch + * index to tell Elasticsearch not to analyse the model state documents + * in any way. (Otherwise Elasticsearch would go into a spin trying to + * make sense of such large JSON documents.) + */ +public class ModelState +{ + /** + * The type of this class used when persisting the data + */ + public static final String TYPE = "modelState"; + + private ModelState() + { + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerConfig.java new file mode 100644 index 00000000000..25a6ca27a8e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerConfig.java @@ -0,0 +1,962 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * Scheduler configuration options. Describes where to proactively pull input + * data from. + *

+ * If a value has not been set it will be null. Object wrappers are + * used around integral types and booleans so they can take null + * values. + */ +public class SchedulerConfig extends ToXContentToBytes implements Writeable { + + /** + * The field name used to specify aggregation fields in Elasticsearch + * aggregations + */ + private static final String FIELD = "field"; + /** + * The field name used to specify document counts in Elasticsearch + * aggregations + */ + public static final String DOC_COUNT = "doc_count"; + + // NORELEASE: no camel casing: + public static final ParseField DATA_SOURCE = new ParseField("dataSource"); + public static final ParseField QUERY_DELAY = new ParseField("queryDelay"); + public static final ParseField FREQUENCY = new ParseField("frequency"); + public static final ParseField FILE_PATH = new ParseField("filePath"); + public static final ParseField TAIL_FILE = new ParseField("tailFile"); + public static final ParseField BASE_URL = new ParseField("baseUrl"); + public static final ParseField USERNAME = new ParseField("username"); + public static final ParseField PASSWORD = new ParseField("password"); + public static final ParseField ENCRYPTED_PASSWORD = new ParseField("encryptedPassword"); + public static final ParseField INDEXES = new ParseField("indexes"); + public static final ParseField TYPES = new ParseField("types"); + public static final ParseField QUERY = new ParseField("query"); + public static final ParseField RETRIEVE_WHOLE_SOURCE = new ParseField("retrieveWholeSource"); + public static final ParseField SCROLL_SIZE = new ParseField("scrollSize"); + public static final ParseField AGGREGATIONS = new ParseField("aggregations"); + public static final ParseField AGGS = new ParseField("aggs"); + /** + * Named to match Elasticsearch, hence lowercase_with_underscores instead of + * camelCase + */ + public static final ParseField SCRIPT_FIELDS = new ParseField("script_fields"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("schedule_config", a -> new SchedulerConfig.Builder((DataSource) a[0])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return DataSource.readFromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, DATA_SOURCE, ObjectParser.ValueType.STRING); + PARSER.declareLong(Builder::setQueryDelay, QUERY_DELAY); + PARSER.declareLong(Builder::setFrequency, FREQUENCY); + PARSER.declareString(Builder::setFilePath, FILE_PATH); + PARSER.declareBoolean(Builder::setTailFile, TAIL_FILE); + PARSER.declareString(Builder::setUsername, USERNAME); + PARSER.declareString(Builder::setPassword, PASSWORD); + PARSER.declareString(Builder::setEncryptedPassword, ENCRYPTED_PASSWORD); + PARSER.declareString(Builder::setBaseUrl, BASE_URL); + PARSER.declareStringArray(Builder::setIndexes, INDEXES); + PARSER.declareStringArray(Builder::setTypes, TYPES); + PARSER.declareObject(Builder::setQuery, (p, c) -> { + try { + return p.map(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, QUERY); + PARSER.declareObject(Builder::setAggregations, (p, c) -> { + try { + return p.map(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, AGGREGATIONS); + PARSER.declareObject(Builder::setAggs, (p, c) -> { + try { + return p.map(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, AGGS); + PARSER.declareObject(Builder::setScriptFields, (p, c) -> { + try { + return p.map(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, SCRIPT_FIELDS); + PARSER.declareBoolean(Builder::setRetrieveWholeSource, RETRIEVE_WHOLE_SOURCE); + PARSER.declareInt(Builder::setScrollSize, SCROLL_SIZE); + } + + // NORELEASE: please use primitives where possible here: + private final DataSource dataSource; + + /** + * The delay in seconds before starting to query a period of time + */ + private final Long queryDelay; + + /** + * The frequency in seconds with which queries are executed + */ + private final Long frequency; + + /** + * These values apply to the FILE data source + */ + private final String filePath; + private final Boolean tailFile; + + /** + * Used for data sources that require credentials. May be null in the case + * where credentials are sometimes needed and sometimes not (e.g. + * Elasticsearch). + */ + private final String username; + private final String password; + private final String encryptedPassword; + + /** + * These values apply to the ELASTICSEARCH data source + */ + private final String baseUrl; + private final List indexes; + private final List types; + // NORELEASE: These 4 fields can be reduced to a single + // SearchSourceBuilder field holding the entire source: + private final Map query; + private final Map aggregations; + private final Map aggs; + private final Map scriptFields; + private final Boolean retrieveWholeSource; + private final Integer scrollSize; + + private SchedulerConfig(DataSource dataSource, Long queryDelay, Long frequency, String filePath, Boolean tailFile, String username, + String password, String encryptedPassword, String baseUrl, List indexes, List types, Map query, + Map aggregations, Map aggs, Map scriptFields, Boolean retrieveWholeSource, + Integer scrollSize) { + this.dataSource = dataSource; + this.queryDelay = queryDelay; + this.frequency = frequency; + this.filePath = filePath; + this.tailFile = tailFile; + this.username = username; + this.password = password; + this.encryptedPassword = encryptedPassword; + this.baseUrl = baseUrl; + this.indexes = indexes; + this.types = types; + this.query = query; + this.aggregations = aggregations; + this.aggs = aggs; + this.scriptFields = scriptFields; + this.retrieveWholeSource = retrieveWholeSource; + this.scrollSize = scrollSize; + } + + public SchedulerConfig(StreamInput in) throws IOException { + this.dataSource = DataSource.readFromStream(in); + this.queryDelay = in.readOptionalLong(); + this.frequency = in.readOptionalLong(); + this.filePath = in.readOptionalString(); + this.tailFile = in.readOptionalBoolean(); + this.username = in.readOptionalString(); + this.password = in.readOptionalString(); + this.encryptedPassword = in.readOptionalString(); + this.baseUrl = in.readOptionalString(); + if (in.readBoolean()) { + this.indexes = in.readList(StreamInput::readString); + } else { + this.indexes = null; + } + if (in.readBoolean()) { + this.types = in.readList(StreamInput::readString); + } else { + this.types = null; + } + if (in.readBoolean()) { + this.query = in.readMap(); + } else { + this.query = null; + } + if (in.readBoolean()) { + this.aggregations = in.readMap(); + } else { + this.aggregations = null; + } + if (in.readBoolean()) { + this.aggs = in.readMap(); + } else { + this.aggs = null; + } + if (in.readBoolean()) { + this.scriptFields = in.readMap(); + } else { + this.scriptFields = null; + } + this.retrieveWholeSource = in.readOptionalBoolean(); + this.scrollSize = in.readOptionalVInt(); + } + + /** + * The data source that the scheduler is to pull data from. + * + * @return The data source. + */ + public DataSource getDataSource() { + return this.dataSource; + } + + public Long getQueryDelay() { + return this.queryDelay; + } + + public Long getFrequency() { + return this.frequency; + } + + /** + * For the FILE data source only, the path to the file. + * + * @return The path to the file, or null if not set. + */ + public String getFilePath() { + return this.filePath; + } + + /** + * For the FILE data source only, should the file be tailed? If not it will + * just be read from once. + * + * @return Should the file be tailed? (null if not set.) + */ + public Boolean getTailFile() { + return this.tailFile; + } + + /** + * For the ELASTICSEARCH data source only, the base URL to connect to + * Elasticsearch on. + * + * @return The URL, or null if not set. + */ + public String getBaseUrl() { + return this.baseUrl; + } + + /** + * The username to use to connect to the data source (if any). + * + * @return The username, or null if not set. + */ + public String getUsername() { + return this.username; + } + + /** + * For the ELASTICSEARCH data source only, one or more indexes to search for + * input data. + * + * @return The indexes to search, or null if not set. + */ + public List getIndexes() { + return this.indexes; + } + + /** + * For the ELASTICSEARCH data source only, one or more types to search for + * input data. + * + * @return The types to search, or null if not set. + */ + public List getTypes() { + return this.types; + } + + /** + * For the ELASTICSEARCH data source only, the Elasticsearch query DSL + * representing the query to submit to Elasticsearch to get the input data. + * This should not include time bounds, as these are added separately. This + * class does not attempt to interpret the query. The map will be converted + * back to an arbitrary JSON object. + * + * @return The search query, or null if not set. + */ + public Map getQuery() { + return this.query; + } + + /** + * For the ELASTICSEARCH data source only, should the whole _source document + * be retrieved for analysis, or just the analysis fields? + * + * @return Should the whole of _source be retrieved? (null if + * not set.) + */ + public Boolean getRetrieveWholeSource() { + return this.retrieveWholeSource; + } + + /** + * For the ELASTICSEARCH data source only, get the size of documents to be + * retrieved from each shard via a scroll search + * + * @return The size of documents to be retrieved from each shard via a + * scroll search + */ + public Integer getScrollSize() { + return this.scrollSize; + } + + /** + * The encrypted password to use to connect to the data source (if any). A + * class outside this package is responsible for encrypting and decrypting + * the password. + * + * @return The password, or null if not set. + */ + public String getEncryptedPassword() { + return encryptedPassword; + } + + /** + * The plain text password to use to connect to the data source (if any). + * This is likely to return null most of the time, as the + * intention is that it is only present it initial configurations, and gets + * replaced with an encrypted password as soon as possible after receipt. + * + * @return The password, or null if not set. + */ + public String getPassword() { + return password; + } + + /** + * For the ELASTICSEARCH data source only, optional Elasticsearch + * script_fields to add to the search to be submitted to Elasticsearch to + * get the input data. This class does not attempt to interpret the script + * fields. The map will be converted back to an arbitrary JSON object. + * + * @return The script fields, or null if not set. + */ + public Map getScriptFields() { + return this.scriptFields; + } + + /** + * For the ELASTICSEARCH data source only, optional Elasticsearch + * aggregations to apply to the search to be submitted to Elasticsearch to + * get the input data. This class does not attempt to interpret the + * aggregations. The map will be converted back to an arbitrary JSON object. + * Synonym for {@link #getAggs()} (like Elasticsearch). + * + * @return The aggregations, or null if not set. + */ + public Map getAggregations() { + return this.aggregations; + } + + /** + * For the ELASTICSEARCH data source only, optional Elasticsearch + * aggregations to apply to the search to be submitted to Elasticsearch to + * get the input data. This class does not attempt to interpret the + * aggregations. The map will be converted back to an arbitrary JSON object. + * Synonym for {@link #getAggregations()} (like Elasticsearch). + * + * @return The aggregations, or null if not set. + */ + public Map getAggs() { + return this.aggs; + } + + /** + * Convenience method to get either aggregations or aggs. + * + * @return The aggregations (whether initially specified in aggregations or + * aggs), or null if neither are set. + */ + public Map getAggregationsOrAggs() { + return (this.aggregations != null) ? this.aggregations : this.aggs; + } + + /** + * Build the list of fields expected in the output from aggregations + * submitted to Elasticsearch. + * + * @return The list of fields, or empty list if there are no aggregations. + */ + public List buildAggregatedFieldList() { + Map aggs = getAggregationsOrAggs(); + if (aggs == null) { + return Collections.emptyList(); + } + + SortedMap orderedFields = new TreeMap<>(); + + scanSubLevel(aggs, 0, orderedFields); + + return new ArrayList<>(orderedFields.values()); + } + + @SuppressWarnings("unchecked") + private void scanSubLevel(Map subLevel, int depth, SortedMap orderedFields) { + for (Map.Entry entry : subLevel.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Map) { + scanSubLevel((Map) value, depth + 1, orderedFields); + } else if (value instanceof String && FIELD.equals(entry.getKey())) { + orderedFields.put(depth, (String) value); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + dataSource.writeTo(out); + out.writeOptionalLong(queryDelay); + out.writeOptionalLong(frequency); + out.writeOptionalString(filePath); + out.writeOptionalBoolean(tailFile); + out.writeOptionalString(username); + out.writeOptionalString(password); + out.writeOptionalString(encryptedPassword); + out.writeOptionalString(baseUrl); + if (indexes != null) { + out.writeBoolean(true); + out.writeStringList(indexes); + } else { + out.writeBoolean(false); + } + if (types != null) { + out.writeBoolean(true); + out.writeStringList(types); + } else { + out.writeBoolean(false); + } + if (query != null) { + out.writeBoolean(true); + out.writeMap(query); + } else { + out.writeBoolean(false); + } + if (aggregations != null) { + out.writeBoolean(true); + out.writeMap(aggregations); + } else { + out.writeBoolean(false); + } + if (aggs != null) { + out.writeBoolean(true); + out.writeMap(aggs); + } else { + out.writeBoolean(false); + } + if (scriptFields != null) { + out.writeBoolean(true); + out.writeMap(scriptFields); + } else { + out.writeBoolean(false); + } + out.writeOptionalBoolean(retrieveWholeSource); + out.writeOptionalVInt(scrollSize); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DATA_SOURCE.getPreferredName(), dataSource.name().toUpperCase(Locale.ROOT)); + if (queryDelay != null) { + builder.field(QUERY_DELAY.getPreferredName(), queryDelay); + } + if (frequency != null) { + builder.field(FREQUENCY.getPreferredName(), frequency); + } + if (filePath != null) { + builder.field(FILE_PATH.getPreferredName(), filePath); + } + if (tailFile != null) { + builder.field(TAIL_FILE.getPreferredName(), tailFile); + } + if (username != null) { + builder.field(USERNAME.getPreferredName(), username); + } + if (password != null) { + builder.field(PASSWORD.getPreferredName(), password); + } + if (encryptedPassword != null) { + builder.field(ENCRYPTED_PASSWORD.getPreferredName(), encryptedPassword); + } + if (baseUrl != null) { + builder.field(BASE_URL.getPreferredName(), baseUrl); + } + if (indexes != null) { + builder.field(INDEXES.getPreferredName(), indexes); + } + if (types != null) { + builder.field(TYPES.getPreferredName(), types); + } + if (query != null) { + builder.field(QUERY.getPreferredName(), query); + } + if (aggregations != null) { + builder.field(AGGREGATIONS.getPreferredName(), aggregations); + } + if (aggs != null) { + builder.field(AGGS.getPreferredName(), aggs); + } + if (scriptFields != null) { + builder.field(SCRIPT_FIELDS.getPreferredName(), scriptFields); + } + if (retrieveWholeSource != null) { + builder.field(RETRIEVE_WHOLE_SOURCE.getPreferredName(), retrieveWholeSource); + } + if (scrollSize != null) { + builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); + } + builder.endObject(); + return builder; + } + + /** + * The lists of indexes and types are compared for equality but they are not + * sorted first so this test could fail simply because the indexes and types + * lists are in different orders. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof SchedulerConfig == false) { + return false; + } + + SchedulerConfig that = (SchedulerConfig) other; + + return Objects.equals(this.dataSource, that.dataSource) && Objects.equals(this.frequency, that.frequency) + && Objects.equals(this.queryDelay, that.queryDelay) && Objects.equals(this.filePath, that.filePath) + && Objects.equals(this.tailFile, that.tailFile) && Objects.equals(this.baseUrl, that.baseUrl) + && Objects.equals(this.username, that.username) && Objects.equals(this.password, that.password) + && Objects.equals(this.encryptedPassword, that.encryptedPassword) && Objects.equals(this.indexes, that.indexes) + && Objects.equals(this.types, that.types) && Objects.equals(this.query, that.query) + && Objects.equals(this.retrieveWholeSource, that.retrieveWholeSource) && Objects.equals(this.scrollSize, that.scrollSize) + && Objects.equals(this.getAggregationsOrAggs(), that.getAggregationsOrAggs()) + && Objects.equals(this.scriptFields, that.scriptFields); + } + + @Override + public int hashCode() { + return Objects.hash(this.dataSource, frequency, queryDelay, this.filePath, tailFile, baseUrl, username, password, encryptedPassword, + this.indexes, types, query, retrieveWholeSource, scrollSize, getAggregationsOrAggs(), this.scriptFields); + } + + /** + * Enum of the acceptable data sources. + */ + public enum DataSource implements Writeable { + + FILE, ELASTICSEARCH; + + /** + * Case-insensitive from string method. Works with ELASTICSEARCH, + * Elasticsearch, ElasticSearch, etc. + * + * @param value + * String representation + * @return The data source + */ + public static DataSource readFromString(String value) { + String valueUpperCase = value.toUpperCase(Locale.ROOT); + return DataSource.valueOf(valueUpperCase); + } + + public static DataSource readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Operator ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + } + + public static class Builder { + + private static final int DEFAULT_SCROLL_SIZE = 1000; + private static final long DEFAULT_ELASTICSEARCH_QUERY_DELAY = 60L; + + /** + * The default query for elasticsearch searches + */ + private static final String MATCH_ALL_ES_QUERY = "match_all"; + + private final DataSource dataSource; + private Long queryDelay; + private Long frequency; + private String filePath; + private Boolean tailFile; + private String username; + private String password; + private String encryptedPassword; + private String baseUrl; + // NORELEASE: use Collections.emptyList() instead of null as initial + // value: + private List indexes = null; + private List types = null; + // NORELEASE: use Collections.emptyMap() instead of null as initial + // value: + // NORELEASE: Use SearchSourceBuilder + private Map query = null; + private Map aggregations = null; + private Map aggs = null; + private Map scriptFields = null; + private Boolean retrieveWholeSource; + private Integer scrollSize; + + // NORELEASE: figure out what the required fields are and made part of + // the only public constructor + public Builder(DataSource dataSource) { + this.dataSource = Objects.requireNonNull(dataSource); + switch (dataSource) { + case FILE: + setTailFile(false); + break; + case ELASTICSEARCH: + Map query = new HashMap<>(); + query.put(MATCH_ALL_ES_QUERY, new HashMap()); + setQuery(query); + setQueryDelay(DEFAULT_ELASTICSEARCH_QUERY_DELAY); + setRetrieveWholeSource(false); + setScrollSize(DEFAULT_SCROLL_SIZE); + break; + default: + throw new UnsupportedOperationException("unsupported datasource " + dataSource); + } + } + + public Builder(SchedulerConfig config) { + this.dataSource = config.dataSource; + this.queryDelay = config.queryDelay; + this.frequency = config.frequency; + this.filePath = config.filePath; + this.tailFile = config.tailFile; + this.username = config.username; + this.password = config.password; + this.encryptedPassword = config.encryptedPassword; + this.baseUrl = config.baseUrl; + this.indexes = config.indexes; + this.types = config.types; + this.query = config.query; + this.aggregations = config.aggregations; + this.aggs = config.aggs; + this.scriptFields = config.scriptFields; + this.retrieveWholeSource = config.retrieveWholeSource; + this.scrollSize = config.scrollSize; + } + + public void setQueryDelay(long queryDelay) { + if (queryDelay < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, + SchedulerConfig.QUERY_DELAY.getPreferredName(), queryDelay); + throw new IllegalArgumentException(msg); + } + this.queryDelay = queryDelay; + } + + public void setFrequency(long frequency) { + if (frequency <= 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, + SchedulerConfig.FREQUENCY.getPreferredName(), frequency); + throw new IllegalArgumentException(msg); + } + this.frequency = frequency; + } + + public void setFilePath(String filePath) { + this.filePath = Objects.requireNonNull(filePath); + } + + public void setTailFile(boolean tailFile) { + this.tailFile = tailFile; + } + + public void setUsername(String username) { + this.username = Objects.requireNonNull(username); + } + + public void setPassword(String password) { + this.password = Objects.requireNonNull(password); + } + + public void setEncryptedPassword(String encryptedPassword) { + this.encryptedPassword = Objects.requireNonNull(encryptedPassword); + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = Objects.requireNonNull(baseUrl); + } + + public void setIndexes(List indexes) { + // NORELEASE: make use of Collections.unmodifiableList(...) + this.indexes = Objects.requireNonNull(indexes); + } + + public void setTypes(List types) { + // NORELEASE: make use of Collections.unmodifiableList(...) + this.types = Objects.requireNonNull(types); + } + + public void setQuery(Map query) { + // NORELEASE: make use of Collections.unmodifiableMap(...) + this.query = Objects.requireNonNull(query); + } + + public void setAggregations(Map aggregations) { + // NORELEASE: make use of Collections.unmodifiableMap(...) + this.aggregations = Objects.requireNonNull(aggregations); + } + + public void setAggs(Map aggs) { + // NORELEASE: make use of Collections.unmodifiableMap(...) + this.aggs = Objects.requireNonNull(aggs); + } + + public void setScriptFields(Map scriptFields) { + // NORELEASE: make use of Collections.unmodifiableMap(...) + this.scriptFields = Objects.requireNonNull(scriptFields); + } + + public void setRetrieveWholeSource(boolean retrieveWholeSource) { + this.retrieveWholeSource = retrieveWholeSource; + } + + public void setScrollSize(int scrollSize) { + if (scrollSize < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, + SchedulerConfig.SCROLL_SIZE.getPreferredName(), scrollSize); + throw new IllegalArgumentException(msg); + } + this.scrollSize = scrollSize; + } + + public DataSource getDataSource() { + return dataSource; + } + + public Long getQueryDelay() { + return queryDelay; + } + + public Long getFrequency() { + return frequency; + } + + public String getFilePath() { + return filePath; + } + + public Boolean getTailFile() { + return tailFile; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getEncryptedPassword() { + return encryptedPassword; + } + + public String getBaseUrl() { + return baseUrl; + } + + public List getIndexes() { + return indexes; + } + + public List getTypes() { + return types; + } + + public Map getQuery() { + return query; + } + + public Map getAggregations() { + return aggregations; + } + + public Map getAggs() { + return aggs; + } + + /** + * Convenience method to get either aggregations or aggs. + * + * @return The aggregations (whether initially specified in aggregations + * or aggs), or null if neither are set. + */ + public Map getAggregationsOrAggs() { + return (this.aggregations != null) ? this.aggregations : this.aggs; + } + + public Map getScriptFields() { + return scriptFields; + } + + public Boolean getRetrieveWholeSource() { + return retrieveWholeSource; + } + + public Integer getScrollSize() { + return scrollSize; + } + + public SchedulerConfig build() { + switch (dataSource) { + case FILE: + if (Strings.hasLength(filePath) == false) { + throw invalidOptionValue(FILE_PATH.getPreferredName(), filePath); + } + if (baseUrl != null) { + throw notSupportedValue(BASE_URL, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (username != null) { + throw notSupportedValue(USERNAME, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (password != null) { + throw notSupportedValue(PASSWORD, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (encryptedPassword != null) { + throw notSupportedValue(ENCRYPTED_PASSWORD, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (indexes != null) { + throw notSupportedValue(INDEXES, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (types != null) { + throw notSupportedValue(TYPES, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (retrieveWholeSource != null) { + throw notSupportedValue(RETRIEVE_WHOLE_SOURCE, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (aggregations != null) { + throw notSupportedValue(AGGREGATIONS, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (query != null) { + throw notSupportedValue(QUERY, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (scriptFields != null) { + throw notSupportedValue(SCRIPT_FIELDS, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (scrollSize != null) { + throw notSupportedValue(SCROLL_SIZE, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + break; + case ELASTICSEARCH: + try { + new URL(baseUrl); + } catch (MalformedURLException e) { + throw invalidOptionValue(BASE_URL.getPreferredName(), baseUrl); + } + boolean isNoPasswordSet = password == null && encryptedPassword == null; + boolean isMultiplePasswordSet = password != null && encryptedPassword != null; + if ((username != null && isNoPasswordSet) || (isNoPasswordSet == false && username == null)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INCOMPLETE_CREDENTIALS); + throw new IllegalArgumentException(msg); + } + if (isMultiplePasswordSet) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_MULTIPLE_PASSWORDS); + throw new IllegalArgumentException(msg); + } + if (indexes == null || indexes.isEmpty() || indexes.contains(null) || indexes.contains("")) { + throw invalidOptionValue(INDEXES.getPreferredName(), indexes); + } + if (types == null || types.isEmpty() || types.contains(null) || types.contains("")) { + throw invalidOptionValue(TYPES.getPreferredName(), types); + } + if (aggregations != null && aggs != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_MULTIPLE_AGGREGATIONS); + throw new IllegalArgumentException(msg); + } + if (Boolean.TRUE.equals(retrieveWholeSource)) { + if (scriptFields != null) { + throw notSupportedValue(SCRIPT_FIELDS, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + } + if (filePath != null) { + throw notSupportedValue(FILE_PATH, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + if (tailFile != null) { + throw notSupportedValue(TAIL_FILE, dataSource, Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED); + } + break; + default: + throw new IllegalStateException("Unexpected datasource [" + dataSource + "]"); + } + return new SchedulerConfig(dataSource, queryDelay, frequency, filePath, tailFile, username, password, encryptedPassword, + baseUrl, indexes, types, query, aggregations, aggs, scriptFields, retrieveWholeSource, scrollSize); + } + + private static ElasticsearchException invalidOptionValue(String fieldName, Object value) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, fieldName, value); + throw new IllegalArgumentException(msg); + } + + private static ElasticsearchException notSupportedValue(ParseField field, DataSource dataSource, String key) { + String msg = Messages.getMessage(key, field.getPreferredName(), dataSource.toString()); + throw new IllegalArgumentException(msg); + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerState.java new file mode 100644 index 00000000000..3bc49a0498c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/SchedulerState.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class SchedulerState extends ToXContentToBytes implements Writeable { + + // NORELEASE: no camel casing: + public static final ParseField TYPE_FIELD = new ParseField("schedulerState"); + public static final ParseField STATUS = new ParseField("status"); + public static final ParseField START_TIME_MILLIS = new ParseField("start"); + public static final ParseField END_TIME_MILLIS = new ParseField("end"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE_FIELD.getPreferredName(), a -> new SchedulerState((JobSchedulerStatus) a[0], (long) a[1], (Long) a[2])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> JobSchedulerStatus.fromString(p.text()), STATUS, + ValueType.STRING); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), START_TIME_MILLIS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), END_TIME_MILLIS); + } + + private JobSchedulerStatus status; + private long startTimeMillis; + @Nullable + private Long endTimeMillis; + + public SchedulerState(JobSchedulerStatus status, long startTimeMillis, Long endTimeMillis) { + this.status = status; + this.startTimeMillis = startTimeMillis; + this.endTimeMillis = endTimeMillis; + } + + public SchedulerState(StreamInput in) throws IOException { + status = JobSchedulerStatus.fromStream(in); + startTimeMillis = in.readLong(); + endTimeMillis = in.readOptionalLong(); + } + + public JobSchedulerStatus getStatus() { + return status; + } + + public long getStartTimeMillis() { + return startTimeMillis; + } + + /** + * The end time as epoch milliseconds. An {@code null} end time indicates + * real-time mode. + * + * @return The optional end time as epoch milliseconds. + */ + @Nullable + public Long getEndTimeMillis() { + return endTimeMillis; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof SchedulerState == false) { + return false; + } + + SchedulerState that = (SchedulerState) other; + + return Objects.equals(this.status, that.status) && Objects.equals(this.startTimeMillis, that.startTimeMillis) + && Objects.equals(this.endTimeMillis, that.endTimeMillis); + } + + @Override + public int hashCode() { + return Objects.hash(status, startTimeMillis, endTimeMillis); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + status.writeTo(out); + out.writeLong(startTimeMillis); + out.writeOptionalLong(endTimeMillis); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(STATUS.getPreferredName(), status.name().toUpperCase(Locale.ROOT)); + builder.field(START_TIME_MILLIS.getPreferredName(), startTimeMillis); + if (endTimeMillis != null) { + builder.field(END_TIME_MILLIS.getPreferredName(), endTimeMillis); + } + builder.endObject(); + return builder; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivity.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivity.java new file mode 100644 index 00000000000..4c204a0df6c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivity.java @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class AuditActivity extends ToXContentToBytes implements Writeable +{ + public static final ParseField TYPE = new ParseField("auditActivity"); + + public static final ParseField TOTAL_JOBS = new ParseField("totalJobs"); + public static final ParseField TOTAL_DETECTORS = new ParseField("totalDetectors"); + public static final ParseField RUNNING_JOBS = new ParseField("runningJobs"); + public static final ParseField RUNNING_DETECTORS = new ParseField("runningDetectors"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), + AuditActivity::new); + + static { + PARSER.declareInt(AuditActivity::setTotalJobs, TOTAL_JOBS); + PARSER.declareInt(AuditActivity::setTotalDetectors, TOTAL_DETECTORS); + PARSER.declareInt(AuditActivity::setRunningJobs, RUNNING_JOBS); + PARSER.declareInt(AuditActivity::setRunningDetectors, RUNNING_DETECTORS); + PARSER.declareField(AuditActivity::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private int totalJobs; + private int totalDetectors; + private int runningJobs; + private int runningDetectors; + private Date timestamp; + + public AuditActivity() { + } + + private AuditActivity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) + { + this.totalJobs = totalJobs; + this.totalDetectors = totalDetectors; + this.runningJobs = runningJobs; + this.runningDetectors = runningDetectors; + timestamp = new Date(); + } + + public AuditActivity(StreamInput in) throws IOException { + totalJobs = in.readInt(); + totalDetectors = in.readInt(); + runningJobs = in.readInt(); + runningDetectors = in.readInt(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalJobs); + out.writeInt(totalDetectors); + out.writeInt(runningJobs); + out.writeInt(runningDetectors); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + public int getTotalJobs() + { + return totalJobs; + } + + public void setTotalJobs(int totalJobs) + { + this.totalJobs = totalJobs; + } + + public int getTotalDetectors() + { + return totalDetectors; + } + + public void setTotalDetectors(int totalDetectors) + { + this.totalDetectors = totalDetectors; + } + + public int getRunningJobs() + { + return runningJobs; + } + + public void setRunningJobs(int runningJobs) + { + this.runningJobs = runningJobs; + } + + public int getRunningDetectors() + { + return runningDetectors; + } + + public void setRunningDetectors(int runningDetectors) + { + this.runningDetectors = runningDetectors; + } + + public Date getTimestamp() + { + return timestamp; + } + + public void setTimestamp(Date timestamp) + { + this.timestamp = timestamp; + } + + public static AuditActivity newActivity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) + { + return new AuditActivity(totalJobs, totalDetectors, runningJobs, runningDetectors); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TOTAL_JOBS.getPreferredName(), totalJobs); + builder.field(TOTAL_DETECTORS.getPreferredName(), totalDetectors); + builder.field(RUNNING_JOBS.getPreferredName(), runningJobs); + builder.field(RUNNING_DETECTORS.getPreferredName(), runningDetectors); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(totalDetectors, totalJobs, runningDetectors, runningJobs, timestamp); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AuditActivity other = (AuditActivity) obj; + return Objects.equals(totalDetectors, other.totalDetectors) && + Objects.equals(totalJobs, other.totalJobs) && + Objects.equals(runningDetectors, other.runningDetectors) && + Objects.equals(runningJobs, other.runningJobs) && + Objects.equals(timestamp, other.timestamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessage.java new file mode 100644 index 00000000000..6058e7aa324 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessage.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class AuditMessage extends ToXContentToBytes implements Writeable +{ + public static final ParseField TYPE = new ParseField("auditMessage"); + + public static final ParseField MESSAGE = new ParseField("message"); + public static final ParseField LEVEL = new ParseField("level"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), + AuditMessage::new); + + static { + PARSER.declareString(AuditMessage::setJobId, Job.ID); + PARSER.declareString(AuditMessage::setMessage, MESSAGE); + PARSER.declareField(AuditMessage::setLevel, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Level.forString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, LEVEL, ValueType.STRING); + PARSER.declareField(AuditMessage::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private String jobId; + private String message; + private Level level; + private Date timestamp; + + public AuditMessage() + { + // Default constructor + } + + private AuditMessage(String jobId, String message, Level severity) + { + this.jobId = jobId; + this.message = message; + level = severity; + timestamp = new Date(); + } + + public AuditMessage(StreamInput in) throws IOException { + jobId = in.readOptionalString(); + message = in.readOptionalString(); + if (in.readBoolean()) { + level = Level.readFromStream(in); + } + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(jobId); + out.writeOptionalString(message); + boolean hasLevel = level != null; + out.writeBoolean(hasLevel); + if (hasLevel) { + level.writeTo(out); + } + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + public String getJobId() + { + return jobId; + } + + public void setJobId(String jobId) + { + this.jobId = jobId; + } + + public String getMessage() + { + return message; + } + + public void setMessage(String message) + { + this.message = message; + } + + public Level getLevel() + { + return level; + } + + public void setLevel(Level level) + { + this.level = level; + } + + public Date getTimestamp() + { + return timestamp; + } + + public void setTimestamp(Date timestamp) + { + this.timestamp = timestamp; + } + + public static AuditMessage newInfo(String jobId, String message) + { + return new AuditMessage(jobId, message, Level.INFO); + } + + public static AuditMessage newWarning(String jobId, String message) + { + return new AuditMessage(jobId, message, Level.WARNING); + } + + public static AuditMessage newActivity(String jobId, String message) + { + return new AuditMessage(jobId, message, Level.ACTIVITY); + } + + public static AuditMessage newError(String jobId, String message) + { + return new AuditMessage(jobId, message, Level.ERROR); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (jobId != null) { + builder.field(Job.ID.getPreferredName(), jobId); + } + if (message != null) { + builder.field(MESSAGE.getPreferredName(), message); + } + if (level != null) { + builder.field(LEVEL.getPreferredName(), level); + } + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, message, level, timestamp); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AuditMessage other = (AuditMessage) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(message, other.message) && + Objects.equals(level, other.level) && + Objects.equals(timestamp, other.timestamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Auditor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Auditor.java new file mode 100644 index 00000000000..b251ec1b53f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Auditor.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +public interface Auditor +{ + void info(String message); + void warning(String message); + void error(String message); + void activity(String message); + void activity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Level.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Level.java new file mode 100644 index 00000000000..a4ff99ceb08 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/audit/Level.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum Level implements Writeable { + INFO("info"), ACTIVITY("activity"), WARNING("warning"), ERROR("error"); + + private String name; + + private Level(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The condition type + */ + public static Level forString(String value) { + return Level.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Level readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Level ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Condition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Condition.java new file mode 100644 index 00000000000..14e85fd5d0d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Condition.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.condition; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * A class that describes a condition. + * The {@linkplain Operator} enum defines the available + * comparisons a condition can use. + */ +public class Condition extends ToXContentToBytes implements Writeable { + public static final ParseField CONDITION_FIELD = new ParseField("condition"); + public static final ParseField FILTER_VALUE_FIELD = new ParseField("value"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + CONDITION_FIELD.getPreferredName(), a -> new Condition((Operator) a[0], (String) a[1])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Operator.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, Operator.OPERATOR_FIELD, ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return p.text(); + } + if (p.currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, FILTER_VALUE_FIELD, ValueType.STRING_OR_NULL); + } + + private final Operator op; + private final String filterValue; + + public Condition(StreamInput in) throws IOException { + op = Operator.readFromStream(in); + filterValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + op.writeTo(out); + out.writeOptionalString(filterValue); + } + + public Condition(Operator op, String filterValue) { + if (filterValue == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL)); + } + + if (op.expectsANumericArgument()) { + try { + Double.parseDouble(filterValue); + } catch (NumberFormatException nfe) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, filterValue); + throw new IllegalArgumentException(msg); + } + } else { + try { + Pattern.compile(filterValue); + } catch (PatternSyntaxException e) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, filterValue); + throw new IllegalArgumentException(msg); + } + } + this.op = op; + this.filterValue = filterValue; + } + + public Operator getOperator() { + return op; + } + + public String getValue() { + return filterValue; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Operator.OPERATOR_FIELD.getPreferredName(), op.getName()); + builder.field(FILTER_VALUE_FIELD.getPreferredName(), filterValue); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(op, filterValue); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + Condition other = (Condition) obj; + return Objects.equals(this.op, other.op) && + Objects.equals(this.filterValue, other.filterValue); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Operator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Operator.java new file mode 100644 index 00000000000..66266e23f5d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/condition/Operator.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.condition; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Enum representing logical comparisons on doubles + */ +public enum Operator implements Writeable { + EQ("eq") { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) == 0; + } + }, + GT("gt") { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) > 0; + } + }, + GTE("gte") { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) >= 0; + } + }, + LT("lt") { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) < 0; + } + }, + LTE("lte") { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) <= 0; + } + }, + MATCH("match") { + @Override + public boolean match(Pattern pattern, String field) { + Matcher match = pattern.matcher(field); + return match.matches(); + } + + @Override + public boolean expectsANumericArgument() { + return false; + } + }; + + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + private final String name; + + private Operator(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public boolean test(double lhs, double rhs) { + return false; + } + + public boolean match(Pattern pattern, String field) { + return false; + } + + public boolean expectsANumericArgument() { + return true; + } + + public static Operator fromString(String name) { + Set all = EnumSet.allOf(Operator.class); + + String ucName = name.toUpperCase(Locale.ROOT); + for (Operator type : all) { + if (type.toString().equals(ucName)) { + return type; + } + } + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_UNKNOWN_OPERATOR, name)); + } + + public static Operator readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Operator ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescription.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescription.java new file mode 100644 index 00000000000..6f716c61524 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescription.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.config; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.utils.PrelertStrings; + + +public final class DefaultDetectorDescription { + private static final String BY_TOKEN = " by "; + private static final String OVER_TOKEN = " over "; + + private static final String USE_NULL_OPTION = " usenull="; + private static final String PARTITION_FIELD_OPTION = " partitionfield="; + private static final String EXCLUDE_FREQUENT_OPTION = " excludefrequent="; + + private DefaultDetectorDescription() { + // do nothing + } + + /** + * Returns the default description for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @return the default description + */ + public static String of(Detector detector) { + StringBuilder sb = new StringBuilder(); + appendOn(detector, sb); + return sb.toString(); + } + + /** + * Appends to the given {@code StringBuilder} the default description + * for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @param sb the {@code StringBuilder} to append to + */ + public static void appendOn(Detector detector, StringBuilder sb) { + if (isNotNullOrEmpty(detector.getFunction())) { + sb.append(detector.getFunction()); + if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append('(').append(quoteField(detector.getFieldName())) + .append(')'); + } + } else if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append(quoteField(detector.getFieldName())); + } + + if (isNotNullOrEmpty(detector.getByFieldName())) { + sb.append(BY_TOKEN).append(quoteField(detector.getByFieldName())); + } + + if (isNotNullOrEmpty(detector.getOverFieldName())) { + sb.append(OVER_TOKEN).append(quoteField(detector.getOverFieldName())); + } + + if (detector.isUseNull()) { + sb.append(USE_NULL_OPTION).append(detector.isUseNull()); + } + + if (isNotNullOrEmpty(detector.getPartitionFieldName())) { + sb.append(PARTITION_FIELD_OPTION).append(quoteField(detector.getPartitionFieldName())); + } + + if (detector.getExcludeFrequent() != null) { + sb.append(EXCLUDE_FREQUENT_OPTION).append(detector.getExcludeFrequent().getToken()); + } + } + + private static String quoteField(String field) { + return PrelertStrings.doubleQuoteIfNotAlphaNumeric(field); + } + + private static boolean isNotNullOrEmpty(String arg) { + return !Strings.isNullOrEmpty(arg); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequency.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequency.java new file mode 100644 index 00000000000..7fa3c924cbf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequency.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.config; + +import java.time.Duration; + +/** + * Factory methods for a sensible default for the scheduler frequency + */ +public final class DefaultFrequency +{ + private static final int SECONDS_IN_MINUTE = 60; + private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; + private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; + private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + private static final Duration TEN_MINUTES = Duration.ofMinutes(10); + private static final Duration ONE_HOUR = Duration.ofHours(1); + + private DefaultFrequency() + { + // Do nothing + } + + /** + * Creates a sensible default frequency for a given bucket span. + * + * The default depends on the bucket span: + *

    + *
  • <= 2 mins -> 1 min
    • + *
  • <= 20 mins -> bucket span / 2
    • + *
  • <= 12 hours -> 10 mins
    • + *
  • > 12 hours -> 1 hour
    • + *
+ * + * @param bucketSpanSeconds the bucket span in seconds + * @return the default frequency + */ + public static Duration ofBucketSpan(long bucketSpanSeconds) + { + if (bucketSpanSeconds <= 0) + { + throw new IllegalArgumentException("Bucket span has to be > 0"); + } + + if (bucketSpanSeconds <= TWO_MINS_SECONDS) + { + return Duration.ofSeconds(SECONDS_IN_MINUTE); + } + if (bucketSpanSeconds <= TWENTY_MINS_SECONDS) + { + return Duration.ofSeconds(bucketSpanSeconds / 2); + } + if (bucketSpanSeconds <= HALF_DAY_SECONDS) + { + return TEN_MINUTES; + } + return ONE_HOUR; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataProcessor.java new file mode 100644 index 00000000000..1bf05ff834f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataProcessor.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.data; + +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; + +import java.io.InputStream; + +public interface DataProcessor { + + /** + * Passes data to the native process. + * This is a blocking call that won't return until all the data has been + * written to the process. + * + * An ElasticsearchStatusException will be thrown is any of these error conditions occur: + *
    + *
  1. If a configured field is missing from the CSV header
    2. + *
  2. If JSON data is malformed and we cannot recover parsing
    3. + *
  3. If a high proportion of the records the timestamp field that cannot be parsed
    4. + *
  4. If a high proportion of the records chronologically out of order
    5. + *
+ * + * @param jobId the jobId + * @param input Data input stream + * @param params Data processing parameters + * @return Count of records, fields, bytes, etc written + */ + DataCounts processData(String jobId, InputStream input, DataLoadParams params); + + /** + * Flush the running job, ensuring that the native process has had the + * opportunity to process all data previously sent to it with none left + * sitting in buffers. + * + * @param jobId The job to flush + * @param interimResultsParams Parameters about whether interim results calculation + * should occur and for which period of time + */ + void flushJob(String jobId, InterimResultsParams interimResultsParams); + + /** + * Stop the running job and mark it as finished.
+ * + * @param jobId The job to stop + */ + void closeJob(String jobId); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamer.java new file mode 100644 index 00000000000..c44a8f4a6cc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamer.java @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.data; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipException; + +public class DataStreamer { + private static final Logger LOGGER = Loggers.getLogger(DataStreamer.class); + + private final DataProcessor dataProccesor; + + public DataStreamer(DataProcessor dataProcessor) { + dataProccesor = Objects.requireNonNull(dataProcessor); + } + + /** + * Stream the data to the native process. + * + * @return Count of records, fields, bytes, etc written + */ + public DataCounts streamData(String contentEncoding, String jobId, InputStream input, DataLoadParams params) throws IOException { + LOGGER.trace("Handle Post data to job {} ", jobId); + + input = tryDecompressingInputStream(contentEncoding, jobId, input); + DataCounts stats = handleStream(jobId, input, params); + + LOGGER.debug("Data uploaded to job {}", jobId); + + return stats; + } + + private InputStream tryDecompressingInputStream(String contentEncoding, String jobId, InputStream input) throws IOException { + if ("gzip".equals(contentEncoding)) { + LOGGER.debug("Decompressing post data in job {}", jobId); + try { + return new GZIPInputStream(input); + } catch (ZipException ze) { + LOGGER.error("Failed to decompress data file", ze); + throw new IllegalArgumentException(Messages.getMessage(Messages.REST_GZIP_ERROR), ze); + } + } + return input; + } + + /** + * Pass the data stream to the native process. + * + * @return Count of records, fields, bytes, etc written + */ + private DataCounts handleStream(String jobId, InputStream input, DataLoadParams params) { + return dataProccesor.processData(jobId, input, params); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThread.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThread.java new file mode 100644 index 00000000000..3c8c4414f0e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThread.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.data; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +// NORELEASE - Use ES ThreadPool +public final class DataStreamerThread extends Thread { + private static final Logger LOGGER = Loggers.getLogger(DataStreamerThread.class); + + private DataCounts stats; + private final String jobId; + private final String contentEncoding; + private final DataLoadParams params; + private final InputStream input; + private final DataStreamer dataStreamer; + private ElasticsearchException jobException; + private IOException iOException; + + public DataStreamerThread(DataStreamer dataStreamer, String jobId, String contentEncoding, + DataLoadParams params, InputStream input) { + super("DataStreamer-" + jobId); + + this.dataStreamer = dataStreamer; + this.jobId = jobId; + this.contentEncoding = contentEncoding; + this.params = params; + this.input = input; + } + + @Override + public void run() { + try { + stats = dataStreamer.streamData(contentEncoding, jobId, input, params); + } catch (ElasticsearchException e) { + jobException = e; + } catch (IOException e) { + iOException = e; + } finally { + try { + input.close(); + } catch (IOException e) { + LOGGER.warn("Exception closing the data input stream", e); + } + } + } + + /** + * This method should only be called after the thread + * has joined other wise the result could be null + * (best case) or undefined. + */ + public DataCounts getDataCounts() { + return stats; + } + + /** + * If a Job exception was thrown during the run of this thread it + * is accessed here. Only call this method after the thread has joined. + */ + public Optional getJobException() { + return Optional.ofNullable(jobException); + } + + /** + * If an IOException was thrown during the run of this thread it + * is accessed here. Only call this method after the thread has joined. + */ + public Optional getIOException() { + return Optional.ofNullable(iOException); + } + + public String getJobId() { + return jobId; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/Connective.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/Connective.java new file mode 100644 index 00000000000..4599d68b6c8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/Connective.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import java.io.IOException; +import java.util.Locale; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +public enum Connective implements Writeable { + OR("or"), + AND("and"); + + private String name; + + private Connective(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The connective type + */ + public static Connective fromString(String value) { + return Connective.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Connective readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Connective ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRule.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRule.java new file mode 100644 index 00000000000..1bc6d0a81e8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRule.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class DetectionRule extends ToXContentToBytes implements Writeable { + public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); + public static final ParseField RULE_ACTION_FIELD = new ParseField("rule_action"); + public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name"); + public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value"); + public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective"); + public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + DETECTION_RULE_FIELD.getPreferredName(), + arr -> { + @SuppressWarnings("unchecked") + List rules = (List) arr[3]; + return new DetectionRule((String) arr[0], (String) arr[1], (Connective) arr[2], rules); + } + ); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_VALUE_FIELD); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Connective.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), + (parser, parseFieldMatcher) -> RuleCondition.PARSER.apply(parser, parseFieldMatcher), RULE_CONDITIONS_FIELD); + } + + private final RuleAction ruleAction = RuleAction.FILTER_RESULTS; + private final String targetFieldName; + private final String targetFieldValue; + private final Connective conditionsConnective; + private final List ruleConditions; + + public DetectionRule(StreamInput in) throws IOException { + conditionsConnective = Connective.readFromStream(in); + int size = in.readVInt(); + ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + ruleConditions.add(new RuleCondition(in)); + } + targetFieldName = in.readOptionalString(); + targetFieldValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + conditionsConnective.writeTo(out); + out.writeVInt(ruleConditions.size()); + for (RuleCondition condition : ruleConditions) { + condition.writeTo(out); + } + out.writeOptionalString(targetFieldName); + out.writeOptionalString(targetFieldValue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective.getName()); + builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions); + if (targetFieldName != null) { + builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName); + } + if (targetFieldValue != null) { + builder.field(TARGET_FIELD_VALUE_FIELD.getPreferredName(), targetFieldValue); + } + builder.endObject(); + return builder; + } + + public DetectionRule(String targetFieldName, String targetFieldValue, Connective conditionsConnective, + List ruleConditions) { + if (targetFieldValue != null && targetFieldName == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue); + throw new IllegalArgumentException(msg); + } + if (ruleConditions == null || ruleConditions.isEmpty()) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION); + throw new IllegalArgumentException(msg); + } + for (RuleCondition condition : ruleConditions) { + if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, + DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName()); + throw new IllegalArgumentException(msg); + } + } + + this.targetFieldName = targetFieldName; + this.targetFieldValue = targetFieldValue; + this.conditionsConnective = conditionsConnective != null ? conditionsConnective : Connective.OR; + this.ruleConditions = Collections.unmodifiableList(ruleConditions); + } + + public RuleAction getRuleAction() { + return ruleAction; + } + + public String getTargetFieldName() { + return targetFieldName; + } + + public String getTargetFieldValue() { + return targetFieldValue; + } + + public Connective getConditionsConnective() { + return conditionsConnective; + } + + public List getRuleConditions() { + return ruleConditions; + } + + public Set extractReferencedLists() { + return ruleConditions.stream().map(RuleCondition::getValueList).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof DetectionRule == false) { + return false; + } + + DetectionRule other = (DetectionRule) obj; + return Objects.equals(ruleAction, other.ruleAction) && Objects.equals(targetFieldName, other.targetFieldName) + && Objects.equals(targetFieldValue, other.targetFieldValue) + && Objects.equals(conditionsConnective, other.conditionsConnective) && Objects.equals(ruleConditions, other.ruleConditions); + } + + @Override + public int hashCode() { + return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleAction.java new file mode 100644 index 00000000000..4a01c08be4b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleAction.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import java.util.Locale; + +public enum RuleAction +{ + FILTER_RESULTS; + + /** + * Case-insensitive from string method. + * + * @param value String representation + * @return The rule action + */ + public static RuleAction forString(String value) + { + return RuleAction.valueOf(value.toUpperCase(Locale.ROOT)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleCondition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleCondition.java new file mode 100644 index 00000000000..1e6450d6eca --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleCondition.java @@ -0,0 +1,248 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Objects; + +public class RuleCondition extends ToXContentToBytes implements Writeable { + public static final ParseField CONDITION_TYPE_FIELD = new ParseField("conditionType"); + public static final ParseField RULE_CONDITION_FIELD = new ParseField("ruleCondition"); + public static final ParseField FIELD_NAME_FIELD = new ParseField("fieldName"); + public static final ParseField FIELD_VALUE_FIELD = new ParseField("fieldValue"); + public static final ParseField VALUE_LIST_FIELD = new ParseField("valueList"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), + a -> new RuleCondition((RuleConditionType) a[0], (String) a[1], (String) a[2], (Condition) a[3], (String) a[4])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return RuleConditionType.forString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, CONDITION_TYPE_FIELD, ValueType.STRING); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), VALUE_LIST_FIELD); + } + + private final RuleConditionType conditionType; + private final String fieldName; + private final String fieldValue; + private final Condition condition; + private final String valueList; + + public RuleCondition(StreamInput in) throws IOException { + conditionType = RuleConditionType.readFromStream(in); + condition = in.readOptionalWriteable(Condition::new); + fieldName = in.readOptionalString(); + fieldValue = in.readOptionalString(); + valueList = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + conditionType.writeTo(out); + out.writeOptionalWriteable(condition); + out.writeOptionalString(fieldName); + out.writeOptionalString(fieldValue); + out.writeOptionalString(valueList); + } + + public RuleCondition(RuleConditionType conditionType, String fieldName, String fieldValue, Condition condition, String valueList) { + this.conditionType = conditionType; + this.fieldName = fieldName; + this.fieldValue = fieldValue; + this.condition = condition; + this.valueList = valueList; + + verifyFieldsBoundToType(this); + verifyFieldValueRequiresFieldName(this); + } + + public RuleCondition(RuleCondition ruleCondition) { + this.conditionType = ruleCondition.conditionType; + this.fieldName = ruleCondition.fieldName; + this.fieldValue = ruleCondition.fieldValue; + this.condition = ruleCondition.condition; + this.valueList = ruleCondition.valueList; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONDITION_TYPE_FIELD.getPreferredName(), conditionType); + if (condition != null) { + builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition); + } + if (fieldName != null) { + builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); + } + if (fieldValue != null) { + builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue); + } + if (valueList != null) { + builder.field(VALUE_LIST_FIELD.getPreferredName(), valueList); + } + builder.endObject(); + return builder; + } + + public RuleConditionType getConditionType() { + return conditionType; + } + + /** + * The field name for which the rule applies. Can be null, meaning rule + * applies to all results. + */ + public String getFieldName() { + return fieldName; + } + + /** + * The value of the field name for which the rule applies. When set, the + * rule applies only to the results that have the fieldName/fieldValue pair. + * When null, the rule applies to all values for of the specified field + * name. Only applicable when fieldName is not null. + */ + public String getFieldValue() { + return fieldValue; + } + + public Condition getCondition() { + return condition; + } + + /** + * The unique identifier of a list. Required when the rule type is + * categorical. Should be null for all other types. + */ + public String getValueList() { + return valueList; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof RuleCondition == false) { + return false; + } + + RuleCondition other = (RuleCondition) obj; + return Objects.equals(conditionType, other.conditionType) && Objects.equals(fieldName, other.fieldName) + && Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition) + && Objects.equals(valueList, other.valueList); + } + + @Override + public int hashCode() { + return Objects.hash(conditionType, fieldName, fieldValue, condition, valueList); + } + + public static RuleCondition createCategorical(String fieldName, String valueList) { + return new RuleCondition(RuleConditionType.CATEGORICAL, fieldName, null, null, valueList); + } + + private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException { + switch (ruleCondition.getConditionType()) { + case CATEGORICAL: + verifyCategorical(ruleCondition); + break; + case NUMERICAL_ACTUAL: + case NUMERICAL_TYPICAL: + case NUMERICAL_DIFF_ABS: + verifyNumerical(ruleCondition); + break; + default: + throw new IllegalStateException(); + } + } + + private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException { + checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); + checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue()); + checkCategoricalHasField(RuleCondition.VALUE_LIST_FIELD.getPreferredName(), ruleCondition.getValueList()); + } + + private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void checkCategoricalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException { + checkNumericalHasNoField(RuleCondition.VALUE_LIST_FIELD.getPreferredName(), ruleCondition.getValueList()); + checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); + if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE); + throw new IllegalArgumentException(msg); + } + checkNumericalConditionOparatorsAreValid(ruleCondition); + } + + private static void checkNumericalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void checkNumericalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyFieldValueRequiresFieldName(RuleCondition ruleCondition) throws ElasticsearchParseException { + if (ruleCondition.getFieldValue() != null && ruleCondition.getFieldName() == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME, + ruleCondition.getFieldValue()); + throw new IllegalArgumentException(msg); + } + } + + static EnumSet VALID_CONDITION_OPERATORS = EnumSet.of(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE); + + private static void checkNumericalConditionOparatorsAreValid(RuleCondition ruleCondition) throws ElasticsearchParseException { + Operator operator = ruleCondition.getCondition().getOperator(); + if (!VALID_CONDITION_OPERATORS.contains(operator)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR, operator); + throw new IllegalArgumentException(msg); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionType.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionType.java new file mode 100644 index 00000000000..317eab56cff --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionType.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + + +import java.io.IOException; +import java.util.Locale; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +public enum RuleConditionType implements Writeable { + CATEGORICAL("categorical"), + NUMERICAL_ACTUAL("numerical_actual"), + NUMERICAL_TYPICAL("numerical_typical"), + NUMERICAL_DIFF_ABS("numerical_diff_abs"); + + private String name; + + private RuleConditionType(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The condition type + */ + public static RuleConditionType forString(String value) { + return RuleConditionType.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static RuleConditionType readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown RuleConditionType ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractor.java new file mode 100644 index 00000000000..f451bb9f10e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractor.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.extraction; + +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +public interface DataExtractor +{ + /** + * Set-up the extractor for a new search + * + * @param start start time + * @param end end time + * @param logger logger + */ + void newSearch(long start, long end, Logger logger) throws IOException; + + /** + * Cleans up after a search. + */ + void clear(); + + /** + * @return {@code true} if the search has not finished yet, or {@code false} otherwise + */ + boolean hasNext(); + + /** + * Returns the next available extracted data. Note that it is possible for the + * extracted data to be empty the last time this method can be called. + * @return an optional input stream with the next available extracted data + * @throws IOException if an error occurs while extracting the data + */ + Optional next() throws IOException; + + /** + * Cancel the current search. + */ + void cancel(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractorFactory.java new file mode 100644 index 00000000000..360712935b7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/extraction/DataExtractorFactory.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.extraction; + +import org.elasticsearch.xpack.prelert.job.Job; + +public interface DataExtractorFactory +{ + DataExtractor newExtractor(Job job); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessage.java new file mode 100644 index 00000000000..a97c78d7aa8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessage.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logging; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Provide access to the C++ log messages that arrive via a named pipe in JSON format. + */ +public class CppLogMessage extends ToXContentToBytes implements Writeable { + /** + * Field Names (these are defined by log4cxx; we have no control over them) + */ + public static final ParseField LOGGER_FIELD = new ParseField("logger"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + public static final ParseField LEVEL_FIELD = new ParseField("level"); + public static final ParseField PID_FIELD = new ParseField("pid"); + public static final ParseField THREAD_FIELD = new ParseField("thread"); + public static final ParseField MESSAGE_FIELD = new ParseField("message"); + public static final ParseField CLASS_FIELD = new ParseField("class"); + public static final ParseField METHOD_FIELD = new ParseField("method"); + public static final ParseField FILE_FIELD = new ParseField("file"); + public static final ParseField LINE_FIELD = new ParseField("line"); + + public static final ObjectParser PARSER = new ObjectParser<>( + LOGGER_FIELD.getPreferredName(), CppLogMessage::new); + + static { + PARSER.declareString(CppLogMessage::setLogger, LOGGER_FIELD); + PARSER.declareField(CppLogMessage::setTimestamp, p -> new Date(p.longValue()), TIMESTAMP_FIELD, ValueType.LONG); + PARSER.declareString(CppLogMessage::setLevel, LEVEL_FIELD); + PARSER.declareLong(CppLogMessage::setPid, PID_FIELD); + PARSER.declareString(CppLogMessage::setThread, THREAD_FIELD); + PARSER.declareString(CppLogMessage::setMessage, MESSAGE_FIELD); + PARSER.declareString(CppLogMessage::setClazz, CLASS_FIELD); + PARSER.declareString(CppLogMessage::setMethod, METHOD_FIELD); + PARSER.declareString(CppLogMessage::setFile, FILE_FIELD); + PARSER.declareLong(CppLogMessage::setLine, LINE_FIELD); + } + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("cppLogMessage"); + + private String logger = ""; + private Date timestamp; + private String level = ""; + private long pid = 0; + private String thread = ""; + private String message = ""; + private String clazz = ""; + private String method = ""; + private String file = ""; + private long line = 0; + + public CppLogMessage() { + timestamp = new Date(); + } + + public CppLogMessage(StreamInput in) throws IOException { + logger = in.readString(); + timestamp = new Date(in.readVLong()); + level = in.readString(); + pid = in.readVLong(); + thread = in.readString(); + message = in.readString(); + clazz = in.readString(); + method = in.readString(); + file = in.readString(); + line = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(logger); + out.writeVLong(timestamp.getTime()); + out.writeString(level); + out.writeVLong(pid); + out.writeString(thread); + out.writeString(message); + out.writeString(clazz); + out.writeString(method); + out.writeString(file); + out.writeVLong(line); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(LOGGER_FIELD.getPreferredName(), logger); + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestamp.getTime()); + builder.field(LEVEL_FIELD.getPreferredName(), level); + builder.field(PID_FIELD.getPreferredName(), pid); + builder.field(THREAD_FIELD.getPreferredName(), thread); + builder.field(MESSAGE_FIELD.getPreferredName(), message); + builder.field(CLASS_FIELD.getPreferredName(), clazz); + builder.field(METHOD_FIELD.getPreferredName(), method); + builder.field(FILE_FIELD.getPreferredName(), file); + builder.field(LINE_FIELD.getPreferredName(), line); + builder.endObject(); + return builder; + } + + public String getLogger() { + return logger; + } + + public void setLogger(String logger) { + this.logger = logger; + } + + public Date getTimestamp() { + return this.timestamp; + } + + public void setTimestamp(Date d) { + this.timestamp = d; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public long getPid() { + return pid; + } + + public void setPid(long pid) { + this.pid = pid; + } + + public String getThread() { + return thread; + } + + public void setThread(String thread) { + this.thread = thread; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + /** + * This is unreliable for some C++ compilers - probably best not to display it prominently + */ + public String getClazz() { + return clazz; + } + + public void setClazz(String clazz) { + this.clazz = clazz; + } + + /** + * This is unreliable for some C++ compilers - probably best not to display it prominently + */ + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getFile() { + return file; + } + + public void setFile(String file) { + this.file = file; + } + + public long getLine() { + return line; + } + + public void setLine(long line) { + this.line = line; + } + + @Override + public int hashCode() { + return Objects.hash(logger, timestamp, level, pid, thread, message, clazz, method, file, line); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (!(other instanceof CppLogMessage)) { + return false; + } + + CppLogMessage that = (CppLogMessage)other; + + return Objects.equals(this.logger, that.logger) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.level, that.level) && this.pid == that.pid + && Objects.equals(this.thread, that.thread) && Objects.equals(this.message, that.message) + && Objects.equals(this.clazz, that.clazz) && Objects.equals(this.method, that.method) + && Objects.equals(this.file, that.file) && this.line == that.line; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandler.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandler.java new file mode 100644 index 00000000000..50644681fa4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandler.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logging; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.util.Deque; +import java.util.Objects; + +/** + * Handle a stream of C++ log messages that arrive via a named pipe in JSON format. + * Retains the last few error messages so that they can be passed back in a REST response + * if the C++ process dies. + */ +public class CppLogMessageHandler implements Closeable { + + private static final int DEFAULT_READBUF_SIZE = 1024; + private static final int DEFAULT_ERROR_STORE_SIZE = 5; + + private final Logger logger; + private final InputStream inputStream; + private final int readBufSize; + private final int errorStoreSize; + private final Deque errorStore; + private volatile boolean hasLogStreamEnded; + private volatile boolean seenFatalError; + + /** + * @param jobId May be null or empty if the logs are from a process not associated with a job. + * @param inputStream May not be null. + */ + public CppLogMessageHandler(String jobId, InputStream inputStream) { + this(inputStream, Strings.isNullOrEmpty(jobId) ? Loggers.getLogger(CppLogMessageHandler.class) : Loggers.getLogger(jobId), + DEFAULT_READBUF_SIZE, DEFAULT_ERROR_STORE_SIZE); + } + + /** + * For testing - allows meddling with the logger, read buffer size and error store size. + */ + CppLogMessageHandler(InputStream inputStream, Logger logger, int readBufSize, int errorStoreSize) { + this.logger = Objects.requireNonNull(logger); + this.inputStream = Objects.requireNonNull(inputStream); + this.readBufSize = readBufSize; + this.errorStoreSize = errorStoreSize; + this.errorStore = ConcurrentCollections.newDeque(); + hasLogStreamEnded = false; + } + + @Override + public void close() throws IOException { + inputStream.close(); + } + + /** + * Tail the InputStream provided to the constructor, handling each complete log document as it arrives. + * This method will not return until either end-of-file is detected on the InputStream or the + * InputStream throws an exception. + */ + public void tailStream() throws IOException { + try { + XContent xContent = XContentFactory.xContent(XContentType.JSON); + BytesReference bytesRef = null; + byte[] readBuf = new byte[readBufSize]; + for (int bytesRead = inputStream.read(readBuf); bytesRead != -1; bytesRead = inputStream.read(readBuf)) { + if (bytesRef == null) { + bytesRef = new BytesArray(readBuf, 0, bytesRead); + } else { + bytesRef = new CompositeBytesReference(bytesRef, new BytesArray(readBuf, 0, bytesRead)); + } + bytesRef = parseMessages(xContent, bytesRef); + readBuf = new byte[readBufSize]; + } + } finally { + hasLogStreamEnded = true; + } + } + + public boolean hasLogStreamEnded() { + return hasLogStreamEnded; + } + + public boolean seenFatalError() { + return seenFatalError; + } + + /** + * Expected to be called very infrequently. + */ + public String getErrors() { + String[] errorSnapshot = errorStore.toArray(new String[0]); + StringBuilder errors = new StringBuilder(); + for (String error : errorSnapshot) { + errors.append(error).append('\n'); + } + return errors.toString(); + } + + private BytesReference parseMessages(XContent xContent, BytesReference bytesRef) { + byte marker = xContent.streamSeparator(); + int from = 0; + while (true) { + int nextMarker = findNextMarker(marker, bytesRef, from); + if (nextMarker == -1) { + // No more markers in this block + break; + } + parseMessage(xContent, bytesRef.slice(from, nextMarker - from)); + from = nextMarker + 1; + } + return bytesRef.slice(from, bytesRef.length() - from); + } + + private void parseMessage(XContent xContent, BytesReference bytesRef) { + try { + XContentParser parser = xContent.createParser(bytesRef); + CppLogMessage msg = CppLogMessage.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT); + Level level = Level.getLevel(msg.getLevel()); + if (level == null) { + // This isn't expected to ever happen + level = Level.WARN; + } else if (level.isMoreSpecificThan(Level.ERROR)) { + // Keep the last few error messages to report if the process dies + storeError(msg.getMessage()); + if (level.isMoreSpecificThan(Level.FATAL)) { + seenFatalError = true; + } + } + // TODO: Is there a way to preserve the original timestamp when re-logging? + logger.log(level, "{}/{} {}@{} {}", msg.getLogger(), msg.getPid(), msg.getFile(), msg.getLine(), msg.getMessage()); + // TODO: Could send the message for indexing instead of or as well as logging it + } catch (IOException e) { + logger.warn("Failed to parse C++ log message: " + bytesRef.utf8ToString(), e); + } + } + + private void storeError(String error) { + if (Strings.isNullOrEmpty(error) || errorStoreSize <= 0) { + return; + } + if (errorStore.size() >= errorStoreSize) { + errorStore.removeFirst(); + } + errorStore.offerLast(error); + } + + private static int findNextMarker(byte marker, BytesReference bytesRef, int from) { + for (int i = from; i < bytesRef.length(); ++i) { + if (bytesRef.get(i) == marker) { + return i; + } + } + return -1; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logs/JobLogs.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logs/JobLogs.java new file mode 100644 index 00000000000..c606aad05c1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/logs/JobLogs.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logs; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Locale; + +/** + * Manage job logs + */ +public class JobLogs { + private static final Logger LOGGER = Loggers.getLogger(JobLogs.class); + /** + * If this system property is set the log files aren't deleted when the job + * is. + */ + public static final Setting DONT_DELETE_LOGS_SETTING = Setting.boolSetting("preserve.logs", false, Property.NodeScope); + + private boolean m_DontDelete; + + public JobLogs(Settings settings) + { + m_DontDelete = DONT_DELETE_LOGS_SETTING.get(settings); + } + + /** + * Delete all the log files and log directory associated with a job. + * + * @param jobId + * the jobId + * @return true if success. + */ + public boolean deleteLogs(Environment env, String jobId) { + return deleteLogs(env.logsFile().resolve(PrelertPlugin.NAME), jobId); + } + + /** + * Delete all the files in the directory + * + * 
+     * logDir / jobId
+     *
+ * + * . + * + * @param logDir + * The base directory of the log files + * @param jobId + * the jobId + */ + public boolean deleteLogs(Path logDir, String jobId) { + if (m_DontDelete) { + return true; + } + + Path logPath = sanitizePath(logDir.resolve(jobId), logDir); + + LOGGER.info(String.format(Locale.ROOT, "Deleting log files %s/%s", logDir, jobId)); + + try (DirectoryStream directoryStream = Files.newDirectoryStream(logPath)) { + for (Path logFile : directoryStream) { + try { + Files.delete(logFile); + } catch (IOException e) { + String msg = "Cannot delete log file " + logDir + ". "; + msg += (e.getCause() != null) ? e.getCause().getMessage() : e.getMessage(); + LOGGER.warn(msg); + } + } + } catch (IOException e) { + String msg = "Cannot open the log directory " + logDir + ". "; + msg += (e.getCause() != null) ? e.getCause().getMessage() : e.getMessage(); + LOGGER.warn(msg); + } + + // delete the directory + try { + Files.delete(logPath); + } catch (IOException e) { + String msg = "Cannot delete log directory " + logDir + ". "; + msg += (e.getCause() != null) ? e.getCause().getMessage() : e.getMessage(); + LOGGER.warn(msg); + return false; + } + + return true; + } + + /** + * Normalize a file path resolving .. and . directories and check the + * resulting path is below the rootDir directory. + * + * Throws an exception if the path is outside the logs directory e.g. + * logs/../lic/license resolves to lic/license and would throw + */ + public Path sanitizePath(Path filePath, Path rootDir) { + Path normalizedPath = filePath.normalize(); + Path rootPath = rootDir.normalize(); + if (normalizedPath.startsWith(rootPath) == false) { + String msg = Messages.getMessage(Messages.LOGFILE_INVALID_PATH, filePath); + LOGGER.warn(msg); + throw new IllegalArgumentException(msg); + } + + return normalizedPath; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManager.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManager.java new file mode 100644 index 00000000000..6f4658a014a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManager.java @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.manager; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobDataCountsPersister; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchPersister; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchUsagePersister; +import org.elasticsearch.xpack.prelert.job.persistence.JobDataCountsPersister; +import org.elasticsearch.xpack.prelert.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectCommunicator; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcessFactory; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutoDetectResultProcessor; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutodetectResultsParser; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.normalizer.noop.NoOpRenormaliser; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.usage.UsageReporter; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeoutException; + +public class AutodetectProcessManager extends AbstractComponent implements DataProcessor { + + private final Client client; + private final Environment env; + private final ThreadPool threadPool; + private final JobManager jobManager; + private final AutodetectResultsParser parser; + private final AutodetectProcessFactory autodetectProcessFactory; + + private final ConcurrentMap autoDetectCommunicatorByJob; + + public AutodetectProcessManager(Settings settings, Client client, Environment env, ThreadPool threadPool, JobManager jobManager, + AutodetectResultsParser parser, AutodetectProcessFactory autodetectProcessFactory) { + super(settings); + this.client = client; + this.env = env; + this.threadPool = threadPool; + this.parser = parser; + this.autodetectProcessFactory = autodetectProcessFactory; + this.jobManager = jobManager; + this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>(); + } + + @Override + public DataCounts processData(String jobId, InputStream input, DataLoadParams params) { + Allocation allocation = jobManager.getJobAllocation(jobId); + if (allocation.getStatus().isAnyOf(JobStatus.PAUSING, JobStatus.PAUSED)) { + return new DataCounts(jobId); + } + + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + communicator = create(jobId, params.isIgnoreDowntime()); + autoDetectCommunicatorByJob.put(jobId, communicator); + } + + try { + if (params.isResettingBuckets()) { + communicator.writeResetBucketsControlMessage(params); + } + return communicator.writeToJob(input); + // TODO check for errors from autodetect + } catch (IOException e) { + String msg = String.format(Locale.ROOT, "Exception writing to process for job %s", jobId); + if (e.getCause() instanceof TimeoutException) { + logger.warn("Connection to process was dropped due to a timeout - if you are feeding this job from a connector it " + + "may be that your connector stalled for too long", e.getCause()); + } + throw ExceptionsHelper.serverError(msg); + } + } + + // TODO (norelease) : here we must validate whether we have enough threads in TP in order start analytical process + // Otherwise we are not able to communicate via all the named pipes and we can run into deadlock + AutodetectCommunicator create(String jobId, boolean ignoreDowntime) { + Job job = jobManager.getJobOrThrowIfUnknown(jobId); + Logger jobLogger = Loggers.getLogger(job.getJobId()); + ElasticsearchUsagePersister usagePersister = new ElasticsearchUsagePersister(client, jobLogger); + UsageReporter usageReporter = new UsageReporter(settings, job.getJobId(), usagePersister, jobLogger); + + JobDataCountsPersister jobDataCountsPersister = new ElasticsearchJobDataCountsPersister(client, jobLogger); + StatusReporter statusReporter = new StatusReporter(env, settings, job.getJobId(), job.getCounts(), usageReporter, + jobDataCountsPersister, jobLogger, job.getAnalysisConfig().getBucketSpanOrDefault()); + + AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, ignoreDowntime); + JobResultsPersister persister = new ElasticsearchPersister(jobId, client); + // TODO Port the normalizer from the old project + AutoDetectResultProcessor processor = new AutoDetectResultProcessor(new NoOpRenormaliser(), persister, parser); + return new AutodetectCommunicator(threadPool, job, process, jobLogger, persister, statusReporter, processor); + } + + @Override + public void flushJob(String jobId, InterimResultsParams params) { + logger.debug("Flushing job {}", jobId); + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + logger.debug("Cannot flush: no active autodetect process for job {}", jobId); + return; + } + try { + communicator.flushJob(params); + // TODO check for errors from autodetect + } catch (IOException ioe) { + String msg = String.format(Locale.ROOT, "Exception flushing process for job %s", jobId); + logger.warn(msg); + throw ExceptionsHelper.serverError(msg, ioe); + } + } + + public void writeUpdateConfigMessage(String jobId, String config) throws IOException { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + logger.debug("Cannot update config: no active autodetect process for job {}", jobId); + return; + } + communicator.writeUpdateConfigMessage(config); + // TODO check for errors from autodetect + } + + @Override + public void closeJob(String jobId) { + logger.debug("Closing job {}", jobId); + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + logger.debug("Cannot close: no active autodetect process for job {}", jobId); + return; + } + + try { + communicator.close(); + // TODO check for errors from autodetect + // TODO delete associated files (model config etc) + } catch (IOException e) { + logger.info("Exception closing stopped process input stream", e); + } finally { + autoDetectCommunicatorByJob.remove(jobId); + setJobFinishedTimeAndStatus(jobId, JobStatus.CLOSED); + } + } + + public int numberOfRunningJobs() { + return autoDetectCommunicatorByJob.size(); + } + + public boolean jobHasActiveAutodetectProcess(String jobId) { + return autoDetectCommunicatorByJob.get(jobId) != null; + } + + public Duration jobUpTime(String jobId) { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + return Duration.ZERO; + } + return Duration.between(communicator.getProcessStartTime(), ZonedDateTime.now()); + } + + private void setJobFinishedTimeAndStatus(String jobId, JobStatus status) { + // NORELEASE Implement this. + // Perhaps move the JobStatus and finish time to a separate document stored outside the cluster state + logger.error("Cannot set finished job status and time- Not Implemented"); + } + + public Optional getModelSizeStats(String jobId) { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + return Optional.empty(); + } + + return communicator.getModelSizeStats(); + } + + public Optional getDataCounts(String jobId) { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + return Optional.empty(); + } + + return communicator.getDataCounts(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/JobManager.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/JobManager.java new file mode 100644 index 00000000000..616d2cf6a25 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/manager/JobManager.java @@ -0,0 +1,559 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.manager; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractLifecycleComponent; +import org.elasticsearch.common.component.Lifecycle; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.xpack.prelert.action.DeleteJobAction; +import org.elasticsearch.xpack.prelert.action.PauseJobAction; +import org.elasticsearch.xpack.prelert.action.PutJobAction; +import org.elasticsearch.xpack.prelert.action.ResumeJobAction; +import org.elasticsearch.xpack.prelert.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.prelert.action.StartJobSchedulerAction; +import org.elasticsearch.xpack.prelert.action.StopJobSchedulerAction; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.IgnoreDowntime; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.logs.JobLogs; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.metadata.PrelertMetadata; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Allows interactions with jobs. The managed interactions include: + *
    + *
  • creation
    • + *
  • deletion
    • + *
  • flushing
    • + *
  • updating
    • + *
  • sending of data
    • + *
  • fetching jobs and results
    • + *
  • starting/stopping of scheduled jobs
    • + *
+ */ +public class JobManager { + + private static final Logger LOGGER = Loggers.getLogger(JobManager.class); + + /** + * Field name in which to store the API version in the usage info + */ + public static final String APP_VER_FIELDNAME = "appVer"; + + public static final String DEFAULT_RECORD_SORT_FIELD = AnomalyRecord.PROBABILITY.getPreferredName(); + private final JobProvider jobProvider; + private final ClusterService clusterService; + private final Environment env; + private final Settings settings; + + /** + * Create a JobManager + */ + public JobManager(Environment env, Settings settings, JobProvider jobProvider, ClusterService clusterService) { + this.env = env; + this.settings = settings; + this.jobProvider = Objects.requireNonNull(jobProvider); + this.clusterService = clusterService; + } + + /** + * Get the details of the specific job wrapped in a Optional + * + * @param jobId + * the jobId + * @return An {@code Optional} containing the {@code Job} if a job + * with the given {@code jobId} exists, or an empty {@code Optional} + * otherwise + */ + public Optional getJob(String jobId, ClusterState clusterState) { + PrelertMetadata prelertMetadata = clusterState.getMetaData().custom(PrelertMetadata.TYPE); + Job job = prelertMetadata.getJobs().get(jobId); + if (job == null) { + return Optional.empty(); + } + + return Optional.of(job); + } + + /** + * Get details of all Jobs. + * + * @param from + * Skip the first N Jobs. This parameter is for paging results if + * not required set to 0. + * @param size + * Take only this number of Jobs + * @return A query page object with hitCount set to the total number of jobs + * not the only the number returned here as determined by the + * size parameter. + */ + public QueryPage getJobs(int from, int size, ClusterState clusterState) { + PrelertMetadata prelertMetadata = clusterState.getMetaData().custom(PrelertMetadata.TYPE); + List jobs = prelertMetadata.getJobs().entrySet().stream() + .skip(from) + .limit(size) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + return new QueryPage<>(jobs, prelertMetadata.getJobs().size()); + } + + /** + * Returns the non-null {@code Job} object for the given + * {@code jobId} or throws + * {@link org.elasticsearch.ResourceNotFoundException} + * + * @param jobId + * the jobId + * @return the {@code Job} if a job with the given {@code jobId} + * exists + * @throws org.elasticsearch.ResourceNotFoundException + * if there is no job with matching the given {@code jobId} + */ + public Job getJobOrThrowIfUnknown(String jobId) { + return getJobOrThrowIfUnknown(clusterService.state(), jobId); + } + + public Allocation getJobAllocation(String jobId) { + return getAllocation(clusterService.state(), jobId); + } + + /** + * Returns the non-null {@code Job} object for the given + * {@code jobId} or throws + * {@link org.elasticsearch.ResourceNotFoundException} + * + * @param jobId + * the jobId + * @return the {@code Job} if a job with the given {@code jobId} + * exists + * @throws org.elasticsearch.ResourceNotFoundException + * if there is no job with matching the given {@code jobId} + */ + public Job getJobOrThrowIfUnknown(ClusterState clusterState, String jobId) { + PrelertMetadata prelertMetadata = clusterState.metaData().custom(PrelertMetadata.TYPE); + Job job = prelertMetadata.getJobs().get(jobId); + if (job == null) { + throw ExceptionsHelper.missingJobException(jobId); + } + return job; + } + + /** + * Stores a job in the cluster state + */ + public void putJob(PutJobAction.Request request, ActionListener actionListener) { + Job job = request.getJob(); + ActionListener delegateListener = new ActionListener() { + @Override + public void onResponse(Boolean jobSaved) { + jobProvider.createJobRelatedIndices(job, new ActionListener() { + @Override + public void onResponse(Boolean indicesCreated) { + // NORELEASE: make auditing async too (we can't do + // blocking stuff here): + // audit(jobDetails.getId()).info(Messages.getMessage(Messages.JOB_AUDIT_CREATED)); + + // Also I wonder if we need to audit log infra + // structure in prelert as when we merge into xpack + // we can use its audit trailing. See: + // https://github.com/elastic/prelert-legacy/issues/48 + actionListener.onResponse(new PutJobAction.Response(jobSaved && indicesCreated, job)); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + + } + }); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }; + clusterService.submitStateUpdateTask("put-job-" + job.getId(), + new AckedClusterStateUpdateTask(request, delegateListener) { + + @Override + protected Boolean newResponse(boolean acknowledged) { + return acknowledged; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerPutJob(job, request.isOverwrite(), currentState); + } + + }); + } + + ClusterState innerPutJob(Job job, boolean overwrite, ClusterState currentState) { + PrelertMetadata currentPrelertMetadata = currentState.metaData().custom(PrelertMetadata.TYPE); + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(currentPrelertMetadata); + builder.putJob(job, overwrite); + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(PrelertMetadata.TYPE, builder.build()).build()); + return newState.build(); + } + + /** + * Deletes a job. + * + * The clean-up involves: + *
    + *
  • Deleting the index containing job results
    • + *
  • Deleting the job logs
    • + *
  • Removing the job from the cluster state
    • + *
+ * + * @param request + * the delete job request + * @param actionListener + * the action listener + */ + public void deleteJob(DeleteJobAction.Request request, ActionListener actionListener) { + String jobId = request.getJobId(); + LOGGER.debug("Deleting job '" + jobId + "'"); + // NORELEASE: Should also delete the running process + ActionListener delegateListener = new ActionListener() { + @Override + public void onResponse(Boolean jobDeleted) { + jobProvider.deleteJobRelatedIndices(request.getJobId(), new ActionListener() { + @Override + public void onResponse(Boolean indicesDeleted) { + + new JobLogs(settings).deleteLogs(env, jobId); + // NORELEASE: This is not the place the audit + // log + // (indexes a document), because this method is + // executed on + // the cluster state update task thread and any + // action performed on that thread should be + // quick. + // audit(jobId).info(Messages.getMessage(Messages.JOB_AUDIT_DELETED)); + + // Also I wonder if we need to audit log infra + // structure in prelert as when we merge into + // xpack + // we can use its audit trailing. See: + // https://github.com/elastic/prelert-legacy/issues/48 + actionListener.onResponse(new DeleteJobAction.Response(jobDeleted && indicesDeleted)); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }; + + clusterService.submitStateUpdateTask("delete-job-" + jobId, + new AckedClusterStateUpdateTask(request, delegateListener) { + + @Override + protected Boolean newResponse(boolean acknowledged) { + return acknowledged; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return removeJobFromClusterState(jobId, currentState); + } + }); + } + + ClusterState removeJobFromClusterState(String jobId, ClusterState currentState) { + PrelertMetadata currentPrelertMetadata = currentState.metaData().custom(PrelertMetadata.TYPE); + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(currentPrelertMetadata); + builder.removeJob(jobId); + + Allocation allocation = currentPrelertMetadata.getAllocations().get(jobId); + if (allocation != null) { + SchedulerState schedulerState = allocation.getSchedulerState(); + if (schedulerState != null && schedulerState.getStatus() != JobSchedulerStatus.STOPPED) { + throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.JOB_CANNOT_DELETE_WHILE_SCHEDULER_RUNS, jobId)); + } + } + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(PrelertMetadata.TYPE, builder.build()).build()); + return newState.build(); + } + + public void startJobScheduler(StartJobSchedulerAction.Request request, + ActionListener actionListener) { + clusterService.submitStateUpdateTask("start-scheduler-job-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + protected StartJobSchedulerAction.Response newResponse(boolean acknowledged) { + return new StartJobSchedulerAction.Response(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + long startTime = request.getSchedulerState().getStartTimeMillis(); + Long endTime = request.getSchedulerState().getEndTimeMillis(); + return innerUpdateSchedulerState(currentState, request.getJobId(), JobSchedulerStatus.STARTING, startTime, endTime); + } + }); + } + + public void stopJobScheduler(StopJobSchedulerAction.Request request, ActionListener actionListener) { + clusterService.submitStateUpdateTask("stop-scheduler-job-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + protected StopJobSchedulerAction.Response newResponse(boolean acknowledged) { + return new StopJobSchedulerAction.Response(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerUpdateSchedulerState(currentState, request.getJobId(), JobSchedulerStatus.STOPPING, null, null); + } + }); + } + + private void checkJobIsScheduled(Job job) { + if (job.getSchedulerConfig() == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_SCHEDULER_NO_SUCH_SCHEDULED_JOB, job.getId())); + } + } + + public void updateSchedulerStatus(String jobId, JobSchedulerStatus newStatus) { + clusterService.submitStateUpdateTask("update-scheduler-status-job-" + jobId, new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerUpdateSchedulerState(currentState, jobId, newStatus, null, null); + } + + @Override + public void onFailure(String source, Exception e) { + LOGGER.error("Error updating scheduler status: source=[" + source + "], status=[" + newStatus + "]", e); + } + }); + } + + private ClusterState innerUpdateSchedulerState(ClusterState currentState, String jobId, JobSchedulerStatus status, + Long startTime, Long endTime) { + Job job = getJobOrThrowIfUnknown(currentState, jobId); + checkJobIsScheduled(job); + + Allocation allocation = getAllocation(currentState, jobId); + if (allocation.getSchedulerState() == null && status != JobSchedulerStatus.STARTING) { + throw new IllegalArgumentException("Can't change status to [" + status + "], because job's [" + jobId + + "] scheduler never started"); + } + + SchedulerState existingState = allocation.getSchedulerState(); + if (existingState != null) { + if (startTime == null) { + startTime = existingState.getStartTimeMillis(); + } + if (endTime == null) { + endTime = existingState.getEndTimeMillis(); + } + } + + existingState = new SchedulerState(status, startTime, endTime); + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setSchedulerState(existingState); + return innerUpdateAllocation(builder.build(), currentState); + } + + private Allocation getAllocation(ClusterState state, String jobId) { + PrelertMetadata prelertMetadata = state.metaData().custom(PrelertMetadata.TYPE); + Allocation allocation = prelertMetadata.getAllocations().get(jobId); + if (allocation == null) { + throw new ResourceNotFoundException("No allocation found for job with id [" + jobId + "]"); + } + return allocation; + } + + private ClusterState innerUpdateAllocation(Allocation newAllocation, ClusterState currentState) { + PrelertMetadata currentPrelertMetadata = currentState.metaData().custom(PrelertMetadata.TYPE); + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(currentPrelertMetadata); + builder.updateAllocation(newAllocation.getJobId(), newAllocation); + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(PrelertMetadata.TYPE, builder.build()).build()); + return newState.build(); + } + + public Auditor audit(String jobId) { + return jobProvider.audit(jobId); + } + + public Auditor systemAudit() { + return jobProvider.audit(""); + } + + public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener actionListener, + ModelSnapshot modelSnapshot) { + + clusterService.submitStateUpdateTask("revert-snapshot-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + protected RevertModelSnapshotAction.Response newResponse(boolean acknowledged) { + RevertModelSnapshotAction.Response response; + + if (acknowledged) { + response = new RevertModelSnapshotAction.Response(modelSnapshot); + + // NORELEASE: This is not the place the audit log + // (indexes a document), because this method is + // executed on the cluster state update task thread + // and any action performed on that thread should be + // quick. (so no indexing documents) + // audit(jobId).info(Messages.getMessage(Messages.JOB_AUDIT_REVERTED, + // modelSnapshot.getDescription())); + + } else { + response = new RevertModelSnapshotAction.Response(); + } + return response; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + Job job = getJobOrThrowIfUnknown(currentState, request.getJobId()); + Job.Builder builder = new Job.Builder(job); + builder.setModelSnapshotId(modelSnapshot.getSnapshotId()); + if (request.getDeleteInterveningResults()) { + builder.setIgnoreDowntime(IgnoreDowntime.NEVER); + Date latestRecordTime = modelSnapshot.getLatestResultTimeStamp(); + LOGGER.info("Resetting latest record time to '" + latestRecordTime + "'"); + builder.setLastDataTime(latestRecordTime); + DataCounts counts = job.getCounts(); + counts.setLatestRecordTimeStamp(latestRecordTime); + builder.setCounts(counts); + } else { + builder.setIgnoreDowntime(IgnoreDowntime.ONCE); + } + + return innerPutJob(builder.build(), true, currentState); + } + }); + } + + public void pauseJob(PauseJobAction.Request request, ActionListener actionListener) { + clusterService.submitStateUpdateTask("pause-job-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + protected PauseJobAction.Response newResponse(boolean acknowledged) { + return new PauseJobAction.Response(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + Job job = getJobOrThrowIfUnknown(currentState, request.getJobId()); + Allocation allocation = getAllocation(currentState, job.getId()); + checkJobIsNotScheduled(job); + if (!allocation.getStatus().isAnyOf(JobStatus.RUNNING, JobStatus.CLOSED)) { + throw ExceptionsHelper.conflictStatusException( + Messages.getMessage(Messages.JOB_CANNOT_PAUSE, job.getId(), allocation.getStatus())); + } + + ClusterState newState = innerSetJobStatus(job.getId(), JobStatus.PAUSING, currentState); + Job.Builder jobBuilder = new Job.Builder(job); + jobBuilder.setIgnoreDowntime(IgnoreDowntime.ONCE); + return innerPutJob(jobBuilder.build(), true, newState); + } + }); + } + + public void resumeJob(ResumeJobAction.Request request, ActionListener actionListener) { + clusterService.submitStateUpdateTask("resume-job-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + getJobOrThrowIfUnknown(request.getJobId()); + Allocation allocation = getJobAllocation(request.getJobId()); + if (allocation.getStatus() != JobStatus.PAUSED) { + throw ExceptionsHelper.conflictStatusException( + Messages.getMessage(Messages.JOB_CANNOT_RESUME, request.getJobId(), allocation.getStatus())); + } + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setStatus(JobStatus.CLOSED); + return innerUpdateAllocation(builder.build(), currentState); + } + + @Override + protected ResumeJobAction.Response newResponse(boolean acknowledged) { + return new ResumeJobAction.Response(acknowledged); + } + }); + } + + public void setJobStatus(String jobId, JobStatus newStatus) { + clusterService.submitStateUpdateTask("set-paused-status-job-" + jobId, new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerSetJobStatus(jobId, newStatus, currentState); + } + + @Override + public void onFailure(String source, Exception e) { + LOGGER.error("Error updating job status: source=[" + source + "], new status [" + newStatus + "]", e); + } + }); + } + + private ClusterState innerSetJobStatus(String jobId, JobStatus newStatus, ClusterState currentState) { + Allocation allocation = getJobAllocation(jobId); + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setStatus(newStatus); + return innerUpdateAllocation(builder.build(), currentState); + } + + private void checkJobIsNotScheduled(Job job) { + if (job.getSchedulerConfig() != null) { + throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_ACTION_NOT_ALLOWED_FOR_SCHEDULED_JOB)); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/messages/Messages.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/messages/Messages.java new file mode 100644 index 00000000000..f1ab944871a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/messages/Messages.java @@ -0,0 +1,304 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.messages; + +import java.text.MessageFormat; +import java.util.Locale; +import java.util.ResourceBundle; + +/** + * Defines the keys for all the message strings + */ +public final class Messages +{ + /** + * The base name of the bundle without the .properties extension + * or locale + */ + private static final String BUNDLE_NAME = "org.elasticsearch.xpack.prelert.job.messages.prelert_messages"; + public static final String AUTODETECT_FLUSH_UNEXPTECTED_DEATH = "autodetect.flush.failed.unexpected.death"; + public static final String AUTODETECT_FLUSH_TIMEOUT = "autodetect.flush.timeout"; + + public static final String CPU_LIMIT_JOB = "cpu.limit.jobs"; + + public static final String DATASTORE_ERROR_DELETING = "datastore.error.deleting"; + public static final String DATASTORE_ERROR_DELETING_MISSING_INDEX = "datastore.error.deleting.missing.index"; + public static final String DATASTORE_ERROR_EXECUTING_SCRIPT = "datastore.error.executing.script"; + + public static final String LICENSE_LIMIT_DETECTORS = "license.limit.detectors"; + public static final String LICENSE_LIMIT_JOBS = "license.limit.jobs"; + public static final String LICENSE_LIMIT_DETECTORS_REACTIVATE = "license.limit.detectors.reactivate"; + public static final String LICENSE_LIMIT_JOBS_REACTIVATE = "license.limit.jobs.reactivate"; + public static final String LICENSE_LIMIT_PARTITIONS = "license.limit.partitions"; + + public static final String LOGFILE_INVALID_CHARS_IN_PATH = "logfile.invalid.chars.path"; + public static final String LOGFILE_INVALID_PATH = "logfile.invalid.path"; + public static final String LOGFILE_MISSING = "logfile.missing"; + public static final String LOGFILE_MISSING_DIRECTORY = "logfile.missing.directory"; + + + public static final String JOB_AUDIT_CREATED = "job.audit.created"; + public static final String JOB_AUDIT_DELETED = "job.audit.deleted"; + public static final String JOB_AUDIT_PAUSED = "job.audit.paused"; + public static final String JOB_AUDIT_RESUMED = "job.audit.resumed"; + public static final String JOB_AUDIT_UPDATED = "job.audit.updated"; + public static final String JOB_AUDIT_REVERTED = "job.audit.reverted"; + public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "job.audit.old.results.deleted"; + public static final String JOB_AUDIT_SNAPSHOT_DELETED = "job.audit.snapshot.deleted"; + public static final String JOB_AUDIT_SCHEDULER_STARTED_FROM_TO = "job.audit.scheduler.started.from.to"; + public static final String JOB_AUDIT_SCHEDULER_CONTINUED_REALTIME = "job.audit.scheduler.continued.realtime"; + public static final String JOB_AUDIT_SCHEDULER_STARTED_REALTIME = "job.audit.scheduler.started.realtime"; + public static final String JOB_AUDIT_SCHEDULER_LOOKBACK_COMPLETED = "job.audit.scheduler.lookback.completed"; + public static final String JOB_AUDIT_SCHEDULER_STOPPED = "job.audit.scheduler.stopped"; + public static final String JOB_AUDIT_SCHEDULER_NO_DATA = "job.audit.scheduler.no.data"; + public static final String JOB_AUDIR_SCHEDULER_DATA_SEEN_AGAIN = "job.audit.scheduler.data.seen.again"; + public static final String JOB_AUDIT_SCHEDULER_DATA_ANALYSIS_ERROR = "job.audit.scheduler.data.analysis.error"; + public static final String JOB_AUDIT_SCHEDULER_DATA_EXTRACTION_ERROR = "job.audit.scheduler.data.extraction.error"; + public static final String JOB_AUDIT_SCHEDULER_RECOVERED = "job.audit.scheduler.recovered"; + + public static final String SYSTEM_AUDIT_STARTED = "system.audit.started"; + public static final String SYSTEM_AUDIT_SHUTDOWN = "system.audit.shutdown"; + + public static final String JOB_CANNOT_DELETE_WHILE_SCHEDULER_RUNS = "job.cannot.delete.while.scheduler.runs"; + public static final String JOB_CANNOT_PAUSE = "job.cannot.pause"; + public static final String JOB_CANNOT_RESUME = "job.cannot.resume"; + + public static final String JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION = "job.config.byField.incompatible.function"; + public static final String JOB_CONFIG_BYFIELD_NEEDS_ANOTHER = "job.config.byField.needs.another"; + public static final String JOB_CONFIG_CANNOT_ENCRYPT_PASSWORD = "job.config.cannot.encrypt.password"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME = "job.config.categorization.filters." + + "require.categorization.field.name"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES = "job.config.categorization.filters.contains" + + ".duplicates"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY = "job.config.categorization.filter.contains.empty"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX = "job.config.categorization.filter.contains." + + "invalid.regex"; + public static final String JOB_CONFIG_CONDITION_INVALID_OPERATOR = "job.config.condition.invalid.operator"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NULL = "job.config.condition.invalid.value.null"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER = "job.config.condition.invalid.value.numeric"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX = "job.config.condition.invalid.value.regex"; + public static final String JOB_CONFIG_CONDITION_UNKNOWN_OPERATOR = "job.config.condition.unknown.operator"; + public static final String JOB_CONFIG_DATAFORMAT_REQUIRES_TRANSFORM = "job.config.dataformat.requires.transform"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION = "job.config.detectionrule.condition." + + "categorical.invalid.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION = "job.config.detectionrule.condition." + + "categorical.missing.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME = "job.config.detectionrule.condition.invalid." + + "fieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME = "job.config.detectionrule.condition.missing." + + "fieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR = "job.config.detectionrule.condition." + + "numerical.invalid.operator"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION = "job.config.detectionrule.condition." + + "numerical.invalid.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION = "job.config.detectionrule.condition." + + "numerical.missing.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE = "job.config." + + "detectionrule.condition.numerical.with.fieldname.requires.fieldvalue"; + public static final String JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME = "job.config.detectionrule.invalid.targetfieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME = "job.config.detectionrule.missing.targetfieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION = "job.config.detectionrule.not.supported.by.function"; + public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION = "job.config.detectionrule.requires.at." + + "least.one.condition"; + public static final String JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION = "job.config.fieldname.incompatible.function"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_BYFIELD = "job.config.function.requires.byfield"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_FIELDNAME = "job.config.function.requires.fieldname"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_OVERFIELD = "job.config.function.requires.overfield"; + public static final String JOB_CONFIG_ID_TOO_LONG = "job.config.id.too.long"; + public static final String JOB_CONFIG_ID_ALREADY_TAKEN = "job.config.id.already.taken"; + public static final String JOB_CONFIG_INVALID_FIELDNAME_CHARS = "job.config.invalid.fieldname.chars"; + public static final String JOB_CONFIG_INVALID_JOBID_CHARS = "job.config.invalid.jobid.chars"; + public static final String JOB_CONFIG_INVALID_TIMEFORMAT = "job.config.invalid.timeformat"; + public static final String JOB_CONFIG_FUNCTION_INCOMPATIBLE_PRESUMMARIZED = "job.config.function.incompatible.presummarized"; + public static final String JOB_CONFIG_MISSING_ANALYSISCONFIG = "job.config.missing.analysisconfig"; + public static final String JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE = "job.config.model.debug.config.invalid.bounds." + + "percentile"; + public static final String JOB_CONFIG_FIELD_VALUE_TOO_LOW = "job.config.field.value.too.low"; + public static final String JOB_CONFIG_NO_ANALYSIS_FIELD = "job.config.no.analysis.field"; + public static final String JOB_CONFIG_NO_ANALYSIS_FIELD_NOT_COUNT = "job.config.no.analysis.field.not.count"; + public static final String JOB_CONFIG_NO_DETECTORS = "job.config.no.detectors"; + public static final String JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION = "job.config.overField.incompatible.function"; + public static final String JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION = "job.config.overlapping.buckets.incompatible." + + "function"; + public static final String JOB_CONFIG_OVERFIELD_NEEDS_ANOTHER = "job.config.overField.needs.another"; + public static final String JOB_CONFIG_MULTIPLE_BUCKETSPANS_REQUIRE_BUCKETSPAN = "job.config.multiple.bucketspans.require.bucketspan"; + public static final String JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE = "job.config.multiple.bucketspans.must.be.multiple"; + public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD = "job.config.per.partition.normalisation." + + "requires.partition.field"; + public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS = "job.config.per.partition.normalisation." + + "cannot.use.influencers"; + + + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_PARSE_ERROR = "job.config.update.analysis.limits.parse.error"; + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_CANNOT_BE_NULL = "job.config.update.analysis.limits.cannot.be.null"; + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED = "job.config.update.analysis." + + "limits.model.memory.limit.cannot.be.decreased"; + public static final String JOB_CONFIG_UPDATE_CATEGORIZATION_FILTERS_INVALID = "job.config.update.categorization.filters.invalid"; + public static final String JOB_CONFIG_UPDATE_CUSTOM_SETTINGS_INVALID = "job.config.update.custom.settings.invalid"; + public static final String JOB_CONFIG_UPDATE_DESCRIPTION_INVALID = "job.config.update.description.invalid"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_INVALID = "job.config.update.detectors.invalid"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_INVALID_DETECTOR_INDEX = "job.config.update.detectors.invalid.detector.index"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_DETECTOR_INDEX_SHOULD_BE_INTEGER = "job.config.update.detectors.detector.index." + + "should.be.integer"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_MISSING_PARAMS = "job.config.update.detectors.missing.params"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_DESCRIPTION_SHOULD_BE_STRING = "job.config.update.detectors.description.should" + + ".be.string"; + public static final String JOB_CONFIG_UPDATE_DETECTOR_RULES_PARSE_ERROR = "job.config.update.detectors.rules.parse.error"; + public static final String JOB_CONFIG_UPDATE_FAILED = "job.config.update.failed"; + public static final String JOB_CONFIG_UPDATE_INVALID_KEY = "job.config.update.invalid.key"; + public static final String JOB_CONFIG_UPDATE_IGNORE_DOWNTIME_PARSE_ERROR = "job.config.update.ignore.downtime.parse.error"; + public static final String JOB_CONFIG_UPDATE_JOB_IS_NOT_CLOSED = "job.config.update.job.is.not.closed"; + public static final String JOB_CONFIG_UPDATE_MODEL_DEBUG_CONFIG_PARSE_ERROR = "job.config.update.model.debug.config.parse.error"; + public static final String JOB_CONFIG_UPDATE_REQUIRES_NON_EMPTY_OBJECT = "job.config.update.requires.non.empty.object"; + public static final String JOB_CONFIG_UPDATE_PARSE_ERROR = "job.config.update.parse.error"; + public static final String JOB_CONFIG_UPDATE_BACKGROUND_PERSIST_INTERVAL_INVALID = "job.config.update.background.persist.interval." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_RENORMALIZATION_WINDOW_DAYS_INVALID = "job.config.update.renormalization.window.days." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_MODEL_SNAPSHOT_RETENTION_DAYS_INVALID = "job.config.update.model.snapshot.retention.days." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_RESULTS_RETENTION_DAYS_INVALID = "job.config.update.results.retention.days.invalid"; + public static final String JOB_CONFIG_UPDATE_SCHEDULE_CONFIG_PARSE_ERROR = "job.config.update.scheduler.config.parse.error"; + public static final String JOB_CONFIG_UPDATE_SCHEDULE_CONFIG_CANNOT_BE_NULL = "job.config.update.scheduler.config.cannot.be.null"; + public static final String JOB_CONFIG_UPDATE_SCHEDULE_CONFIG_DATA_SOURCE_INVALID = "job.config.update.scheduler.config.data.source." + + "invalid"; + + public static final String JOB_CONFIG_TRANSFORM_CIRCULAR_DEPENDENCY = "job.config.transform.circular.dependency"; + public static final String JOB_CONFIG_TRANSFORM_CONDITION_REQUIRED = "job.config.transform.condition.required"; + public static final String JOB_CONFIG_TRANSFORM_DUPLICATED_OUTPUT_NAME = "job.config.transform.duplicated.output.name"; + public static final String JOB_CONFIG_TRANSFORM_EXTRACT_GROUPS_SHOULD_MATCH_OUTPUT_COUNT = "job.config.transform.extract.groups.should." + + "match.output.count"; + public static final String JOB_CONFIG_TRANSFORM_INPUTS_CONTAIN_EMPTY_STRING = "job.config.transform.inputs.contain.empty.string"; + public static final String JOB_CONFIG_TRANSFORM_INVALID_ARGUMENT = "job.config.transform.invalid.argument"; + public static final String JOB_CONFIG_TRANSFORM_INVALID_ARGUMENT_COUNT = "job.config.transform.invalid.argument.count"; + public static final String JOB_CONFIG_TRANSFORM_INVALID_INPUT_COUNT = "job.config.transform.invalid.input.count"; + public static final String JOB_CONFIG_TRANSFORM_INVALID_OUTPUT_COUNT = "job.config.transform.invalid.output.count"; + public static final String JOB_CONFIG_TRANSFORM_OUTPUTS_CONTAIN_EMPTY_STRING = "job.config.transform.outputs.contain.empty.string"; + public static final String JOB_CONFIG_TRANSFORM_OUTPUTS_UNUSED = "job.config.transform.outputs.unused"; + public static final String JOB_CONFIG_TRANSFORM_OUTPUT_NAME_USED_MORE_THAN_ONCE = "job.config.transform.output.name.used.more.than" + + ".once"; + public static final String JOB_CONFIG_TRANSFORM_UNKNOWN_TYPE = "job.config.transform.unknown.type"; + public static final String JOB_CONFIG_UNKNOWN_FUNCTION = "job.config.unknown.function"; + + public static final String JOB_CONFIG_SCHEDULER_UNKNOWN_DATASOURCE = "job.config.scheduler.unknown.datasource"; + public static final String JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED = "job.config.scheduler.field.not.supported"; + public static final String JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE = "job.config.scheduler.invalid.option.value"; + public static final String JOB_CONFIG_SCHEDULER_REQUIRES_BUCKET_SPAN = "job.config.scheduler.requires.bucket.span"; + public static final String JOB_CONFIG_SCHEDULER_ELASTICSEARCH_DOES_NOT_SUPPORT_LATENCY = "job.config.scheduler.elasticsearch.does.not." + + "support.latency"; + public static final String JOB_CONFIG_SCHEDULER_AGGREGATIONS_REQUIRES_SUMMARY_COUNT_FIELD = "job.config.scheduler.aggregations." + + "requires.summary.count.field"; + public static final String JOB_CONFIG_SCHEDULER_ELASTICSEARCH_REQUIRES_DATAFORMAT_ELASTICSEARCH = "job.config.scheduler.elasticsearch." + + "requires.dataformat.elasticsearch"; + public static final String JOB_CONFIG_SCHEDULER_INCOMPLETE_CREDENTIALS = "job.config.scheduler.incomplete.credentials"; + public static final String JOB_CONFIG_SCHEDULER_MULTIPLE_PASSWORDS = "job.config.scheduler.multiple.passwords"; + public static final String JOB_CONFIG_SCHEDULER_MULTIPLE_AGGREGATIONS = "job.config.scheduler.multiple.aggregations"; + + public static final String JOB_DATA_CONCURRENT_USE_CLOSE = "job.data.concurrent.use.close"; + public static final String JOB_DATA_CONCURRENT_USE_DELETE = "job.data.concurrent.use.delete"; + public static final String JOB_DATA_CONCURRENT_USE_FLUSH = "job.data.concurrent.use.flush"; + public static final String JOB_DATA_CONCURRENT_USE_PAUSE = "job.data.concurrent.use.pause"; + public static final String JOB_DATA_CONCURRENT_USE_RESUME = "job.data.concurrent.use.resume"; + public static final String JOB_DATA_CONCURRENT_USE_REVERT = "job.data.concurrent.use.revert"; + public static final String JOB_DATA_CONCURRENT_USE_UPDATE = "job.data.concurrent.use.update"; + public static final String JOB_DATA_CONCURRENT_USE_UPLOAD = "job.data.concurrent.use.upload"; + + public static final String JOB_SCHEDULER_CANNOT_START = "job.scheduler.cannot.start"; + public static final String JOB_SCHEDULER_CANNOT_STOP_IN_CURRENT_STATE = "job.scheduler.cannot.stop.in.current.state"; + public static final String JOB_SCHEDULER_CANNOT_UPDATE_IN_CURRENT_STATE = "job.scheduler.cannot.update.in.current.state"; + public static final String JOB_SCHEDULER_FAILED_TO_STOP = "job.scheduler.failed.to.stop"; + public static final String JOB_SCHEDULER_NO_SUCH_SCHEDULED_JOB = "job.scheduler.no.such.scheduled.job"; + public static final String JOB_SCHEDULER_STATUS_STARTED = "job.scheduler.status.started"; + public static final String JOB_SCHEDULER_STATUS_STOPPING = "job.scheduler.status.stopping"; + public static final String JOB_SCHEDULER_STATUS_STOPPED = "job.scheduler.status.stopped"; + public static final String JOB_SCHEDULER_STATUS_UPDATING = "job.scheduler.status.updating"; + public static final String JOB_SCHEDULER_STATUS_DELETING = "job.scheduler.status.deleting"; + + public static final String JOB_MISSING_QUANTILES = "job.missing.quantiles"; + public static final String JOB_UNKNOWN_ID = "job.unknown.id"; + + public static final String JSON_JOB_CONFIG_MAPPING = "json.job.config.mapping.error"; + public static final String JSON_JOB_CONFIG_PARSE = "json.job.config.parse.error"; + + public static final String JSON_DETECTOR_CONFIG_MAPPING = "json.detector.config.mapping.error"; + public static final String JSON_DETECTOR_CONFIG_PARSE = "json.detector.config.parse.error"; + + public static final String JSON_LIST_DOCUMENT_MAPPING_ERROR = "json.list.document.mapping.error"; + public static final String JSON_LIST_DOCUMENT_PARSE_ERROR = "json.list.document.parse.error"; + + public static final String JSON_TRANSFORM_CONFIG_MAPPING = "json.transform.config.mapping.error"; + public static final String JSON_TRANSFORM_CONFIG_PARSE = "json.transform.config.parse.error"; + + public static final String ON_HOST = "on.host"; + + public static final String REST_ACTION_NOT_ALLOWED_FOR_SCHEDULED_JOB = "rest.action.not.allowed.for.scheduled.job"; + + public static final String REST_INVALID_DATETIME_PARAMS = "rest.invalid.datetime.params"; + public static final String REST_INVALID_FLUSH_PARAMS_MISSING = "rest.invalid.flush.params.missing.argument"; + public static final String REST_INVALID_FLUSH_PARAMS_UNEXPECTED = "rest.invalid.flush.params.unexpected"; + public static final String REST_INVALID_RESET_PARAMS = "rest.invalid.reset.params"; + public static final String REST_INVALID_FROM = "rest.invalid.from"; + public static final String REST_INVALID_SIZE = "rest.invalid.size"; + public static final String REST_INVALID_FROM_SIZE_SUM = "rest.invalid.from.size.sum"; + public static final String REST_GZIP_ERROR = "rest.gzip.error"; + public static final String REST_START_AFTER_END = "rest.start.after.end"; + public static final String REST_RESET_BUCKET_NO_LATENCY = "rest.reset.bucket.no.latency"; + public static final String REST_INVALID_REVERT_PARAMS = "rest.invalid.revert.params"; + public static final String REST_JOB_NOT_CLOSED_REVERT = "rest.job.not.closed.revert"; + public static final String REST_NO_SUCH_MODEL_SNAPSHOT = "rest.no.such.model.snapshot"; + public static final String REST_INVALID_DESCRIPTION_PARAMS = "rest.invalid.description.params"; + public static final String REST_DESCRIPTION_ALREADY_USED = "rest.description.already.used"; + public static final String REST_CANNOT_DELETE_HIGHEST_PRIORITY = "rest.cannot.delete.highest.priority"; + + public static final String REST_ALERT_MISSING_ARGUMENT = "rest.alert.missing.argument"; + public static final String REST_ALERT_INVALID_TIMEOUT = "rest.alert.invalid.timeout"; + public static final String REST_ALERT_INVALID_THRESHOLD = "rest.alert.invalid.threshold"; + public static final String REST_ALERT_CANT_USE_PROB = "rest.alert.cant.use.prob"; + public static final String REST_ALERT_INVALID_TYPE = "rest.alert.invalid.type"; + + public static final String PROCESS_ACTION_SLEEPING_JOB = "process.action.sleeping.job"; + public static final String PROCESS_ACTION_CLOSED_JOB = "process.action.closed.job"; + public static final String PROCESS_ACTION_CLOSING_JOB = "process.action.closing.job"; + public static final String PROCESS_ACTION_DELETING_JOB = "process.action.deleting.job"; + public static final String PROCESS_ACTION_FLUSHING_JOB = "process.action.flushing.job"; + public static final String PROCESS_ACTION_PAUSING_JOB = "process.action.pausing.job"; + public static final String PROCESS_ACTION_RESUMING_JOB = "process.action.resuming.job"; + public static final String PROCESS_ACTION_REVERTING_JOB = "process.action.reverting.job"; + public static final String PROCESS_ACTION_UPDATING_JOB = "process.action.updating.job"; + public static final String PROCESS_ACTION_WRITING_JOB = "process.action.writing.job"; + + public static final String SUPPORT_BUNDLE_SCRIPT_ERROR = "support.bundle.script.error"; + + private Messages() + { + } + + public static ResourceBundle load() + { + return ResourceBundle.getBundle(Messages.BUNDLE_NAME, Locale.getDefault()); + } + + /** + * Look up the message string from the resource bundle. + * + * @param key Must be one of the statics defined in this file] + */ + public static String getMessage(String key) + { + return load().getString(key); + } + + /** + * Look up the message string from the resource bundle and format with + * the supplied arguments + * @param key the key for the message + * @param args MessageFormat arguments. See {@linkplain MessageFormat#format(Object)}] + */ + public static String getMessage(String key, Object...args) + { + return new MessageFormat(load().getString(key), Locale.ROOT).format(args); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/Allocation.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/Allocation.java new file mode 100644 index 00000000000..55dbc9ee3de --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/Allocation.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class Allocation extends AbstractDiffable implements ToXContent { + + private static final ParseField NODE_ID_FIELD = new ParseField("node_id"); + private static final ParseField JOB_ID_FIELD = new ParseField("job_id"); + public static final ParseField STATUS = new ParseField("status"); + public static final ParseField SCHEDULER_STATE = new ParseField("scheduler_state"); + + static final Allocation PROTO = new Allocation(null, null, null, null); + + static final ObjectParser PARSER = new ObjectParser<>("allocation", Builder::new); + + static { + PARSER.declareString(Builder::setNodeId, NODE_ID_FIELD); + PARSER.declareString(Builder::setJobId, JOB_ID_FIELD); + PARSER.declareField(Builder::setStatus, (p, c) -> JobStatus.fromString(p.text()), STATUS, ObjectParser.ValueType.STRING); + PARSER.declareObject(Builder::setSchedulerState, SchedulerState.PARSER, SCHEDULER_STATE); + } + + private final String nodeId; + private final String jobId; + private final JobStatus status; + private final SchedulerState schedulerState; + + public Allocation(String nodeId, String jobId, JobStatus status, SchedulerState schedulerState) { + this.nodeId = nodeId; + this.jobId = jobId; + this.status = status; + this.schedulerState = schedulerState; + } + + public Allocation(StreamInput in) throws IOException { + this.nodeId = in.readString(); + this.jobId = in.readString(); + this.status = JobStatus.fromStream(in); + this.schedulerState = in.readOptionalWriteable(SchedulerState::new); + } + + public String getNodeId() { + return nodeId; + } + + public String getJobId() { + return jobId; + } + + public JobStatus getStatus() { + return status; + } + + public SchedulerState getSchedulerState() { + return schedulerState; + } + + @Override + public Allocation readFrom(StreamInput in) throws IOException { + return new Allocation(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(nodeId); + out.writeString(jobId); + status.writeTo(out); + out.writeOptionalWriteable(schedulerState); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(NODE_ID_FIELD.getPreferredName(), nodeId); + builder.field(JOB_ID_FIELD.getPreferredName(), jobId); + builder.field(STATUS.getPreferredName(), status); + if (schedulerState != null) { + builder.field(SCHEDULER_STATE.getPreferredName(), schedulerState); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Allocation that = (Allocation) o; + return Objects.equals(nodeId, that.nodeId) && + Objects.equals(jobId, that.jobId) && + Objects.equals(status, that.status) && + Objects.equals(schedulerState, that.schedulerState); + } + + @Override + public int hashCode() { + return Objects.hash(nodeId, jobId, status, schedulerState); + } + + // Class alreadt extends from AbstractDiffable, so copied from ToXContentToBytes#toString() + @SuppressWarnings("deprecation") + @Override + public final String toString() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.prettyPrint(); + toXContent(builder, EMPTY_PARAMS); + return builder.string(); + } catch (Exception e) { + // So we have a stack trace logged somewhere + return "{ \"error\" : \"" + org.elasticsearch.ExceptionsHelper.detailedMessage(e) + "\"}"; + } + } + + public static class Builder { + + private String nodeId; + private String jobId; + private JobStatus status = JobStatus.CLOSED; + private SchedulerState schedulerState; + + public Builder() { + } + + public Builder(Allocation allocation) { + this.nodeId = allocation.nodeId; + this.jobId = allocation.jobId; + this.status = allocation.status; + this.schedulerState = allocation.schedulerState; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public void setStatus(JobStatus status) { + this.status = status; + } + + public void setSchedulerState(SchedulerState schedulerState) { + JobSchedulerStatus currentSchedulerStatus = this.schedulerState == null ? + JobSchedulerStatus.STOPPED : this.schedulerState.getStatus(); + JobSchedulerStatus newSchedulerStatus = schedulerState.getStatus(); + switch (newSchedulerStatus) { + case STARTING: + if (currentSchedulerStatus != JobSchedulerStatus.STOPPED) { + String msg = Messages.getMessage(Messages.JOB_SCHEDULER_CANNOT_START, jobId, newSchedulerStatus); + throw ExceptionsHelper.conflictStatusException(msg); + } + break; + case STARTED: + if (currentSchedulerStatus != JobSchedulerStatus.STARTING) { + String msg = Messages.getMessage(Messages.JOB_SCHEDULER_CANNOT_START, jobId, newSchedulerStatus); + throw ExceptionsHelper.conflictStatusException(msg); + } + break; + case STOPPING: + if (currentSchedulerStatus != JobSchedulerStatus.STARTED) { + String msg = Messages.getMessage(Messages.JOB_SCHEDULER_CANNOT_STOP_IN_CURRENT_STATE, jobId, newSchedulerStatus); + throw ExceptionsHelper.conflictStatusException(msg); + } + break; + case STOPPED: + if (currentSchedulerStatus != JobSchedulerStatus.STOPPING) { + String msg = Messages.getMessage(Messages.JOB_SCHEDULER_CANNOT_STOP_IN_CURRENT_STATE, jobId, newSchedulerStatus); + throw ExceptionsHelper.conflictStatusException(msg); + } + break; + default: + throw new IllegalArgumentException("Invalid requested job scheduler status: " + newSchedulerStatus); + } + + this.schedulerState = schedulerState; + } + + public Allocation build() { + return new Allocation(nodeId, jobId, status, schedulerState); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocator.java new file mode 100644 index 00000000000..0ae90cb6aa0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocator.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; + +/** + * Runs only on the elected master node and decides to what nodes jobs should be allocated + */ +public class JobAllocator extends AbstractComponent implements ClusterStateListener { + + private final ThreadPool threadPool; + private final ClusterService clusterService; + + public JobAllocator(Settings settings, ClusterService clusterService, ThreadPool threadPool) { + super(settings); + this.threadPool = threadPool; + this.clusterService = clusterService; + clusterService.add(this); + } + + ClusterState allocateJobs(ClusterState current) { + if (shouldAllocate(current) == false) { + // returning same instance, so no cluster state update is performed + return current; + } + + DiscoveryNodes nodes = current.getNodes(); + if (nodes.getSize() != 1) { + throw new IllegalStateException("Current prelert doesn't support multiple nodes"); + } + + // NORELEASE: Assumes prelert always runs on a single node: + PrelertMetadata prelertMetadata = current.getMetaData().custom(PrelertMetadata.TYPE); + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(prelertMetadata); + DiscoveryNode prelertNode = nodes.getMasterNode(); // prelert is now always master node + + for (String jobId : prelertMetadata.getJobs().keySet()) { + if (prelertMetadata.getAllocations().containsKey(jobId) == false) { + builder.putAllocation(prelertNode.getId(), jobId); + } + } + + return ClusterState.builder(current) + .metaData(MetaData.builder(current.metaData()).putCustom(PrelertMetadata.TYPE, builder.build())) + .build(); + } + + boolean shouldAllocate(ClusterState current) { + PrelertMetadata prelertMetadata = current.getMetaData().custom(PrelertMetadata.TYPE); + for (String jobId : prelertMetadata.getJobs().keySet()) { + if (prelertMetadata.getAllocations().containsKey(jobId) == false) { + return true; + } + } + return false; + } + + boolean prelertMetaDataMissing(ClusterState clusterState) { + return clusterState.getMetaData().custom(PrelertMetadata.TYPE) == null; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster()) { + if (prelertMetaDataMissing(event.state())) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + clusterService.submitStateUpdateTask("install-prelert-metadata", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState.Builder builder = new ClusterState.Builder(currentState); + MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); + metadataBuilder.putCustom(PrelertMetadata.TYPE, PrelertMetadata.PROTO); + builder.metaData(metadataBuilder.build()); + return builder.build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.error("unable to install prelert metadata upon startup", e); + } + }); + }); + } else if (shouldAllocate(event.state())) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + clusterService.submitStateUpdateTask("allocate_jobs", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return allocateJobs(currentState); + } + + @Override + public void onFailure(String source, Exception e) { + logger.error("failed to allocate jobs", e); + } + }); + }); + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleService.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleService.java new file mode 100644 index 00000000000..57d4b482c0f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleService.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.prelert.action.UpdateJobStatusAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.scheduler.ScheduledJobService; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.Executor; + +public class JobLifeCycleService extends AbstractComponent implements ClusterStateListener { + + volatile Set localAllocatedJobs = Collections.emptySet(); + private final Client client; + private final ScheduledJobService scheduledJobService; + private DataProcessor dataProcessor; + private final Executor executor; + + public JobLifeCycleService(Settings settings, Client client, ClusterService clusterService, ScheduledJobService scheduledJobService, + DataProcessor dataProcessor, Executor executor) { + super(settings); + clusterService.add(this); + this.client = Objects.requireNonNull(client); + this.scheduledJobService = Objects.requireNonNull(scheduledJobService); + this.dataProcessor = Objects.requireNonNull(dataProcessor); + this.executor = Objects.requireNonNull(executor); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + PrelertMetadata prelertMetadata = event.state().getMetaData().custom(PrelertMetadata.TYPE); + if (prelertMetadata == null) { + logger.debug("Prelert metadata not installed"); + return; + } + + // Single volatile read: + Set localAllocatedJobs = this.localAllocatedJobs; + + DiscoveryNode localNode = event.state().nodes().getLocalNode(); + for (Allocation allocation : prelertMetadata.getAllocations().values()) { + if (localNode.getId().equals(allocation.getNodeId())) { + handleLocallyAllocatedJob(prelertMetadata, allocation); + } + } + + for (String localAllocatedJob : localAllocatedJobs) { + Allocation allocation = prelertMetadata.getAllocations().get(localAllocatedJob); + if (allocation != null) { + if (localNode.getId().equals(allocation.getNodeId()) == false) { + stopJob(localAllocatedJob); + } + } else { + stopJob(localAllocatedJob); + } + } + } + + private void handleLocallyAllocatedJob(PrelertMetadata prelertMetadata, Allocation allocation) { + Job job = prelertMetadata.getJobs().get(allocation.getJobId()); + if (localAllocatedJobs.contains(allocation.getJobId()) == false) { + startJob(job); + } + + handleJobStatusChange(job, allocation.getStatus()); + handleSchedulerStatusChange(job, allocation); + } + + private void handleJobStatusChange(Job job, JobStatus status) { + switch (status) { + case PAUSING: + executor.execute(() -> pauseJob(job)); + break; + case RUNNING: + break; + case CLOSING: + break; + case CLOSED: + break; + case PAUSED: + break; + default: + throw new IllegalStateException("Unknown job status [" + status + "]"); + } + } + + private void handleSchedulerStatusChange(Job job, Allocation allocation) { + SchedulerState schedulerState = allocation.getSchedulerState(); + if (schedulerState != null) { + switch (schedulerState.getStatus()) { + case STARTING: + scheduledJobService.start(job, allocation); + break; + case STARTED: + break; + case STOPPING: + scheduledJobService.stop(allocation); + break; + case STOPPED: + break; + default: + throw new IllegalStateException("Unhandled scheduler state [" + schedulerState.getStatus() + "]"); + } + } + } + + void startJob(Job job) { + logger.info("Starting job [" + job.getId() + "]"); + // noop now, but should delegate to a task / ProcessManager that actually starts the job + + // update which jobs are now allocated locally + Set newSet = new HashSet<>(localAllocatedJobs); + newSet.add(job.getId()); + localAllocatedJobs = newSet; + } + + void stopJob(String jobId) { + logger.info("Stopping job [" + jobId + "]"); + // noop now, but should delegate to a task / ProcessManager that actually stops the job + + // update which jobs are now allocated locally + Set newSet = new HashSet<>(localAllocatedJobs); + newSet.remove(jobId); + localAllocatedJobs = newSet; + } + + private void pauseJob(Job job) { + try { + // NORELEASE Ensure this also removes the job auto-close timeout task + dataProcessor.closeJob(job.getId()); + } catch (ElasticsearchException e) { + logger.error("Failed to close job [" + job.getId() + "] while pausing", e); + updateJobStatus(job.getId(), JobStatus.FAILED); + return; + } + updateJobStatus(job.getId(), JobStatus.PAUSED); + } + + private void updateJobStatus(String jobId, JobStatus status) { + UpdateJobStatusAction.Request request = new UpdateJobStatusAction.Request(jobId, status); + client.execute(UpdateJobStatusAction.INSTANCE, request, new ActionListener() { + @Override + public void onResponse(UpdateJobStatusAction.Response response) { + logger.info("Successfully set job status to [{}] for job [{}]", status, jobId); + // NORELEASE Audit job paused + // audit(jobId).info(Messages.getMessage(Messages.JOB_AUDIT_PAUSED)); + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not set job status to [" + status + "] for job [" + jobId +"]", e); + } + }); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java new file mode 100644 index 00000000000..d6ecbeb3217 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadata.java @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.DiffableUtils; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; +import java.util.Objects; +import java.util.SortedMap; +import java.util.TreeMap; + +public class PrelertMetadata implements MetaData.Custom { + + private static final ParseField JOBS_FIELD = new ParseField("jobs"); + private static final ParseField ALLOCATIONS_FIELD = new ParseField("allocations"); + + public static final String TYPE = "prelert"; + public static final PrelertMetadata PROTO = new PrelertMetadata(Collections.emptySortedMap(), Collections.emptySortedMap()); + + private static final ObjectParser PRELERT_METADATA_PARSER = new ObjectParser<>("prelert_metadata", + Builder::new); + + static { + PRELERT_METADATA_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.PARSER.apply(p, c).build(), JOBS_FIELD); + PRELERT_METADATA_PARSER.declareObjectArray(Builder::putAllocations, Allocation.PARSER, ALLOCATIONS_FIELD); + } + + // NORELEASE: A few fields of job details change frequently and this needs to be stored elsewhere + // performance issue will occur if we don't change that + private final SortedMap jobs; + private final SortedMap allocations; + + private PrelertMetadata(SortedMap jobs, SortedMap allocations) { + this.jobs = Collections.unmodifiableSortedMap(jobs); + this.allocations = Collections.unmodifiableSortedMap(allocations); + } + + public Map getJobs() { + // NORELEASE jobs should be immutable or a job can be modified in the + // cluster state of a single node without a cluster state update + return jobs; + } + + public SortedMap getAllocations() { + return allocations; + } + + @Override + public String type() { + return TYPE; + } + + @Override + public MetaData.Custom fromXContent(XContentParser parser) throws IOException { + return PRELERT_METADATA_PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build(); + } + + @Override + public EnumSet context() { + // NORELEASE: Also include SNAPSHOT, but then we need to split the allocations from here and add them + // as ClusterState.Custom metadata, because only the job definitions should be stored in snapshots. + return MetaData.API_AND_GATEWAY; + } + + @Override + public Diff diff(MetaData.Custom previousState) { + return new PrelertMetadataDiff((PrelertMetadata) previousState, this); + } + + @Override + public Diff readDiffFrom(StreamInput in) throws IOException { + return new PrelertMetadataDiff(in); + } + + @Override + public MetaData.Custom readFrom(StreamInput in) throws IOException { + int size = in.readVInt(); + TreeMap jobs = new TreeMap<>(); + for (int i = 0; i < size; i++) { + jobs.put(in.readString(), new Job(in)); + } + size = in.readVInt(); + TreeMap allocations = new TreeMap<>(); + for (int i = 0; i < size; i++) { + allocations.put(in.readString(), Allocation.PROTO.readFrom(in)); + } + return new PrelertMetadata(jobs, allocations); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(jobs.size()); + for (Map.Entry entry : jobs.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + out.writeVInt(allocations.size()); + for (Map.Entry entry : allocations.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray(JOBS_FIELD.getPreferredName()); + for (Job job : jobs.values()) { + builder.value(job); + } + builder.endArray(); + builder.startArray(ALLOCATIONS_FIELD.getPreferredName()); + for (Map.Entry entry : allocations.entrySet()) { + builder.value(entry.getValue()); + } + builder.endArray(); + return builder; + } + + static class PrelertMetadataDiff implements Diff { + + final Diff> jobs; + final Diff> allocations; + + PrelertMetadataDiff(PrelertMetadata before, PrelertMetadata after) { + this.jobs = DiffableUtils.diff(before.jobs, after.jobs, DiffableUtils.getStringKeySerializer()); + this.allocations = DiffableUtils.diff(before.allocations, after.allocations, DiffableUtils.getStringKeySerializer()); + } + + PrelertMetadataDiff(StreamInput in) throws IOException { + jobs = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Job.PROTO); + allocations = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Allocation.PROTO); + } + + @Override + public MetaData.Custom apply(MetaData.Custom part) { + TreeMap newJobs = new TreeMap<>(jobs.apply(((PrelertMetadata) part).jobs)); + TreeMap newAllocations = new TreeMap<>(allocations.apply(((PrelertMetadata) part).allocations)); + return new PrelertMetadata(newJobs, newAllocations); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + jobs.writeTo(out); + allocations.writeTo(out); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + PrelertMetadata that = (PrelertMetadata) o; + return Objects.equals(jobs, that.jobs) && Objects.equals(allocations, that.allocations); + } + + @Override + public int hashCode() { + return Objects.hash(jobs, allocations); + } + + public static class Builder { + + private TreeMap jobs; + private TreeMap allocations; + + public Builder() { + this.jobs = new TreeMap<>(); + this.allocations = new TreeMap<>(); + } + + public Builder(PrelertMetadata previous) { + jobs = new TreeMap<>(previous.jobs); + allocations = new TreeMap<>(previous.allocations); + } + + public Builder putJob(Job job, boolean overwrite) { + if (jobs.containsKey(job.getId()) && overwrite == false) { + throw ExceptionsHelper.jobAlreadyExists(job.getId()); + } + this.jobs.put(job.getId(), job); + return this; + } + + public Builder removeJob(String jobId) { + if (jobs.remove(jobId) == null) { + throw new ResourceNotFoundException("job [" + jobId + "] does not exist"); + } + this.allocations.remove(jobId); + return this; + } + + public Builder putAllocation(String nodeId, String jobId) { + Allocation.Builder builder = new Allocation.Builder(); + builder.setJobId(jobId); + builder.setNodeId(nodeId); + this.allocations.put(jobId, builder.build()); + return this; + } + + public Builder updateAllocation(String jobId, Allocation updated) { + Allocation previous = this.allocations.put(jobId, updated); + if (previous == null) { + throw new IllegalStateException("Expected that job [" + jobId + "] was already allocated"); + } + return this; + } + + // only for parsing + private Builder putAllocations(Collection allocations) { + for (Allocation.Builder allocationBuilder : allocations) { + Allocation allocation = allocationBuilder.build(); + this.allocations.put(allocation.getJobId(), allocation); + } + return this; + } + + private Builder putJobs(Collection jobs) { + for (Job job : jobs) { + putJob(job, true); + } + return this; + } + + public PrelertMetadata build() { + return new PrelertMetadata(jobs, allocations); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BatchedDocumentsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BatchedDocumentsIterator.java new file mode 100644 index 00000000000..933c881a680 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BatchedDocumentsIterator.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import java.util.Deque; +import java.util.NoSuchElementException; + +/** + * An iterator useful to fetch a big number of documents of type T + * and iterate through them in batches. + */ +public interface BatchedDocumentsIterator +{ + /** + * Query documents whose timestamp is within the given time range + * + * @param startEpochMs the start time as epoch milliseconds (inclusive) + * @param endEpochMs the end time as epoch milliseconds (exclusive) + * @return the iterator itself + */ + BatchedDocumentsIterator timeRange(long startEpochMs, long endEpochMs); + + /** + * Include interim documents + * + * @param interimFieldName Name of the include interim field + */ + BatchedDocumentsIterator includeInterim(String interimFieldName); + + /** + * The first time next() is called, the search will be performed and the first + * batch will be returned. Any subsequent call will return the following batches. + *

+ * Note that in some implementations it is possible that when there are no + * results at all, the first time this method is called an empty {@code Deque} is returned. + * + * @return a {@code Deque} with the next batch of documents + * @throws NoSuchElementException if the iteration has no more elements + */ + Deque next(); + + /** + * Returns {@code true} if the iteration has more elements. + * (In other words, returns {@code true} if {@link #next} would + * return an element rather than throwing an exception.) + * + * @return {@code true} if the iteration has more elements + */ + boolean hasNext(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilder.java new file mode 100644 index 00000000000..90594d032cb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilder.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.common.Strings; + +import java.util.Objects; + +/** + * One time query builder for a single buckets. + *

    + *
  • Timestamp (Required) - Timestamp of the bucket
    • + *
  • Expand- Include anomaly records. Default= false
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • partitionValue Set the bucket's max normalised probabiltiy to this + * partiton field value's max normalised probability. Default = null
    • + *
+ */ +public final class BucketQueryBuilder { + public static int DEFAULT_SIZE = 100; + + private BucketQuery bucketQuery; + + public BucketQueryBuilder(String timestamp) { + bucketQuery = new BucketQuery(timestamp); + } + + public BucketQueryBuilder expand(boolean expand) { + bucketQuery.expand = expand; + return this; + } + + public BucketQueryBuilder includeInterim(boolean include) { + bucketQuery.includeInterim = include; + return this; + } + + /** + * partitionValue must be non null and not empty else it + * is not set + */ + public BucketQueryBuilder partitionValue(String partitionValue) { + if (!Strings.isNullOrEmpty(partitionValue)) { + bucketQuery.partitionValue = partitionValue; + } + return this; + } + + public BucketQueryBuilder.BucketQuery build() { + return bucketQuery; + } + + public class BucketQuery { + private String timestamp; + private boolean expand = false; + private boolean includeInterim = false; + private String partitionValue = null; + + public BucketQuery(String timestamp) { + this.timestamp = timestamp; + } + + public String getTimestamp() { + return timestamp; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public boolean isExpand() { + return expand; + } + + /** + * @return Null if not set + */ + public String getPartitionValue() { + return partitionValue; + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, expand, includeInterim, + partitionValue); + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + BucketQuery other = (BucketQuery) obj; + return Objects.equals(timestamp, other.timestamp) && + Objects.equals(expand, other.expand) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(partitionValue, other.partitionValue); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilder.java new file mode 100644 index 00000000000..c24eb392ff2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilder.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.common.Strings; + +import java.util.Objects; + +/** + * One time query builder for buckets. + *
    + *
  • From- Skip the first N Buckets. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of Buckets. Default = + * {@value DEFAULT_SIZE}
    • + *
  • Expand- Include anomaly records. Default= false
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • normalizedProbabilityThreshold- Return only buckets with a + * maxNormalizedProbability >= this value. Default = 0.0
    • + *
  • epochStart- The start bucket time. A bucket with this timestamp will be + * included in the results. If 0 all buckets up to endEpochMs are + * returned. Default = -1
    • + *
  • epochEnd- The end bucket timestamp buckets up to but NOT including this + * timestamp are returned. If 0 all buckets from startEpochMs are + * returned. Default = -1
    • + *
  • partitionValue Set the bucket's max normalised probability to this + * partition field value's max normalised probability. Default = null
    • + *
+ */ +public final class BucketsQueryBuilder { + public static final int DEFAULT_SIZE = 100; + + private BucketsQuery bucketsQuery = new BucketsQuery(); + + public BucketsQueryBuilder from(int from) { + bucketsQuery.from = from; + return this; + } + + public BucketsQueryBuilder size(int size) { + bucketsQuery.size = size; + return this; + } + + public BucketsQueryBuilder expand(boolean expand) { + bucketsQuery.expand = expand; + return this; + } + + public BucketsQueryBuilder includeInterim(boolean include) { + bucketsQuery.includeInterim = include; + return this; + } + + public BucketsQueryBuilder anomalyScoreThreshold(Double anomalyScoreFilter) { + bucketsQuery.anomalyScoreFilter = anomalyScoreFilter; + return this; + } + + public BucketsQueryBuilder normalizedProbabilityThreshold(Double normalizedProbability) { + bucketsQuery.normalizedProbability = normalizedProbability; + return this; + } + + /** + * @param partitionValue Not set if null or empty + */ + public BucketsQueryBuilder partitionValue(String partitionValue) { + if (!Strings.isNullOrEmpty(partitionValue)) { + bucketsQuery.partitionValue = partitionValue; + } + return this; + } + + public BucketsQueryBuilder sortField(String sortField) { + bucketsQuery.sortField = sortField; + return this; + } + + public BucketsQueryBuilder sortDescending(boolean sortDescending) { + bucketsQuery.sortDescending = sortDescending; + return this; + } + + /** + * If startTime <= 0 the parameter is not set + */ + public BucketsQueryBuilder epochStart(String startTime) { + bucketsQuery.epochStart = startTime; + return this; + } + + /** + * If endTime <= 0 the parameter is not set + */ + public BucketsQueryBuilder epochEnd(String endTime) { + bucketsQuery.epochEnd = endTime; + return this; + } + + public BucketsQueryBuilder.BucketsQuery build() { + return bucketsQuery; + } + + public void clear() { + bucketsQuery = new BucketsQueryBuilder.BucketsQuery(); + } + + + public class BucketsQuery { + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean expand = false; + private boolean includeInterim = false; + private double anomalyScoreFilter = 0.0d; + private double normalizedProbability = 0.0d; + private String epochStart; + private String epochEnd; + private String partitionValue = null; + private String sortField = Bucket.TIMESTAMP.getPreferredName(); + private boolean sortDescending = false; + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + public boolean isExpand() { + return expand; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public double getNormalizedProbability() { + return normalizedProbability; + } + + public String getEpochStart() { + return epochStart; + } + + public String getEpochEnd() { + return epochEnd; + } + + /** + * @return Null if not set + */ + public String getPartitionValue() { + return partitionValue; + } + + public String getSortField() { + return sortField; + } + + public boolean isSortDescending() { + return sortDescending; + } + + @Override + public int hashCode() { + return Objects.hash(from, size, expand, includeInterim, anomalyScoreFilter, normalizedProbability, epochStart, epochEnd, + partitionValue, sortField, sortDescending); + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + BucketsQuery other = (BucketsQuery) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size) && + Objects.equals(expand, other.expand) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(epochStart, other.epochStart) && + Objects.equals(epochStart, other.epochStart) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(normalizedProbability, other.normalizedProbability) && + Objects.equals(partitionValue, other.partitionValue) && + Objects.equals(sortField, other.sortField) && + this.sortDescending == other.sortDescending; + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditor.java new file mode 100644 index 00000000000..c0455bb3efd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditor.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.xpack.prelert.job.audit.AuditActivity; +import org.elasticsearch.xpack.prelert.job.audit.AuditMessage; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class ElasticsearchAuditor implements Auditor +{ + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchAuditor.class); + + private final Client client; + private final String index; + private final String jobId; + + public ElasticsearchAuditor(Client client, String index, String jobId) + { + this.client = Objects.requireNonNull(client); + this.index = index; + this.jobId = jobId; + } + + @Override + public void info(String message) + { + persistAuditMessage(AuditMessage.newInfo(jobId, message)); + } + + @Override + public void warning(String message) + { + persistAuditMessage(AuditMessage.newWarning(jobId, message)); + } + + @Override + public void error(String message) + { + persistAuditMessage(AuditMessage.newError(jobId, message)); + } + + @Override + public void activity(String message) + { + persistAuditMessage(AuditMessage.newActivity(jobId, message)); + } + + @Override + public void activity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) + { + persistAuditActivity(AuditActivity.newActivity(totalJobs, totalDetectors, runningJobs, runningDetectors)); + } + + private void persistAuditMessage(AuditMessage message) + { + try + { + client.prepareIndex(index, AuditMessage.TYPE.getPreferredName()) + .setSource(serialiseMessage(message)) + .execute().actionGet(); + } + catch (IOException | IndexNotFoundException e) + { + LOGGER.error("Error writing auditMessage", e); + } + } + + private void persistAuditActivity(AuditActivity activity) + { + try + { + client.prepareIndex(index, AuditActivity.TYPE.getPreferredName()) + .setSource(serialiseActivity(activity)) + .execute().actionGet(); + } + catch (IOException | IndexNotFoundException e) + { + LOGGER.error("Error writing auditActivity", e); + } + } + + private XContentBuilder serialiseMessage(AuditMessage message) throws IOException + { + return message.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS); + } + + private XContentBuilder serialiseActivity(AuditActivity activity) throws IOException + { + return activity.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedBucketsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedBucketsIterator.java new file mode 100644 index 00000000000..3a9f3c0a563 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedBucketsIterator.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +import org.elasticsearch.xpack.prelert.job.results.Bucket; + +class ElasticsearchBatchedBucketsIterator extends ElasticsearchBatchedDocumentsIterator +{ + public ElasticsearchBatchedBucketsIterator(Client client, String jobId, ParseFieldMatcher parserFieldMatcher) + { + super(client, ElasticsearchPersister.getJobIndexName(jobId), parserFieldMatcher); + } + + @Override + protected String getType() + { + return Bucket.TYPE.getPreferredName(); + } + + @Override + protected Bucket map(SearchHit hit) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse bucket", e); + } + Bucket bucket = Bucket.PARSER.apply(parser, () -> parseFieldMatcher); + bucket.setId(hit.getId()); + return bucket; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIterator.java new file mode 100644 index 00000000000..749d70efaae --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIterator.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortBuilders; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.NoSuchElementException; +import java.util.Objects; + +abstract class ElasticsearchBatchedDocumentsIterator implements BatchedDocumentsIterator { + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchBatchedDocumentsIterator.class); + + private static final String CONTEXT_ALIVE_DURATION = "5m"; + private static final int BATCH_SIZE = 10000; + + private final Client client; + private final String index; + private final ResultsFilterBuilder filterBuilder; + private volatile long count; + private volatile long totalHits; + private volatile String scrollId; + private volatile boolean isScrollInitialised; + protected ParseFieldMatcher parseFieldMatcher; + + public ElasticsearchBatchedDocumentsIterator(Client client, String index, ParseFieldMatcher parseFieldMatcher) { + this.parseFieldMatcher = parseFieldMatcher; + this.client = Objects.requireNonNull(client); + this.index = Objects.requireNonNull(index); + this.parseFieldMatcher = Objects.requireNonNull(parseFieldMatcher); + totalHits = 0; + count = 0; + filterBuilder = new ResultsFilterBuilder(); + isScrollInitialised = false; + } + + @Override + public BatchedDocumentsIterator timeRange(long startEpochMs, long endEpochMs) { + filterBuilder.timeRange(ElasticsearchMappings.ES_TIMESTAMP, startEpochMs, endEpochMs); + return this; + } + + @Override + public BatchedDocumentsIterator includeInterim(String interimFieldName) { + filterBuilder.interim(interimFieldName, true); + return this; + } + + @Override + public boolean hasNext() { + return !isScrollInitialised || count != totalHits; + } + + @Override + public Deque next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + SearchResponse searchResponse = (scrollId == null) ? initScroll() + : client.prepareSearchScroll(scrollId).setScroll(CONTEXT_ALIVE_DURATION).get(); + scrollId = searchResponse.getScrollId(); + return mapHits(searchResponse); + } + + private SearchResponse initScroll() { + LOGGER.trace("ES API CALL: search all of type " + getType() + " from index " + index); + + isScrollInitialised = true; + + SearchResponse searchResponse = client.prepareSearch(index).setScroll(CONTEXT_ALIVE_DURATION).setSize(BATCH_SIZE) + .setTypes(getType()).setQuery(filterBuilder.build()).addSort(SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC)).get(); + totalHits = searchResponse.getHits().getTotalHits(); + scrollId = searchResponse.getScrollId(); + return searchResponse; + } + + private Deque mapHits(SearchResponse searchResponse) { + Deque results = new ArrayDeque<>(); + + SearchHit[] hits = searchResponse.getHits().getHits(); + for (SearchHit hit : hits) { + T mapped = map(hit); + if (mapped != null) { + results.add(mapped); + } + } + count += hits.length; + + if (!hasNext() && scrollId != null) { + client.prepareClearScroll().setScrollIds(Arrays.asList(scrollId)).get(); + } + return results; + } + + protected abstract String getType(); + + /** + * Maps the search hit to the document type + * @param hit + * the search hit + * @return The mapped document or {@code null} if the mapping failed + */ + protected abstract T map(SearchHit hit); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedInfluencersIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedInfluencersIterator.java new file mode 100644 index 00000000000..41e7fb7480e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedInfluencersIterator.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +import org.elasticsearch.xpack.prelert.job.results.Influencer; + +class ElasticsearchBatchedInfluencersIterator extends ElasticsearchBatchedDocumentsIterator +{ + public ElasticsearchBatchedInfluencersIterator(Client client, String jobId, + ParseFieldMatcher parserFieldMatcher) + { + super(client, ElasticsearchPersister.getJobIndexName(jobId), parserFieldMatcher); + } + + @Override + protected String getType() + { + return Influencer.TYPE.getPreferredName(); + } + + @Override + protected Influencer map(SearchHit hit) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser influencer", e); + } + + Influencer influencer = Influencer.PARSER.apply(parser, () -> parseFieldMatcher); + influencer.setId(hit.getId()); + return influencer; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelDebugOutputIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelDebugOutputIterator.java new file mode 100644 index 00000000000..abb1c669484 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelDebugOutputIterator.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; + +class ElasticsearchBatchedModelDebugOutputIterator extends ElasticsearchBatchedDocumentsIterator +{ + public ElasticsearchBatchedModelDebugOutputIterator(Client client, String jobId, ParseFieldMatcher parserFieldMatcher) + { + super(client, ElasticsearchPersister.getJobIndexName(jobId), parserFieldMatcher); + } + + @Override + protected String getType() + { + return ModelDebugOutput.TYPE.getPreferredName(); + } + + @Override + protected ModelDebugOutput map(SearchHit hit) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser model debug output", e); + } + ModelDebugOutput result = ModelDebugOutput.PARSER.apply(parser, () -> parseFieldMatcher); + result.setId(hit.getId()); + return result; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSizeStatsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSizeStatsIterator.java new file mode 100644 index 00000000000..c4bcc8fc61c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSizeStatsIterator.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; + +public class ElasticsearchBatchedModelSizeStatsIterator extends ElasticsearchBatchedDocumentsIterator +{ + public ElasticsearchBatchedModelSizeStatsIterator(Client client, String jobId, ParseFieldMatcher parserFieldMatcher) + { + super(client, ElasticsearchPersister.getJobIndexName(jobId), parserFieldMatcher); + } + + @Override + protected String getType() + { + return ModelSizeStats.TYPE.getPreferredName(); + } + + @Override + protected ModelSizeStats map(SearchHit hit) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser model size stats", e); + } + + ModelSizeStats.Builder result = ModelSizeStats.PARSER.apply(parser, () -> parseFieldMatcher); + result.setId(hit.getId()); + return result.build(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSnapshotIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSnapshotIterator.java new file mode 100644 index 00000000000..8cd88cf7e07 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedModelSnapshotIterator.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; + +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; + +class ElasticsearchBatchedModelSnapshotIterator extends ElasticsearchBatchedDocumentsIterator +{ + public ElasticsearchBatchedModelSnapshotIterator(Client client, String jobId, ParseFieldMatcher parserFieldMatcher) + { + super(client, ElasticsearchPersister.getJobIndexName(jobId), parserFieldMatcher); + } + + @Override + protected String getType() + { + return ModelSnapshot.TYPE.getPreferredName(); + } + + @Override + protected ModelSnapshot map(SearchHit hit) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser model snapshot", e); + } + + return ModelSnapshot.PARSER.apply(parser, () -> parseFieldMatcher); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleter.java new file mode 100644 index 00000000000..ed2f34b6b6e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleter.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.ModelState; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; + +import java.util.Objects; +import java.util.function.LongSupplier; + +public class ElasticsearchBulkDeleter implements JobDataDeleter { + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchBulkDeleter.class); + + private static final int SCROLL_SIZE = 1000; + private static final String SCROLL_CONTEXT_DURATION = "5m"; + + private final Client client; + private final String jobId; + private final BulkRequestBuilder bulkRequestBuilder; + private long deletedBucketCount; + private long deletedRecordCount; + private long deletedBucketInfluencerCount; + private long deletedInfluencerCount; + private long deletedModelSnapshotCount; + private long deletedModelStateCount; + private boolean quiet; + + public ElasticsearchBulkDeleter(Client client, String jobId, boolean quiet) { + this.client = Objects.requireNonNull(client); + this.jobId = Objects.requireNonNull(jobId); + bulkRequestBuilder = client.prepareBulk(); + deletedBucketCount = 0; + deletedRecordCount = 0; + deletedBucketInfluencerCount = 0; + deletedInfluencerCount = 0; + deletedModelSnapshotCount = 0; + deletedModelStateCount = 0; + this.quiet = quiet; + } + + public ElasticsearchBulkDeleter(Client client, String jobId) { + this(client, jobId, false); + } + + @Override + public void deleteBucket(Bucket bucket) { + deleteRecords(bucket); + deleteBucketInfluencers(bucket); + bulkRequestBuilder.add( + client.prepareDelete(ElasticsearchPersister.getJobIndexName(jobId), Bucket.TYPE.getPreferredName(), bucket.getId())); + ++deletedBucketCount; + } + + @Override + public void deleteRecords(Bucket bucket) { + // Find the records using the time stamp rather than a parent-child + // relationship. The parent-child filter involves two queries behind + // the scenes, and Elasticsearch documentation claims it's significantly + // slower. Here we rely on the record timestamps being identical to the + // bucket timestamp. + deleteTypeByBucket(bucket, AnomalyRecord.TYPE.getPreferredName(), () -> ++deletedRecordCount); + } + + private void deleteTypeByBucket(Bucket bucket, String type, LongSupplier deleteCounter) { + QueryBuilder query = QueryBuilders.termQuery(ElasticsearchMappings.ES_TIMESTAMP, + bucket.getTimestamp().getTime()); + + int done = 0; + boolean finished = false; + while (finished == false) { + SearchResponse searchResponse = SearchAction.INSTANCE.newRequestBuilder(client) + .setIndices(ElasticsearchPersister.getJobIndexName(jobId)) + .setTypes(type) + .setQuery(query) + .addSort(SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC)) + .setSize(SCROLL_SIZE) + .setFrom(done) + .execute().actionGet(); + + for (SearchHit hit : searchResponse.getHits()) { + ++done; + addDeleteRequest(hit); + deleteCounter.getAsLong(); + } + if (searchResponse.getHits().getTotalHits() == done) { + finished = true; + } + } + } + + private void addDeleteRequest(SearchHit hit) { + DeleteRequestBuilder deleteRequest = DeleteAction.INSTANCE.newRequestBuilder(client) + .setIndex(ElasticsearchPersister.getJobIndexName(jobId)) + .setType(hit.getType()) + .setId(hit.getId()); + SearchHitField parentField = hit.field(ElasticsearchMappings.PARENT); + if (parentField != null) { + deleteRequest.setParent(parentField.getValue().toString()); + } + bulkRequestBuilder.add(deleteRequest); + } + + public void deleteBucketInfluencers(Bucket bucket) { + // Find the bucket influencers using the time stamp, relying on the + // bucket influencer timestamps being identical to the bucket timestamp. + deleteTypeByBucket(bucket, BucketInfluencer.TYPE.getPreferredName(), () -> ++deletedBucketInfluencerCount); + } + + public void deleteInfluencers(Bucket bucket) { + // Find the influencers using the time stamp, relying on the influencer + // timestamps being identical to the bucket timestamp. + deleteTypeByBucket(bucket, Influencer.TYPE.getPreferredName(), () -> ++deletedInfluencerCount); + } + + public void deleteBucketByTime(Bucket bucket) { + deleteTypeByBucket(bucket, Bucket.TYPE.getPreferredName(), () -> ++deletedBucketCount); + } + + @Override + public void deleteInfluencer(Influencer influencer) { + String id = influencer.getId(); + if (id == null) { + LOGGER.error("Cannot delete specific influencer without an ID", + // This means we get a stack trace to show where the request came from + new NullPointerException()); + return; + } + bulkRequestBuilder.add( + client.prepareDelete(ElasticsearchPersister.getJobIndexName(jobId), Influencer.TYPE.getPreferredName(), id)); + ++deletedInfluencerCount; + } + + @Override + public void deleteModelSnapshot(ModelSnapshot modelSnapshot) { + String snapshotId = modelSnapshot.getSnapshotId(); + int docCount = modelSnapshot.getSnapshotDocCount(); + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + // Deduce the document IDs of the state documents from the information + // in the snapshot document - we cannot query the state itself as it's + // too big and has no mappings + for (int i = 0; i < docCount; ++i) { + String stateId = snapshotId + '_' + i; + bulkRequestBuilder.add( + client.prepareDelete(indexName, ModelState.TYPE, stateId)); + ++deletedModelStateCount; + } + + bulkRequestBuilder.add( + client.prepareDelete(indexName, ModelSnapshot.TYPE.getPreferredName(), snapshotId)); + ++deletedModelSnapshotCount; + } + + @Override + public void deleteModelDebugOutput(ModelDebugOutput modelDebugOutput) { + String id = modelDebugOutput.getId(); + bulkRequestBuilder.add( + client.prepareDelete(ElasticsearchPersister.getJobIndexName(jobId), ModelDebugOutput.TYPE.getPreferredName(), id)); + } + + @Override + public void deleteModelSizeStats(ModelSizeStats modelSizeStats) { + bulkRequestBuilder.add(client.prepareDelete( + ElasticsearchPersister.getJobIndexName(jobId), ModelSizeStats.TYPE.getPreferredName(), modelSizeStats.getId())); + } + + public void deleteInterimResults() { + QueryBuilder qb = QueryBuilders.termQuery(Bucket.IS_INTERIM.getPreferredName(), true); + + SearchResponse searchResponse = client.prepareSearch(ElasticsearchPersister.getJobIndexName(jobId)) + .setTypes(Bucket.TYPE.getPreferredName(), AnomalyRecord.TYPE.getPreferredName(), Influencer.TYPE.getPreferredName(), + BucketInfluencer.TYPE.getPreferredName()) + .setQuery(qb) + .addSort(SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC)) + .setScroll(SCROLL_CONTEXT_DURATION) + .setSize(SCROLL_SIZE) + .get(); + + String scrollId = searchResponse.getScrollId(); + long totalHits = searchResponse.getHits().totalHits(); + long totalDeletedCount = 0; + while (totalDeletedCount < totalHits) { + for (SearchHit hit : searchResponse.getHits()) { + LOGGER.trace("Search hit for bucket: " + hit.toString() + ", " + hit.getId()); + String type = hit.getType(); + if (type.equals(Bucket.TYPE)) { + ++deletedBucketCount; + } else if (type.equals(AnomalyRecord.TYPE)) { + ++deletedRecordCount; + } else if (type.equals(BucketInfluencer.TYPE)) { + ++deletedBucketInfluencerCount; + } else if (type.equals(Influencer.TYPE)) { + ++deletedInfluencerCount; + } + ++totalDeletedCount; + addDeleteRequest(hit); + } + + searchResponse = client.prepareSearchScroll(scrollId).setScroll(SCROLL_CONTEXT_DURATION).get(); + } + } + + /** + * Commits the deletions and if {@code forceMerge} is {@code true}, it + * forces a merge which removes the data from disk. + */ + @Override + public void commit(ActionListener listener) { + if (bulkRequestBuilder.numberOfActions() == 0) { + listener.onResponse(new BulkResponse(new BulkItemResponse[0], 0L)); + return; + } + + if (!quiet) { + LOGGER.debug("Requesting deletion of " + + deletedBucketCount + " buckets, " + + deletedRecordCount + " records, " + + deletedBucketInfluencerCount + " bucket influencers, " + + deletedInfluencerCount + " influencers, " + + deletedModelSnapshotCount + " model snapshots, " + + " and " + + deletedModelStateCount + " model state documents"); + } + + try { + bulkRequestBuilder.execute(listener); + } catch (Exception e) { + listener.onFailure(e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleterFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleterFactory.java new file mode 100644 index 00000000000..3acaf471947 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBulkDeleterFactory.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.client.Client; + +import java.util.function.Function; + +/** + * TODO This is all just silly static typing shenanigans because Guice can't inject + * anonymous lambdas. This can all be removed once Guice goes away. + */ +public class ElasticsearchBulkDeleterFactory implements Function { + + private final Client client; + + public ElasticsearchBulkDeleterFactory(Client client) { + this.client = client; + } + + @Override + public ElasticsearchBulkDeleter apply(String jobId) { + return new ElasticsearchBulkDeleter(client, jobId); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverser.java new file mode 100644 index 00000000000..cd99f0a31aa --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverser.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Pattern; + +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; + +/** + * Interprets field names containing dots as nested JSON structures. + * This matches what Elasticsearch does. + */ +class ElasticsearchDotNotationReverser +{ + private static final char DOT = '.'; + private static final Pattern DOT_PATTERN = Pattern.compile("\\."); + + private final Map resultsMap; + + public ElasticsearchDotNotationReverser() + { + resultsMap = new TreeMap<>(); + } + + // TODO - could handle values of all types Elasticsearch does, e.g. date, + // long, int, double, etc. However, at the moment field values in our + // results are only strings, so it's not "minimum viable product" right + // now. Hence this method only takes fieldValue as a String and there are + // no overloads. + /** + * Given a field name and value, convert it to a map representation of the + * (potentially nested) JSON structure Elasticsearch would use to store it. + * For example: + * foo = x goes to { "foo" : "x" } and + * foo.bar = y goes to { "foo" : { "bar" : "y" } } + */ + @SuppressWarnings("unchecked") + public void add(String fieldName, String fieldValue) + { + if (fieldName == null || fieldValue == null) + { + return; + } + + // Minimise processing in the simple case of no dots in the field name. + if (fieldName.indexOf(DOT) == -1) + { + if (ReservedFieldNames.RESERVED_FIELD_NAMES.contains(fieldName)) + { + return; + } + resultsMap.put(fieldName, fieldValue); + return; + } + + String[] segments = DOT_PATTERN.split(fieldName); + + // If any segment created by the split is a reserved word then ignore + // the whole field. + for (String segment : segments) + { + if (ReservedFieldNames.RESERVED_FIELD_NAMES.contains(segment)) + { + return; + } + } + + Map layerMap = resultsMap; + for (int i = 0; i < segments.length; ++i) + { + String segment = segments[i]; + if (i == segments.length - 1) + { + layerMap.put(segment, fieldValue); + } + else + { + Object existingLayerValue = layerMap.get(segment); + if (existingLayerValue == null) + { + Map nextLayerMap = new TreeMap<>(); + layerMap.put(segment, nextLayerMap); + layerMap = nextLayerMap; + } + else + { + if (existingLayerValue instanceof Map) + { + layerMap = (Map)existingLayerValue; + } + else + { + // This implies an inconsistency - different additions + // imply the same path leads to both an object and a + // value. For example: + // foo.bar = x + // foo.bar.baz = y + return; + } + } + } + } + } + + public Map getResultsMap() + { + return resultsMap; + } + + /** + * Mappings for a given hierarchical structure are more complex than the + * basic results. + */ + public Map getMappingsMap() + { + Map mappingsMap = new TreeMap<>(); + recurseMappingsLevel(resultsMap, mappingsMap); + return mappingsMap; + } + + @SuppressWarnings("unchecked") + private void recurseMappingsLevel(Map resultsMap, + Map mappingsMap) + { + for (Map.Entry entry : resultsMap.entrySet()) + { + Map typeMap = new TreeMap<>(); + + String name = entry.getKey(); + Object value = entry.getValue(); + if (value instanceof Map) + { + Map propertiesMap = new TreeMap<>(); + recurseMappingsLevel((Map)value, propertiesMap); + + typeMap.put(ElasticsearchMappings.TYPE, + ElasticsearchMappings.OBJECT); + typeMap.put(ElasticsearchMappings.PROPERTIES, propertiesMap); + mappingsMap.put(name, typeMap); + } + else + { + String fieldType = value.getClass().getSimpleName().toLowerCase(Locale.ROOT); + if ("string".equals(fieldType)) { + fieldType = "keyword"; + } + typeMap.put(ElasticsearchMappings.TYPE, + // Even though the add() method currently only supports + // strings, this way of getting the type would work for + // many Elasticsearch types, e.g. date, int, long, + // double and boolean + fieldType); + mappingsMap.put(name, typeMap); + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDataCountsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDataCountsPersister.java new file mode 100644 index 00000000000..747e7edf7e7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDataCountsPersister.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import java.io.IOException; +import java.util.Locale; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.xpack.prelert.job.DataCounts; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + + +public class ElasticsearchJobDataCountsPersister implements JobDataCountsPersister { + + private Client client; + private Logger logger; + + public ElasticsearchJobDataCountsPersister(Client client, Logger logger) { + this.client = client; + this.logger = logger; + } + + private XContentBuilder serialiseCounts(DataCounts counts) throws IOException { + XContentBuilder builder = jsonBuilder(); + return counts.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + + + @Override + public void persistDataCounts(String jobId, DataCounts counts) { + // NORELEASE - Should these stats be stored in memory? + + + try { + XContentBuilder content = serialiseCounts(counts); + + client.prepareIndex(ElasticsearchPersister.getJobIndexName(jobId), DataCounts.TYPE.getPreferredName(), + jobId + DataCounts.DOCUMENT_SUFFIX) + .setSource(content).execute().actionGet(); + } + catch (IOException ioe) { + logger.warn("Error serialising DataCounts stats", ioe); + } + catch (IndexNotFoundException e) { + String msg = String.format(Locale.ROOT, "Error writing the job '%s' status stats.", jobId); + logger.warn(msg, e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapper.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapper.java new file mode 100644 index 00000000000..f3cbf38cabf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapper.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; + +import java.io.IOException; +import java.util.Objects; + +class ElasticsearchJobDetailsMapper { + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchJobDetailsMapper.class); + + private final Client client; + private final ParseFieldMatcher parseFieldMatcher; + + public ElasticsearchJobDetailsMapper(Client client, ParseFieldMatcher parseFieldMatcher) { + this.client = Objects.requireNonNull(client); + this.parseFieldMatcher = Objects.requireNonNull(parseFieldMatcher); + } + + /** + * Maps an Elasticsearch source map to a {@link Job} object + * + * @param source The source of an Elasticsearch search response + * @return the {@code Job} object + */ + public Job map(BytesReference source) { + try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) { + Job.Builder builder = Job.PARSER.apply(parser, () -> parseFieldMatcher); + addModelSizeStats(builder, builder.getId()); + addBucketProcessingTime(builder, builder.getId()); + return builder.build(); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser job", e); + } + } + + private void addModelSizeStats(Job.Builder job, String jobId) { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + // Pull out the modelSizeStats document, and add this to the Job + LOGGER.trace("ES API CALL: get ID " + ModelSizeStats.TYPE + + " type " + ModelSizeStats.TYPE + " from index " + indexName); + GetResponse modelSizeStatsResponse = client.prepareGet( + indexName, ModelSizeStats.TYPE.getPreferredName(), ModelSizeStats.TYPE.getPreferredName()).get(); + + if (!modelSizeStatsResponse.isExists()) { + String msg = "No memory usage details for job with id " + jobId; + LOGGER.warn(msg); + } else { + // Remove the Kibana/Logstash '@timestamp' entry as stored in Elasticsearch, + // and replace using the API 'timestamp' key. + Object timestamp = modelSizeStatsResponse.getSource().remove(ElasticsearchMappings.ES_TIMESTAMP); + modelSizeStatsResponse.getSource().put(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName(), timestamp); + BytesReference source = modelSizeStatsResponse.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser model size stats", e); + } + ModelSizeStats.Builder modelSizeStats = ModelSizeStats.PARSER.apply(parser, () -> parseFieldMatcher); + job.setModelSizeStats(modelSizeStats); + } + } + + private void addBucketProcessingTime(Job.Builder job, String jobId) { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + // Pull out the modelSizeStats document, and add this to the Job + LOGGER.trace("ES API CALL: get ID " + ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE + + " type " + ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS + " from index " + indexName); + GetResponse procTimeResponse = client.prepareGet( + indexName, ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE, + ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS).get(); + + if (!procTimeResponse.isExists()) { + String msg = "No average bucket processing time details for job with id " + jobId; + LOGGER.warn(msg); + } else { + Object averageTime = procTimeResponse.getSource() + .get(ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS); + if (averageTime instanceof Double) { + job.setAverageBucketProcessingTimeMs((Double) averageTime); + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProvider.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProvider.java new file mode 100644 index 00000000000..5c42c591bd7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProvider.java @@ -0,0 +1,1192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.prelert.job.CategorizerState; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.ModelState; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.persistence.BucketsQueryBuilder.BucketsQuery; +import org.elasticsearch.xpack.prelert.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; +import org.elasticsearch.xpack.prelert.job.usage.Usage; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +public class ElasticsearchJobProvider implements JobProvider +{ + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchJobProvider.class); + + /** + * The index to store total usage/metering information + */ + public static final String PRELERT_USAGE_INDEX = "prelert-usage"; + + /** + * Where to store the prelert info in Elasticsearch - must match what's + * expected by kibana/engineAPI/app/directives/prelertLogUsage.js + */ + private static final String PRELERT_INFO_INDEX = "prelert-int"; + + private static final String SETTING_TRANSLOG_DURABILITY = "index.translog.durability"; + private static final String ASYNC = "async"; + private static final String SETTING_MAPPER_DYNAMIC = "index.mapper.dynamic"; + private static final String SETTING_DEFAULT_ANALYZER_TYPE = "index.analysis.analyzer.default.type"; + private static final String KEYWORD = "keyword"; + + private static final List SECONDARY_SORT = Arrays.asList( + AnomalyRecord.ANOMALY_SCORE.getPreferredName(), + AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyRecord.BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.FIELD_NAME.getPreferredName(), + AnomalyRecord.FUNCTION.getPreferredName() + ); + + private static final int RECORDS_SIZE_PARAM = 500; + + + private final Client client; + private final int numberOfReplicas; + private final ParseFieldMatcher parseFieldMatcher; + + public ElasticsearchJobProvider(Client client, int numberOfReplicas, ParseFieldMatcher parseFieldMatcher) { + this.parseFieldMatcher = parseFieldMatcher; + this.client = Objects.requireNonNull(client); + this.numberOfReplicas = numberOfReplicas; + } + + public void initialize() { + LOGGER.info("Connecting to Elasticsearch cluster '" + client.settings().get("cluster.name") + + "'"); + + // This call was added because if we try to connect to Elasticsearch + // while it's doing the recovery operations it does at startup then we + // can get weird effects like indexes being reported as not existing + // when they do. See EL16-182 in Jira. + LOGGER.trace("ES API CALL: wait for yellow status on whole cluster"); + ClusterHealthResponse response = client.admin().cluster() + .prepareHealth() + .setWaitForYellowStatus() + .execute().actionGet(); + + // The wait call above can time out. + // Throw an error if in cluster health is red + if (response.getStatus() == ClusterHealthStatus.RED) { + String msg = "Waited for the Elasticsearch status to be YELLOW but is RED after wait timeout"; + LOGGER.error(msg); + throw new IllegalStateException(msg); + } + + LOGGER.info("Elasticsearch cluster '" + client.settings().get("cluster.name") + + "' now ready to use"); + + + createUsageMeteringIndex(); + } + + /** + * If the {@value ElasticsearchJobProvider#PRELERT_USAGE_INDEX} index does + * not exist then create it here with the usage document mapping. + */ + private void createUsageMeteringIndex() { + try { + LOGGER.trace("ES API CALL: index exists? " + PRELERT_USAGE_INDEX); + boolean indexExists = client.admin().indices() + .exists(new IndicesExistsRequest(PRELERT_USAGE_INDEX)) + .get().isExists(); + + if (indexExists == false) { + LOGGER.info("Creating the internal '" + PRELERT_USAGE_INDEX + "' index"); + + XContentBuilder usageMapping = ElasticsearchMappings.usageMapping(); + + LOGGER.trace("ES API CALL: create index " + PRELERT_USAGE_INDEX); + client.admin().indices().prepareCreate(PRELERT_USAGE_INDEX) + .setSettings(prelertIndexSettings()) + .addMapping(Usage.TYPE, usageMapping) + .get(); + LOGGER.trace("ES API CALL: wait for yellow status " + PRELERT_USAGE_INDEX); + client.admin().cluster().prepareHealth(PRELERT_USAGE_INDEX).setWaitForYellowStatus().execute().actionGet(); + } + } catch (InterruptedException | ExecutionException | IOException e) { + LOGGER.warn("Error checking the usage metering index", e); + } catch (ResourceAlreadyExistsException e) { + LOGGER.debug("Usage metering index already exists", e); + } + } + + private boolean indexExists(String jobId) + { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: index exists? " + indexName); + IndicesExistsResponse res = + client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet(); + + return res.isExists(); + } + + /** + * Build the Elasticsearch index settings that we want to apply to Prelert + * indexes. It's better to do this in code rather than in elasticsearch.yml + * because then the settings can be applied regardless of whether we're + * using our own Elasticsearch to store results or a customer's pre-existing + * Elasticsearch. + * @return An Elasticsearch builder initialised with the desired settings + * for Prelert indexes. + */ + private Settings.Builder prelertIndexSettings() + { + return Settings.builder() + // Our indexes are small and one shard puts the + // least possible burden on Elasticsearch + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + // Sacrifice durability for performance: in the event of power + // failure we can lose the last 5 seconds of changes, but it's + // much faster + .put(SETTING_TRANSLOG_DURABILITY, ASYNC) + // We need to allow fields not mentioned in the mappings to + // pick up default mappings and be used in queries + .put(SETTING_MAPPER_DYNAMIC, true) + // By default "analyzed" fields won't be tokenised + .put(SETTING_DEFAULT_ANALYZER_TYPE, KEYWORD); + } + + /** + * Create the Elasticsearch index and the mappings + */ + @Override + public void createJobRelatedIndices(Job job, ActionListener listener) { + Collection termFields = (job.getAnalysisConfig() != null) ? job.getAnalysisConfig().termFields() : null; + Collection influencers = (job.getAnalysisConfig() != null) ? job.getAnalysisConfig().getInfluencers() : null; + try { + XContentBuilder bucketMapping = ElasticsearchMappings.bucketMapping(); + XContentBuilder bucketInfluencerMapping = ElasticsearchMappings.bucketInfluencerMapping(); + XContentBuilder categorizerStateMapping = ElasticsearchMappings.categorizerStateMapping(); + XContentBuilder categoryDefinitionMapping = ElasticsearchMappings.categoryDefinitionMapping(); + XContentBuilder recordMapping = ElasticsearchMappings.recordMapping(termFields); + XContentBuilder quantilesMapping = ElasticsearchMappings.quantilesMapping(); + XContentBuilder modelStateMapping = ElasticsearchMappings.modelStateMapping(); + XContentBuilder modelSnapshotMapping = ElasticsearchMappings.modelSnapshotMapping(); + XContentBuilder modelSizeStatsMapping = ElasticsearchMappings.modelSizeStatsMapping(); + XContentBuilder influencerMapping = ElasticsearchMappings.influencerMapping(influencers); + XContentBuilder modelDebugMapping = ElasticsearchMappings.modelDebugOutputMapping(termFields); + XContentBuilder processingTimeMapping = ElasticsearchMappings.processingTimeMapping(); + XContentBuilder partitionScoreMapping = ElasticsearchMappings.bucketPartitionMaxNormalizedScores(); + XContentBuilder dataCountsMapping = ElasticsearchMappings.dataCountsMapping(); + + String jobId = job.getId(); + LOGGER.trace("ES API CALL: create index " + job.getId()); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(ElasticsearchPersister.getJobIndexName(jobId)); + createIndexRequest.settings(prelertIndexSettings()); + createIndexRequest.mapping(Bucket.TYPE.getPreferredName(), bucketMapping); + createIndexRequest.mapping(BucketInfluencer.TYPE.getPreferredName(), bucketInfluencerMapping); + createIndexRequest.mapping(CategorizerState.TYPE, categorizerStateMapping); + createIndexRequest.mapping(CategoryDefinition.TYPE.getPreferredName(), categoryDefinitionMapping); + createIndexRequest.mapping(AnomalyRecord.TYPE.getPreferredName(), recordMapping); + createIndexRequest.mapping(Quantiles.TYPE.getPreferredName(), quantilesMapping); + createIndexRequest.mapping(ModelState.TYPE, modelStateMapping); + createIndexRequest.mapping(ModelSnapshot.TYPE.getPreferredName(), modelSnapshotMapping); + createIndexRequest.mapping(ModelSizeStats.TYPE.getPreferredName(), modelSizeStatsMapping); + createIndexRequest.mapping(Influencer.TYPE.getPreferredName(), influencerMapping); + createIndexRequest.mapping(ModelDebugOutput.TYPE.getPreferredName(), modelDebugMapping); + createIndexRequest.mapping(ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE, processingTimeMapping); + createIndexRequest.mapping(ReservedFieldNames.PARTITION_NORMALIZED_PROB_TYPE, partitionScoreMapping); + createIndexRequest.mapping(DataCounts.TYPE.getPreferredName(), dataCountsMapping); + + client.admin().indices().create(createIndexRequest, new ActionListener() { + @Override + public void onResponse(CreateIndexResponse createIndexResponse) { + listener.onResponse(true); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + @Override + public void deleteJobRelatedIndices(String jobId, ActionListener listener) { + if (indexExists(jobId) == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobId)); + return; + } + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: delete index " + indexName); + + try { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); + client.admin().indices().delete(deleteIndexRequest, new ActionListener() { + @Override + public void onResponse(DeleteIndexResponse deleteIndexResponse) { + listener.onResponse(deleteIndexResponse.isAcknowledged()); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public DataCounts dataCounts(String jobId) { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + + try { + GetResponse response = client.prepareGet(indexName, DataCounts.TYPE.getPreferredName(), + jobId + DataCounts.DOCUMENT_SUFFIX).get(); + if (response.isExists() == false) { + return new DataCounts(jobId); + } else { + BytesReference source = response.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + return DataCounts.PARSER.apply(parser, () -> parseFieldMatcher); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser bucket", e); + } + } + + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + } + + @Override + public QueryPage buckets(String jobId, BucketsQuery query) + throws ResourceNotFoundException { + QueryBuilder fb = new ResultsFilterBuilder() + .timeRange(ElasticsearchMappings.ES_TIMESTAMP, query.getEpochStart(), query.getEpochEnd()) + .score(Bucket.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreFilter()) + .score(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbability()) + .interim(Bucket.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) + .build(); + + SortBuilder sortBuilder = new FieldSortBuilder(esSortField(query.getSortField())) + .order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); + QueryPage buckets = buckets(jobId, query.isIncludeInterim(), query.getFrom(), query.getSize(), fb, sortBuilder); + + if (Strings.isNullOrEmpty(query.getPartitionValue())) { + for (Bucket b : buckets.hits()) { + if (query.isExpand() && b.getRecordCount() > 0) { + expandBucket(jobId, query.isIncludeInterim(), b); + } + } + } else { + List scores = + partitionScores(jobId, query.getEpochStart(), query.getEpochEnd(), query.getPartitionValue()); + + mergePartitionScoresIntoBucket(scores, buckets.hits()); + + for (Bucket b : buckets.hits()) { + if (query.isExpand() && b.getRecordCount() > 0) { + this.expandBucketForPartitionValue(jobId, query.isIncludeInterim(), b, query.getPartitionValue()); + } + + b.setAnomalyScore(b.partitionAnomalyScore(query.getPartitionValue())); + } + + } + + return buckets; + } + + void mergePartitionScoresIntoBucket(List scores, List buckets) { + Iterator itr = scores.iterator(); + ScoreTimestamp score = itr.hasNext() ? itr.next() : null; + for (Bucket b : buckets) { + if (score == null) { + b.setMaxNormalizedProbability(0.0); + } else { + if (score.timestamp.equals(b.getTimestamp())) { + b.setMaxNormalizedProbability(score.score); + score = itr.hasNext() ? itr.next() : null; + } else { + b.setMaxNormalizedProbability(0.0); + } + } + } + } + + private QueryPage buckets(String jobId, boolean includeInterim, int from, int size, + QueryBuilder fb, SortBuilder sb) throws ResourceNotFoundException { + SearchResponse searchResponse; + try { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type " + Bucket.TYPE + + " from index " + indexName + " sort ascending " + ElasticsearchMappings.ES_TIMESTAMP + + " with filter after sort from " + from + " size " + size); + searchResponse = client.prepareSearch(indexName) + .setTypes(Bucket.TYPE.getPreferredName()) + .addSort(sb) + .setQuery(new ConstantScoreQueryBuilder(fb)) + .setFrom(from).setSize(size) + .get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + List results = new ArrayList<>(); + + for (SearchHit hit : searchResponse.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser bucket", e); + } + Bucket bucket = Bucket.PARSER.apply(parser, () -> parseFieldMatcher); + bucket.setId(hit.getId()); + + if (includeInterim || bucket.isInterim() == false) { + results.add(bucket); + } + } + + return new QueryPage<>(results, searchResponse.getHits().getTotalHits()); + } + + + @Override + public QueryPage bucket(String jobId, BucketQueryBuilder.BucketQuery query) throws ResourceNotFoundException { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + SearchHits hits; + try { + LOGGER.trace("ES API CALL: get Bucket with timestamp " + query.getTimestamp() + + " from index " + indexName); + QueryBuilder qb = QueryBuilders.matchQuery(ElasticsearchMappings.ES_TIMESTAMP, + query.getTimestamp()); + + SearchResponse searchResponse = client.prepareSearch(indexName) + .setTypes(Bucket.TYPE.getPreferredName()) + .setQuery(qb) + .addSort(SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC)) + .get(); + hits = searchResponse.getHits(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + if (hits.getTotalHits() == 0) { + return new QueryPage<>(Collections.emptyList(), 0); + } else if (hits.getTotalHits() > 1L) { + LOGGER.error("Found more than one bucket with timestamp [" + query.getTimestamp() + "]" + + " from index " + indexName); + } + + SearchHit hit = hits.getAt(0); + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser bucket", e); + } + Bucket bucket = Bucket.PARSER.apply(parser, () -> parseFieldMatcher); + bucket.setId(hit.getId()); + + // don't return interim buckets if not requested + if (bucket.isInterim() && query.isIncludeInterim() == false) { + return new QueryPage<>(Collections.emptyList(), 0); + } + + if (Strings.isNullOrEmpty(query.getPartitionValue())) { + if (query.isExpand() && bucket.getRecordCount() > 0) { + expandBucket(jobId, query.isIncludeInterim(), bucket); + } + } else { + List scores = + partitionScores(jobId, + query.getTimestamp(), query.getTimestamp() + 1, + query.getPartitionValue()); + + bucket.setMaxNormalizedProbability(scores.isEmpty() == false ? + scores.get(0).score : 0.0d); + if (query.isExpand() && bucket.getRecordCount() > 0) { + this.expandBucketForPartitionValue(jobId, query.isIncludeInterim(), + bucket, query.getPartitionValue()); + } + + bucket.setAnomalyScore( + bucket.partitionAnomalyScore(query.getPartitionValue())); + } + return new QueryPage<>(Collections.singletonList(bucket), 1); + } + + final class ScoreTimestamp + { + double score; + Date timestamp; + + public ScoreTimestamp(Date timestamp, double score) + { + this.score = score; + this.timestamp = timestamp; + } + } + + private List partitionScores(String jobId, Object epochStart, + Object epochEnd, String partitionFieldValue) + throws ResourceNotFoundException + { + QueryBuilder qb = new ResultsFilterBuilder() + .timeRange(ElasticsearchMappings.ES_TIMESTAMP, epochStart, epochEnd) + .build(); + + FieldSortBuilder sb = new FieldSortBuilder(ElasticsearchMappings.ES_TIMESTAMP) + .order(SortOrder.ASC); + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + SearchRequestBuilder searchBuilder = client + .prepareSearch(indexName) + .setPostFilter(qb) + .addSort(sb) + .setTypes(ReservedFieldNames.PARTITION_NORMALIZED_PROB_TYPE); + + SearchResponse searchResponse; + try { + searchResponse = searchBuilder.get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + List results = new ArrayList<>(); + + // expect 1 document per bucket + if (searchResponse.getHits().totalHits() > 0) + { + + Map m = searchResponse.getHits().getAt(0).getSource(); + + @SuppressWarnings("unchecked") + List> probs = (List>) + m.get(ReservedFieldNames.PARTITION_NORMALIZED_PROBS); + for (Map prob : probs) + { + if (partitionFieldValue.equals(prob.get(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()))) + { + Date ts = new Date(TimeUtils.dateStringToEpoch((String) m.get(ElasticsearchMappings.ES_TIMESTAMP))); + results.add(new ScoreTimestamp(ts, + (Double) prob.get(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()))); + } + } + } + + return results; + } + + public int expandBucketForPartitionValue(String jobId, boolean includeInterim, Bucket bucket, + String partitionFieldValue) throws ResourceNotFoundException + { + int from = 0; + + QueryPage page = bucketRecords( + jobId, bucket, from, RECORDS_SIZE_PARAM, includeInterim, + AnomalyRecord.PROBABILITY.getPreferredName(), false, partitionFieldValue); + bucket.setRecords(page.hits()); + + while (page.hitCount() > from + RECORDS_SIZE_PARAM) + { + from += RECORDS_SIZE_PARAM; + page = bucketRecords( + jobId, bucket, from, RECORDS_SIZE_PARAM, includeInterim, + AnomalyRecord.PROBABILITY.getPreferredName(), false, partitionFieldValue); + bucket.getRecords().addAll(page.hits()); + } + + return bucket.getRecords().size(); + } + + + @Override + public BatchedDocumentsIterator newBatchedBucketsIterator(String jobId) + { + return new ElasticsearchBatchedBucketsIterator(client, jobId, parseFieldMatcher); + } + + @Override + public int expandBucket(String jobId, boolean includeInterim, Bucket bucket) throws ResourceNotFoundException { + int from = 0; + + QueryPage page = bucketRecords( + jobId, bucket, from, RECORDS_SIZE_PARAM, includeInterim, + AnomalyRecord.PROBABILITY.getPreferredName(), false, null); + bucket.setRecords(page.hits()); + + while (page.hitCount() > from + RECORDS_SIZE_PARAM) + { + from += RECORDS_SIZE_PARAM; + page = bucketRecords( + jobId, bucket, from, RECORDS_SIZE_PARAM, includeInterim, + AnomalyRecord.PROBABILITY.getPreferredName(), false, null); + bucket.getRecords().addAll(page.hits()); + } + + return bucket.getRecords().size(); + } + + QueryPage bucketRecords(String jobId, + Bucket bucket, int from, int size, boolean includeInterim, + String sortField, boolean descending, String partitionFieldValue) + throws ResourceNotFoundException + { + // Find the records using the time stamp rather than a parent-child + // relationship. The parent-child filter involves two queries behind + // the scenes, and Elasticsearch documentation claims it's significantly + // slower. Here we rely on the record timestamps being identical to the + // bucket timestamp. + QueryBuilder recordFilter = QueryBuilders.termQuery(ElasticsearchMappings.ES_TIMESTAMP, + bucket.getTimestamp().getTime()); + + recordFilter = new ResultsFilterBuilder(recordFilter) + .interim(AnomalyRecord.IS_INTERIM.getPreferredName(), includeInterim) + .term(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue) + .build(); + + FieldSortBuilder sb = null; + if (sortField != null) + { + sb = new FieldSortBuilder(esSortField(sortField)) + .missing("_last") + .order(descending ? SortOrder.DESC : SortOrder.ASC); + } + + return records(jobId, from, size, recordFilter, sb, SECONDARY_SORT, + descending); + } + + @Override + public QueryPage categoryDefinitions(String jobId, int from, int size) { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type " + CategoryDefinition.TYPE + + " from index " + indexName + " sort ascending " + CategoryDefinition.CATEGORY_ID + + " from " + from + " size " + size); + SearchRequestBuilder searchBuilder = client.prepareSearch(indexName) + .setTypes(CategoryDefinition.TYPE.getPreferredName()) + .setFrom(from).setSize(size) + .addSort(new FieldSortBuilder(CategoryDefinition.CATEGORY_ID.getPreferredName()).order(SortOrder.ASC)); + + SearchResponse searchResponse; + try { + searchResponse = searchBuilder.get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + SearchHit[] hits = searchResponse.getHits().getHits(); + List results = new ArrayList<>(hits.length); + for (SearchHit hit : hits) { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser category definition", e); + } + CategoryDefinition categoryDefinition = CategoryDefinition.PARSER.apply(parser, () -> parseFieldMatcher); + results.add(categoryDefinition); + } + + return new QueryPage<>(results, searchResponse.getHits().getTotalHits()); + } + + + @Override + public QueryPage categoryDefinition(String jobId, String categoryId) { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + GetResponse response; + + try { + LOGGER.trace("ES API CALL: get ID " + categoryId + " type " + CategoryDefinition.TYPE + + " from index " + indexName); + response = client.prepareGet(indexName, CategoryDefinition.TYPE.getPreferredName(), categoryId).get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + if (response.isExists()) { + BytesReference source = response.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser category definition", e); + } + CategoryDefinition definition = CategoryDefinition.PARSER.apply(parser, () -> parseFieldMatcher); + return new QueryPage<>(Collections.singletonList(definition), 1); + } + return new QueryPage<>(Collections.emptyList(), 0); + } + + @Override + public QueryPage records(String jobId, RecordsQueryBuilder.RecordsQuery query) + throws ResourceNotFoundException { + QueryBuilder fb = new ResultsFilterBuilder() + .timeRange(ElasticsearchMappings.ES_TIMESTAMP, query.getEpochStart(), query.getEpochEnd()) + .score(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreThreshold()) + .score(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbabilityThreshold()) + .interim(AnomalyRecord.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) + .term(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), query.getPartitionFieldValue()).build(); + + return records(jobId, query.getFrom(), query.getSize(), fb, query.getSortField(), query.isSortDescending()); + } + + + private QueryPage records(String jobId, + int from, int size, QueryBuilder recordFilter, + String sortField, boolean descending) + throws ResourceNotFoundException + { + FieldSortBuilder sb = null; + if (sortField != null) + { + sb = new FieldSortBuilder(esSortField(sortField)) + .missing("_last") + .order(descending ? SortOrder.DESC : SortOrder.ASC); + } + + return records(jobId, from, size, recordFilter, sb, SECONDARY_SORT, descending); + } + + + /** + * The returned records have the parent bucket id set. + */ + private QueryPage records(String jobId, int from, int size, + QueryBuilder recordFilter, FieldSortBuilder sb, List secondarySort, + boolean descending) throws ResourceNotFoundException { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + + recordFilter = new BoolQueryBuilder() + .filter(recordFilter) + .filter(new TermsQueryBuilder(AnomalyRecord.RESULT_TYPE.getPreferredName(), AnomalyRecord.RESULT_TYPE_VALUE)); + + SearchRequestBuilder searchBuilder = client.prepareSearch(indexName) + .setTypes(AnomalyRecord.TYPE.getPreferredName()) + .setQuery(recordFilter) + .setFrom(from).setSize(size) + .addSort(sb == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) : sb) + .setFetchSource(true); // the field option turns off source so request it explicitly + + for (String sortField : secondarySort) + { + searchBuilder.addSort(esSortField(sortField), descending ? SortOrder.DESC : SortOrder.ASC); + } + + SearchResponse searchResponse; + try { + LOGGER.trace("ES API CALL: search all of type " + AnomalyRecord.TYPE + + " from index " + indexName + ((sb != null) ? " with sort" : "") + + (secondarySort.isEmpty() ? "" : " with secondary sort") + + " with filter after sort from " + from + " size " + size); + searchResponse = searchBuilder.get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + List results = new ArrayList<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser records", e); + } + AnomalyRecord record = AnomalyRecord.PARSER.apply(parser, () -> parseFieldMatcher); + + // set the ID and parent ID + record.setId(hit.getId()); + results.add(record); + } + + return new QueryPage<>(results, searchResponse.getHits().getTotalHits()); + } + + @Override + public QueryPage influencers(String jobId, InfluencersQuery query) throws ResourceNotFoundException + { + + QueryBuilder fb = new ResultsFilterBuilder() + .timeRange(ElasticsearchMappings.ES_TIMESTAMP, query.getEpochStart(), query.getEpochEnd()) + .score(Bucket.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreFilter()) + .interim(Bucket.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) + .build(); + + return influencers(jobId, query.getFrom(), query.getSize(), fb, query.getSortField(), + query.isSortDescending()); + } + + private QueryPage influencers(String jobId, int from, int size, QueryBuilder filterBuilder, String sortField, + boolean sortDescending) throws ResourceNotFoundException { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type " + Influencer.TYPE + " from index " + indexName + + ((sortField != null) + ? " with sort " + (sortDescending ? "descending" : "ascending") + " on field " + esSortField(sortField) : "") + + " with filter after sort from " + from + " size " + size); + + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexName) + .setTypes(Influencer.TYPE.getPreferredName()) + .setPostFilter(filterBuilder) + .setFrom(from).setSize(size); + + FieldSortBuilder sb = sortField == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) + : new FieldSortBuilder(esSortField(sortField)).order(sortDescending ? SortOrder.DESC : SortOrder.ASC); + searchRequestBuilder.addSort(sb); + + SearchResponse response = null; + try + { + response = searchRequestBuilder.get(); + } + catch (IndexNotFoundException e) + { + throw new ResourceNotFoundException("job " + jobId + " not found"); + } + + List influencers = new ArrayList<>(); + for (SearchHit hit : response.getHits().getHits()) + { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser list", e); + } + Influencer influencer = Influencer.PARSER.apply(parser, () -> parseFieldMatcher); + influencer.setId(hit.getId()); + + influencers.add(influencer); + } + + return new QueryPage<>(influencers, response.getHits().getTotalHits()); + } + + @Override + public Optional influencer(String jobId, String influencerId) + { + throw new IllegalStateException(); + } + + @Override + public BatchedDocumentsIterator newBatchedInfluencersIterator(String jobId) + { + return new ElasticsearchBatchedInfluencersIterator(client, jobId, parseFieldMatcher); + } + + @Override + public BatchedDocumentsIterator newBatchedModelSnapshotIterator(String jobId) + { + return new ElasticsearchBatchedModelSnapshotIterator(client, jobId, parseFieldMatcher); + } + + @Override + public BatchedDocumentsIterator newBatchedModelDebugOutputIterator(String jobId) + { + return new ElasticsearchBatchedModelDebugOutputIterator(client, jobId, parseFieldMatcher); + } + + @Override + public BatchedDocumentsIterator newBatchedModelSizeStatsIterator(String jobId) + { + return new ElasticsearchBatchedModelSizeStatsIterator(client, jobId, parseFieldMatcher); + } + + @Override + public Optional getQuantiles(String jobId) + { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + try + { + LOGGER.trace("ES API CALL: get ID " + Quantiles.QUANTILES_ID + + " type " + Quantiles.TYPE + " from index " + indexName); + GetResponse response = client.prepareGet( + indexName, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID).get(); + if (!response.isExists()) + { + LOGGER.info("There are currently no quantiles for job " + jobId); + return Optional.empty(); + } + return Optional.of(createQuantiles(jobId, response)); + } + catch (IndexNotFoundException e) + { + LOGGER.error("Missing index when getting quantiles", e); + throw e; + } + } + + @Override + public QueryPage modelSnapshots(String jobId, int from, int size) + { + return modelSnapshots(jobId, from, size, null, null, null, true, null, null); + } + + @Override + public QueryPage modelSnapshots(String jobId, int from, int size, + String startEpochMs, String endEpochMs, String sortField, boolean sortDescending, + String snapshotId, String description) + { + boolean haveId = snapshotId != null && !snapshotId.isEmpty(); + boolean haveDescription = description != null && !description.isEmpty(); + ResultsFilterBuilder fb; + if (haveId || haveDescription) + { + BoolQueryBuilder query = QueryBuilders.boolQuery(); + if (haveId) + { + query.must(QueryBuilders.termQuery(ModelSnapshot.SNAPSHOT_ID.getPreferredName(), snapshotId)); + } + if (haveDescription) + { + query.must(QueryBuilders.termQuery(ModelSnapshot.DESCRIPTION.getPreferredName(), description)); + } + + fb = new ResultsFilterBuilder(query); + } + else + { + fb = new ResultsFilterBuilder(); + } + + return modelSnapshots(jobId, from, size, + (sortField == null || sortField.isEmpty()) ? ModelSnapshot.RESTORE_PRIORITY.getPreferredName() : sortField, + sortDescending, fb.timeRange( + ElasticsearchMappings.ES_TIMESTAMP, startEpochMs, endEpochMs).build()); + } + + private QueryPage modelSnapshots(String jobId, int from, int size, + String sortField, boolean sortDescending, QueryBuilder fb) + { + FieldSortBuilder sb = new FieldSortBuilder(esSortField(sortField)) + .order(sortDescending ? SortOrder.DESC : SortOrder.ASC); + + // Wrap in a constant_score because we always want to + // run it as a filter + fb = new ConstantScoreQueryBuilder(fb); + + SearchResponse searchResponse; + try + { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type " + ModelSnapshot.TYPE + + " from index " + indexName + " sort ascending " + esSortField(sortField) + + " with filter after sort from " + from + " size " + size); + searchResponse = client.prepareSearch(indexName) + .setTypes(ModelSnapshot.TYPE.getPreferredName()) + .addSort(sb) + .setQuery(fb) + .setFrom(from).setSize(size) + .get(); + } + catch (IndexNotFoundException e) + { + LOGGER.error("Failed to read modelSnapshots", e); + throw e; + } + + List results = new ArrayList<>(); + + for (SearchHit hit : searchResponse.getHits().getHits()) + { + // Remove the Kibana/Logstash '@timestamp' entry as stored in Elasticsearch, + // and replace using the API 'timestamp' key. + Object timestamp = hit.getSource().remove(ElasticsearchMappings.ES_TIMESTAMP); + hit.getSource().put(ModelSnapshot.TIMESTAMP.getPreferredName(), timestamp); + + Object o = hit.getSource().get(ModelSizeStats.TYPE.getPreferredName()); + if (o instanceof Map) + { + @SuppressWarnings("unchecked") + Map map = (Map)o; + Object ts = map.remove(ElasticsearchMappings.ES_TIMESTAMP); + map.put(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName(), ts); + } + + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser list", e); + } + ModelSnapshot modelSnapshot = ModelSnapshot.PARSER.apply(parser, () -> parseFieldMatcher); + results.add(modelSnapshot); + } + + return new QueryPage<>(results, searchResponse.getHits().getTotalHits()); + } + + @Override + public void updateModelSnapshot(String jobId, ModelSnapshot modelSnapshot, + boolean restoreModelSizeStats) + { + // For Elasticsearch the update can be done in exactly the same way as + // the original persist + ElasticsearchPersister persister = new ElasticsearchPersister(jobId, client); + persister.persistModelSnapshot(modelSnapshot); + + if (restoreModelSizeStats) + { + if (modelSnapshot.getModelSizeStats() != null) + { + persister.persistModelSizeStats(modelSnapshot.getModelSizeStats()); + } + if (modelSnapshot.getQuantiles() != null) + { + persister.persistQuantiles(modelSnapshot.getQuantiles()); + } + } + + // Commit so that when the REST API call that triggered the update + // returns the updated document is searchable + persister.commitWrites(); + } + + public void restoreStateToStream(String jobId, ModelSnapshot modelSnapshot, OutputStream restoreStream) throws IOException { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + + // First try to restore categorizer state. There are no snapshots for this, so the IDs simply + // count up until a document is not found. It's NOT an error to have no categorizer state. + int docNum = 0; + while (true) { + String docId = Integer.toString(++docNum); + + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName); + + GetResponse stateResponse = client.prepareGet(indexName, CategorizerState.TYPE, docId).get(); + if (!stateResponse.isExists()) { + break; + } + writeStateToStream(stateResponse.getSourceAsBytesRef(), restoreStream); + } + + // Finally try to restore model state. This must come after categorizer state because that's + // the order the C++ process expects. + int numDocs = modelSnapshot.getSnapshotDocCount(); + for (docNum = 1; docNum <= numDocs; ++docNum) { + String docId = String.format(Locale.ROOT, "%s_%d", modelSnapshot.getSnapshotId(), docNum); + + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, ModelState.TYPE, indexName); + + GetResponse stateResponse = client.prepareGet(indexName, ModelState.TYPE, docId).get(); + if (!stateResponse.isExists()) { + LOGGER.error("Expected {} documents for model state for {} snapshot {} but failed to find {}", + numDocs, jobId, modelSnapshot.getSnapshotId(), docId); + break; + } + writeStateToStream(stateResponse.getSourceAsBytesRef(), restoreStream); + } + } + + private void writeStateToStream(BytesReference source, OutputStream stream) throws IOException { + // The source bytes are already UTF-8. The C++ process wants UTF-8, so we + // can avoid converting to a Java String only to convert back again. + BytesRefIterator iterator = source.iterator(); + for (BytesRef ref = iterator.next(); ref != null; ref = iterator.next()) { + // There's a complication that the source can already have trailing 0 bytes + int length = ref.bytes.length; + while (length > 0 && ref.bytes[length - 1] == 0) { + --length; + } + if (length > 0) { + stream.write(ref.bytes, 0, length); + } + } + // This is dictated by RapidJSON on the C++ side; it treats a '\0' as end-of-file + // even when it's not really end-of-file, and this is what we need because we're + // sending multiple JSON documents via the same named pipe. + stream.write(0); + } + + private Quantiles createQuantiles(String jobId, GetResponse response) { + BytesReference source = response.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser quantiles", e); + } + Quantiles quantiles = Quantiles.PARSER.apply(parser, () -> parseFieldMatcher); + if (quantiles.getQuantileState() == null) + { + LOGGER.error("Inconsistency - no " + Quantiles.QUANTILE_STATE + + " field in quantiles for job " + jobId); + } + return quantiles; + } + + @Override + public Optional modelSizeStats(String jobId) + { + String indexName = ElasticsearchPersister.getJobIndexName(jobId); + + try + { + LOGGER.trace("ES API CALL: get ID " + ModelSizeStats.TYPE + + " type " + ModelSizeStats.TYPE + " from index " + indexName); + + GetResponse modelSizeStatsResponse = client.prepareGet( + indexName, ModelSizeStats.TYPE.getPreferredName(), ModelSizeStats.TYPE.getPreferredName()).get(); + + if (!modelSizeStatsResponse.isExists()) + { + String msg = "No memory usage details for job with id " + jobId; + LOGGER.warn(msg); + return Optional.empty(); + } + else + { + BytesReference source = modelSizeStatsResponse.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser model size stats", e); + } + ModelSizeStats modelSizeStats = ModelSizeStats.PARSER.apply(parser, () -> parseFieldMatcher).build(); + return Optional.of(modelSizeStats); + } + } + catch (IndexNotFoundException e) + { + LOGGER.warn("Missing index " + indexName, e); + return Optional.empty(); + } + } + + @Override + public Optional getList(String listId) { + GetResponse response = client.prepareGet(PRELERT_INFO_INDEX, ListDocument.TYPE.getPreferredName(), listId).get(); + if (!response.isExists()) + { + return Optional.empty(); + } + BytesReference source = response.getSourceAsBytesRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser list", e); + } + ListDocument listDocument = ListDocument.PARSER.apply(parser, () -> parseFieldMatcher); + return Optional.of(listDocument); + } + + @Override + public Auditor audit(String jobId) + { + // NORELEASE Create proper auditor or remove + // return new ElasticsearchAuditor(client, PRELERT_INFO_INDEX, jobId); + return new Auditor() { + @Override + public void info(String message) { + } + + @Override + public void warning(String message) { + } + + @Override + public void error(String message) { + } + + @Override + public void activity(String message) { + } + + @Override + public void activity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) { + } + }; + + } + + private String esSortField(String sortField) + { + // Beware: There's an assumption here that Bucket.TIMESTAMP, + // AnomalyRecord.TIMESTAMP, Influencer.TIMESTAMP and + // ModelSnapshot.TIMESTAMP are all the same + return sortField.equals(Bucket.TIMESTAMP.getPreferredName()) ? ElasticsearchMappings.ES_TIMESTAMP : sortField; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappings.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappings.java new file mode 100644 index 00000000000..f9c5e017f70 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappings.java @@ -0,0 +1,892 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.CategorizerState; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.ModelState; +import org.elasticsearch.xpack.prelert.job.audit.AuditActivity; +import org.elasticsearch.xpack.prelert.job.audit.AuditMessage; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AnomalyCause; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influence; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; +import org.elasticsearch.xpack.prelert.job.usage.Usage; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Static methods to create Elasticsearch mappings for the autodetect + * persisted objects/documents + *

+ * ElasticSearch automatically recognises array types so they are + * not explicitly mapped as such. For arrays of objects the type + * must be set to nested so the arrays are searched properly + * see https://www.elastic.co/guide/en/elasticsearch/guide/current/nested-objects.html + *

+ * It is expected that indexes to which these mappings are applied have their + * default analyzer set to "keyword", which does not tokenise fields. The + * index-wide default analyzer cannot be set via these mappings, so needs to be + * set in the index settings during index creation. Then the _all field has its + * analyzer set to "whitespace" by these mappings, so that _all gets tokenised + * using whitespace. + */ +public class ElasticsearchMappings { + /** + * String constants used in mappings + */ + static final String INDEX = "index"; + static final String NO = "false"; + static final String ALL = "_all"; + static final String ENABLED = "enabled"; + static final String ANALYZER = "analyzer"; + static final String WHITESPACE = "whitespace"; + static final String INCLUDE_IN_ALL = "include_in_all"; + static final String NESTED = "nested"; + static final String COPY_TO = "copy_to"; + static final String PARENT = "_parent"; + static final String PROPERTIES = "properties"; + static final String TYPE = "type"; + static final String DYNAMIC = "dynamic"; + + /** + * Name of the field used to store the timestamp in Elasticsearch. + * Note the field name is different to {@link org.elasticsearch.xpack.prelert.job.results.Bucket#TIMESTAMP} used by the + * API Bucket Resource, and is chosen for consistency with the default field name used by + * Logstash and Kibana. + */ + static final String ES_TIMESTAMP = "timestamp"; + + /** + * Name of the Elasticsearch field by which documents are sorted by default + */ + static final String ES_DOC = "_doc"; + + /** + * Elasticsearch data types + */ + static final String BOOLEAN = "boolean"; + static final String DATE = "date"; + static final String DOUBLE = "double"; + static final String INTEGER = "integer"; + static final String KEYWORD = "keyword"; + static final String LONG = "long"; + static final String OBJECT = "object"; + static final String TEXT = "text"; + + private ElasticsearchMappings() { + } + + + public static XContentBuilder dataCountsMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(DataCounts.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_BYTES.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain org.elasticsearch.xpack.prelert.job.results.Bucket}. + * The '_all' field is disabled as the document isn't meant to be searched. + */ + public static XContentBuilder bucketMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(Bucket.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(Bucket.RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.EVENT_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .startObject(Bucket.PARTITION_SCORES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain org.elasticsearch.xpack.prelert.job.results.BucketInfluencer}. + */ + public static XContentBuilder bucketInfluencerMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(BucketInfluencer.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(Bucket.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Partition normalized scores. There is one per bucket + * so the timestamp is sufficient to uniquely identify + * the document per bucket per job + *

+ * Partition field values and scores are nested objects. + */ + public static XContentBuilder bucketPartitionMaxNormalizedScores() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ReservedFieldNames.PARTITION_NORMALIZED_PROB_TYPE) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(ReservedFieldNames.PARTITION_NORMALIZED_PROBS) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder categorizerStateMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(CategorizerState.TYPE) + .field(ENABLED, false) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder categoryDefinitionMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(CategoryDefinition.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(CategoryDefinition.TERMS.getPreferredName()) + .field(TYPE, TEXT).field(INDEX, NO) + .endObject() + .startObject(CategoryDefinition.REGEX.getPreferredName()) + .field(TYPE, TEXT).field(INDEX, NO) + .endObject() + .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(CategoryDefinition.EXAMPLES.getPreferredName()) + .field(TYPE, TEXT).field(INDEX, NO) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Records have a _parent mapping to a {@linkplain org.elasticsearch.xpack.prelert.job.results.Bucket}. + * + * @param termFieldNames Optionally, other field names to include in the + * mappings. Pass null if not required. + */ + public static XContentBuilder recordMapping(Collection termFieldNames) throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(AnomalyRecord.TYPE.getPreferredName()) + .startObject(ALL) + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.DETECTOR_INDEX.getPreferredName()) + .field(TYPE, INTEGER).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.ACTUAL.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.TYPICAL.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.FUNCTION.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.FUNCTION_DESCRIPTION.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.BY_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.OVER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.OVER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.CAUSES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyCause.ACTUAL.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.TYPICAL.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.FUNCTION.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.FUNCTION_DESCRIPTION.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.BY_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.CORRELATED_BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.OVER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyCause.OVER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject() + .startObject(AnomalyRecord.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(AnomalyRecord.INFLUENCERS.getPreferredName()) + /* Array of influences */ + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(Influence.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influence.INFLUENCER_FIELD_VALUES.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .endObject() + .endObject(); + + if (termFieldNames != null) { + ElasticsearchDotNotationReverser reverser = new ElasticsearchDotNotationReverser(); + for (String fieldName : termFieldNames) { + reverser.add(fieldName, ""); + } + for (Map.Entry entry : reverser.getMappingsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + } + + return builder + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain Quantiles}. + * The '_all' field is disabled as the document isn't meant to be searched. + *

+ * The quantile state string is not searchable (index = 'no') as it could be + * very large. + */ + public static XContentBuilder quantilesMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(Quantiles.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(Quantiles.QUANTILE_STATE.getPreferredName()) + .field(TYPE, TEXT).field(INDEX, NO) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain ModelState}. + * The model state could potentially be huge (over a gigabyte in size) + * so all analysis by Elasticsearch is disabled. The only way to + * retrieve the model state is by knowing the ID of a particular + * document or by searching for all documents of this type. + */ + public static XContentBuilder modelStateMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ModelState.TYPE) + .field(ENABLED, false) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain ModelState}. + * The model state could potentially be huge (over a gigabyte in size) + * so all analysis by Elasticsearch is disabled. The only way to + * retrieve the model state is by knowing the ID of a particular + * document or by searching for all documents of this type. + */ + public static XContentBuilder modelSnapshotMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ModelSnapshot.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + // "description" is analyzed so that it has the same + // mapping as a user field of the same name - this means + // it doesn't have to be a reserved field name + .startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(ModelSnapshot.RESTORE_PRIORITY.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(ModelSizeStats.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .startObject(Quantiles.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(Quantiles.QUANTILE_STATE.getPreferredName()) + .field(TYPE, TEXT).field(INDEX, NO) + .endObject() + .endObject() + .endObject() + .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain ModelSizeStats}. + */ + public static XContentBuilder modelSizeStatsMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ModelSizeStats.TYPE.getPreferredName()) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Mapping for model debug output + * + * @param termFieldNames Optionally, other field names to include in the + * mappings. Pass null if not required. + */ + public static XContentBuilder modelDebugOutputMapping(Collection termFieldNames) throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(ModelDebugOutput.TYPE.getPreferredName()) + .startObject(ALL) + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ModelDebugOutput.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelDebugOutput.OVER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelDebugOutput.BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelDebugOutput.DEBUG_FEATURE.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ModelDebugOutput.DEBUG_LOWER.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ModelDebugOutput.DEBUG_UPPER.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ModelDebugOutput.DEBUG_MEDIAN.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ModelDebugOutput.ACTUAL.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject(); + + if (termFieldNames != null) { + ElasticsearchDotNotationReverser reverser = new ElasticsearchDotNotationReverser(); + for (String fieldName : termFieldNames) { + reverser.add(fieldName, ""); + } + for (Map.Entry entry : reverser.getMappingsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + } + + return builder + .endObject() + .endObject() + .endObject(); + } + + /** + * Influence results mapping + * + * @param influencerFieldNames Optionally, other field names to include in the + * mappings. Pass null if not required. + */ + public static XContentBuilder influencerMapping(Collection influencerFieldNames) throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(Influencer.TYPE.getPreferredName()) + .startObject(ALL) + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD).field(INCLUDE_IN_ALL, false) + .endObject() + .startObject(Influencer.INFLUENCER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN).field(INCLUDE_IN_ALL, false) + .endObject(); + + if (influencerFieldNames != null) { + ElasticsearchDotNotationReverser reverser = new ElasticsearchDotNotationReverser(); + for (String fieldName : influencerFieldNames) { + reverser.add(fieldName, ""); + } + for (Map.Entry entry : reverser.getMappingsMap().entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + } + + return builder + .endObject() + .endObject() + .endObject(); + } + + /** + * The Elasticsearch mappings for the usage documents + */ + public static XContentBuilder usageMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(Usage.TYPE) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .startObject(Usage.INPUT_BYTES) + .field(TYPE, LONG) + .endObject() + .startObject(Usage.INPUT_FIELD_COUNT) + .field(TYPE, LONG) + .endObject() + .startObject(Usage.INPUT_RECORD_COUNT) + .field(TYPE, LONG) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder auditMessageMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(AuditMessage.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder auditActivityMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(AuditActivity.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ES_TIMESTAMP) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder processingTimeMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE) + .startObject(ALL) + .field(ENABLED, false) + // analyzer must be specified even though _all is disabled + // because all types in the same index must have the same + // analyzer for a given field + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(PROPERTIES) + .startObject(ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + .endObject(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersister.java new file mode 100644 index 00000000000..25a91ad28b2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersister.java @@ -0,0 +1,558 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Map.Entry; +import java.util.function.Supplier; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Saves result Buckets and Quantiles to Elasticsearch
+ * + * Buckets are written with the following structure: + *

Bucket

The results of each job are stored in buckets, this is the + * top level structure for the results. A bucket contains multiple anomaly + * records. The anomaly score of the bucket may not match the summed score of + * all the records as all the records may not have been outputted for the + * bucket. + *

Anomaly Record

In Elasticsearch records have a parent <-< + * child relationship with buckets and should only exist is relation to a parent + * bucket. Each record was generated by a detector which can be identified via + * the detectorIndex field. + *

Detector

The Job has a fixed number of detectors but there may not + * be output for every detector in each bucket.
+ * Quantiles may contain model quantiles used in normalisation and are + * stored in documents of type {@link Quantiles#TYPE}
+ *

ModelSizeStats

This is stored in a flat structure
+ * + * @see org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchMappings + */ +public class ElasticsearchPersister implements JobResultsPersister, JobRenormaliser +{ + + private static final Logger LOGGER = Loggers.getLogger(ElasticsearchPersister.class); + + public static final String INDEX_PREFIX = "prelertresults-"; + + public static String getJobIndexName(String jobId) { + return INDEX_PREFIX + jobId; + } + + private final Client client; + // TODO norelease: remove this field, the job id can be interred from most of method's paramters here and for cases + // where there are now parameters we can supply the job id. This way we don't have to create an instance of the class + // per job + private final String jobId; + + /** + * Create with the Elasticsearch client. Data will be written to + * the index jobId + * + * @param jobId The job Id/Elasticsearch index + * @param client The Elasticsearch client + */ + public ElasticsearchPersister(String jobId, Client client) + { + this.jobId = jobId; + this.client = client; + } + + @Override + public void persistBucket(Bucket bucket) + { + if (bucket.getRecords() == null) + { + return; + } + + try + { + XContentBuilder content = serialiseWithJobId(Bucket.TYPE.getPreferredName(), bucket); + + String indexName = getJobIndexName(jobId); + LOGGER.trace("ES API CALL: index type " + Bucket.TYPE + + " to index " + indexName + " at epoch " + bucket.getEpoch()); + IndexResponse response = client.prepareIndex(indexName, Bucket.TYPE.getPreferredName()) + .setSource(content) + .execute().actionGet(); + + bucket.setId(response.getId()); + + persistBucketInfluencersStandalone(bucket.getId(), bucket.getBucketInfluencers(), + bucket.getTimestamp(), bucket.isInterim()); + + if (bucket.getInfluencers() != null && bucket.getInfluencers().isEmpty() == false) + { + BulkRequestBuilder addInfluencersRequest = client.prepareBulk(); + + for (Influencer influencer : bucket.getInfluencers()) + { + influencer.setTimestamp(bucket.getTimestamp()); + influencer.setInterim(bucket.isInterim()); + content = serialiseWithJobId(Influencer.TYPE.getPreferredName(), influencer); + LOGGER.trace("ES BULK ACTION: index type " + Influencer.TYPE + + " to index " + indexName + " with auto-generated ID"); + addInfluencersRequest.add( + client.prepareIndex(indexName, Influencer.TYPE.getPreferredName()) + .setSource(content)); + } + + LOGGER.trace("ES API CALL: bulk request with " + addInfluencersRequest.numberOfActions() + " actions"); + BulkResponse addInfluencersResponse = addInfluencersRequest.execute().actionGet(); + if (addInfluencersResponse.hasFailures()) + { + LOGGER.error("Bulk index of Influencers has errors: " + + addInfluencersResponse.buildFailureMessage()); + } + } + + if (bucket.getRecords().isEmpty() == false) + { + BulkRequestBuilder addRecordsRequest = client.prepareBulk(); + for (AnomalyRecord record : bucket.getRecords()) + { + record.setTimestamp(bucket.getTimestamp()); + content = serialiseWithJobId(AnomalyRecord.TYPE.getPreferredName(), record); + + LOGGER.trace("ES BULK ACTION: index type " + AnomalyRecord.TYPE + + " to index " + indexName + " with auto-generated ID, for bucket " + + bucket.getId()); + addRecordsRequest.add(client.prepareIndex(indexName, AnomalyRecord.TYPE.getPreferredName()) + .setSource(content) + .setParent(bucket.getId())); + } + + LOGGER.trace("ES API CALL: bulk request with " + addRecordsRequest.numberOfActions() + " actions"); + BulkResponse addRecordsResponse = addRecordsRequest.execute().actionGet(); + if (addRecordsResponse.hasFailures()) + { + LOGGER.error("Bulk index of AnomalyRecord has errors: " + + addRecordsResponse.buildFailureMessage()); + } + } + + persistPerPartitionMaxProbabilities(bucket); + } + catch (IOException e) + { + LOGGER.error("Error writing bucket state", e); + } + } + + private void persistPerPartitionMaxProbabilities(Bucket bucket) + { + if (bucket.getPerPartitionMaxProbability().isEmpty()) + { + return; + } + + try + { + XContentBuilder builder = jsonBuilder(); + + builder.startObject() + .field(ElasticsearchMappings.ES_TIMESTAMP, bucket.getTimestamp()) + .field(Job.ID.getPreferredName(), jobId); + builder.startArray(ReservedFieldNames.PARTITION_NORMALIZED_PROBS); + for (Entry entry : bucket.getPerPartitionMaxProbability().entrySet()) + { + builder.startObject() + .field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), entry.getKey()) + .field(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), entry.getValue()) + .endObject(); + } + builder.endArray().endObject(); + + String indexName = getJobIndexName(jobId); + LOGGER.trace("ES API CALL: index type " + ReservedFieldNames.PARTITION_NORMALIZED_PROB_TYPE + + " to index " + indexName + " at epoch " + bucket.getEpoch()); + client.prepareIndex(indexName, ReservedFieldNames.PARTITION_NORMALIZED_PROB_TYPE) + .setSource(builder) + .setId(bucket.getId()) + .execute().actionGet(); + } + catch (IOException e) + { + LOGGER.error("Error updating bucket per partition max normalized scores", e); + return; + } + } + + @Override + public void persistCategoryDefinition(CategoryDefinition category) + { + Persistable persistable = new Persistable(category, () -> CategoryDefinition.TYPE.getPreferredName(), + () -> String.valueOf(category.getCategoryId()), + () -> serialiseCategoryDefinition(category)); + persistable.persist(); + + // Don't commit as we expect masses of these updates and they're not + // read again by this process + } + + /** + * The quantiles objects are written with a fixed ID, so that the + * latest quantiles will overwrite the previous ones. For each ES index, + * which corresponds to a job, there can only be one quantiles document. + * @param quantiles If null then returns straight away. + */ + @Override + public void persistQuantiles(Quantiles quantiles) + { + Persistable persistable = new Persistable(quantiles, () -> Quantiles.TYPE.getPreferredName(), + () -> Quantiles.QUANTILES_ID, () -> serialiseWithJobId(Quantiles.TYPE.getPreferredName(), quantiles)); + if (persistable.persist()) + { + // Refresh the index when persisting quantiles so that previously + // persisted results will be available for searching. Do this using the + // indices API rather than the index API (used to write the quantiles + // above), because this will refresh all shards rather than just the + // shard that the quantiles document itself was written to. + commitWrites(); + } + } + + /** + * Write a model snapshot description to Elasticsearch. Note that this is + * only the description - the actual model state is persisted separately. + * @param modelSnapshot If null then returns straight away. + */ + @Override + public void persistModelSnapshot(ModelSnapshot modelSnapshot) + { + Persistable persistable = new Persistable(modelSnapshot, () -> ModelSnapshot.TYPE.getPreferredName(), + () -> modelSnapshot.getSnapshotId(), () -> serialiseWithJobId(ModelSnapshot.TYPE.getPreferredName(), modelSnapshot)); + persistable.persist(); + } + + /** + * Persist the memory usage data + * @param modelSizeStats If null then returns straight away. + */ + @Override + public void persistModelSizeStats(ModelSizeStats modelSizeStats) + { + LOGGER.trace("Persisting model size stats, for size " + modelSizeStats.getModelBytes()); + Persistable persistable = new Persistable(modelSizeStats, () -> ModelSizeStats.TYPE.getPreferredName(), + () -> modelSizeStats.getId(), + () -> serialiseWithJobId(ModelSizeStats.TYPE.getPreferredName(), modelSizeStats)); + persistable.persist(); + + persistable = new Persistable(modelSizeStats, () -> ModelSizeStats.TYPE.getPreferredName(), + () -> null, + () -> serialiseWithJobId(ModelSizeStats.TYPE.getPreferredName(), modelSizeStats)); + persistable.persist(); + + // Don't commit as we expect masses of these updates and they're only + // for information at the API level + } + + /** + * Persist model debug output + * @param modelDebugOutput If null then returns straight away. + */ + @Override + public void persistModelDebugOutput(ModelDebugOutput modelDebugOutput) + { + Persistable persistable = new Persistable(modelDebugOutput, () -> ModelDebugOutput.TYPE.getPreferredName(), + () -> null, () -> serialiseWithJobId(ModelDebugOutput.TYPE.getPreferredName(), modelDebugOutput)); + persistable.persist(); + + // Don't commit as we expect masses of these updates and they're not + // read again by this process + } + + @Override + public void persistInfluencer(Influencer influencer) + { + Persistable persistable = new Persistable(influencer, () -> Influencer.TYPE.getPreferredName(), + () -> influencer.getId(), () -> serialiseWithJobId(Influencer.TYPE.getPreferredName(), influencer)); + persistable.persist(); + + // Don't commit as we expect masses of these updates and they're not + // read again by this process + } + + @Override + public void persistBulkState(BytesReference bytesRef) { + try { + // No validation - assume the native process has formatted the state correctly + byte[] bytes = bytesRef.toBytesRef().bytes; + LOGGER.trace("ES API CALL: bulk index"); + client.prepareBulk() + .add(bytes, 0, bytes.length) + .execute().actionGet(); + } catch (Exception e) { + LOGGER.error("Error persisting bulk state", e); + } + } + + /** + * Refreshes the Elasticsearch index. + * Blocks until results are searchable. + */ + @Override + public boolean commitWrites() + { + String indexName = getJobIndexName(jobId); + // Refresh should wait for Lucene to make the data searchable + LOGGER.trace("ES API CALL: refresh index " + indexName); + client.admin().indices().refresh(new RefreshRequest(indexName)).actionGet(); + return true; + } + + @Override + public void updateBucket(Bucket bucket) + { + try + { + String indexName = getJobIndexName(jobId); + LOGGER.trace("ES API CALL: index type " + Bucket.TYPE + + " to index " + indexName + " with ID " + bucket.getId()); + client.prepareIndex(indexName, Bucket.TYPE.getPreferredName(), bucket.getId()) + .setSource(serialiseWithJobId(Bucket.TYPE.getPreferredName(), bucket)).execute().actionGet(); + } + catch (IOException e) + { + LOGGER.error("Error updating bucket state", e); + return; + } + + // If the update to the bucket was successful, also update the + // standalone copies of the nested bucket influencers + try + { + persistBucketInfluencersStandalone(bucket.getId(), bucket.getBucketInfluencers(), + bucket.getTimestamp(), bucket.isInterim()); + } + catch (IOException e) + { + LOGGER.error("Error updating standalone bucket influencer state", e); + return; + } + + persistPerPartitionMaxProbabilities(bucket); + } + + private void persistBucketInfluencersStandalone(String bucketId, List bucketInfluencers, + Date bucketTime, boolean isInterim) throws IOException + { + if (bucketInfluencers != null && bucketInfluencers.isEmpty() == false) + { + BulkRequestBuilder addBucketInfluencersRequest = client.prepareBulk(); + + for (BucketInfluencer bucketInfluencer : bucketInfluencers) + { + XContentBuilder content = serialiseBucketInfluencerStandalone(bucketInfluencer, + bucketTime, isInterim); + // Need consistent IDs to ensure overwriting on renormalisation + String id = bucketId + bucketInfluencer.getInfluencerFieldName(); + String indexName = getJobIndexName(jobId); + LOGGER.trace("ES BULK ACTION: index type " + BucketInfluencer.TYPE + + " to index " + indexName + " with ID " + id); + addBucketInfluencersRequest.add( + client.prepareIndex(indexName, BucketInfluencer.TYPE.getPreferredName(), id) + .setSource(content)); + } + + LOGGER.trace("ES API CALL: bulk request with " + addBucketInfluencersRequest.numberOfActions() + " actions"); + BulkResponse addBucketInfluencersResponse = addBucketInfluencersRequest.execute().actionGet(); + if (addBucketInfluencersResponse.hasFailures()) + { + LOGGER.error("Bulk index of Bucket Influencers has errors: " + + addBucketInfluencersResponse.buildFailureMessage()); + } + } + } + + @Override + public void updateRecords(String bucketId, List records) + { + try + { + // Now bulk update the records within the bucket + BulkRequestBuilder bulkRequest = client.prepareBulk(); + boolean addedAny = false; + for (AnomalyRecord record : records) + { + String recordId = record.getId(); + String indexName = getJobIndexName(jobId); + LOGGER.trace("ES BULK ACTION: update ID " + recordId + " type " + AnomalyRecord.TYPE + + " in index " + indexName + " using map of new values, for bucket " + + bucketId); + + bulkRequest.add( + client.prepareIndex(indexName, AnomalyRecord.TYPE.getPreferredName(), recordId) + .setSource(serialiseWithJobId(AnomalyRecord.TYPE.getPreferredName(), record)) + // Need to specify the parent ID when updating a child + .setParent(bucketId)); + + addedAny = true; + } + + if (addedAny) + { + LOGGER.trace("ES API CALL: bulk request with " + + bulkRequest.numberOfActions() + " actions"); + BulkResponse bulkResponse = bulkRequest.execute().actionGet(); + if (bulkResponse.hasFailures()) + { + LOGGER.error("BulkResponse has errors: " + bulkResponse.buildFailureMessage()); + } + } + } + catch (IOException | ElasticsearchException e) + { + LOGGER.error("Error updating anomaly records", e); + } + } + + @Override + public void updateInfluencer(Influencer influencer) + { + persistInfluencer(influencer); + } + + @Override + public void deleteInterimResults() + { + ElasticsearchBulkDeleter deleter = new ElasticsearchBulkDeleter(client, jobId, true); + deleter.deleteInterimResults(); + + // NOCOMMIT This is called from AutodetectResultsParser, feels wrong... + deleter.commit(new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + // don't care? + } + + @Override + public void onFailure(Exception e) { + // don't care? + } + }); + } + + private interface Serialiser + { + XContentBuilder serialise() throws IOException; + } + + private class Persistable + { + private final Object object; + private final Supplier typeSupplier; + private final Supplier idSupplier; + private final Serialiser serialiser; + + Persistable(Object object, Supplier typeSupplier, Supplier idSupplier, + Serialiser serialiser) + { + this.object = object; + this.typeSupplier = typeSupplier; + this.idSupplier = idSupplier; + this.serialiser = serialiser; + } + + boolean persist() + { + String type = typeSupplier.get(); + String id = idSupplier.get(); + + if (object == null) + { + LOGGER.warn("No " + type + " to persist for job " + jobId); + return false; + } + + logCall(type, id); + + try + { + String indexName = getJobIndexName(jobId); + client.prepareIndex(indexName, type, idSupplier.get()) + .setSource(serialiser.serialise()) + .execute().actionGet(); + return true; + } + catch (IOException e) + { + LOGGER.error("Error writing " + typeSupplier.get(), e); + return false; + } + } + + private void logCall(String type, String id) + { + String indexName = getJobIndexName(jobId); + String msg = "ES API CALL: index type " + type + " to index " + indexName; + if (id != null) + { + msg += " with ID " + idSupplier.get(); + } + else + { + msg += " with auto-generated ID"; + } + LOGGER.trace(msg); + } + } + + private XContentBuilder serialiseWithJobId(String objField, ToXContent obj) throws IOException + { + XContentBuilder builder = jsonBuilder(); + obj.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + private XContentBuilder serialiseCategoryDefinition(CategoryDefinition categoryDefinition) + throws IOException + { + XContentBuilder builder = jsonBuilder(); + categoryDefinition.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + private XContentBuilder serialiseBucketInfluencerStandalone(BucketInfluencer bucketInfluencer, + Date bucketTime, boolean isInterim) throws IOException + { + BucketInfluencer influencer = new BucketInfluencer(bucketInfluencer); + influencer.setIsInterim(isInterim); + influencer.setTimestamp(bucketTime); + XContentBuilder builder = jsonBuilder(); + influencer.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScripts.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScripts.java new file mode 100644 index 00000000000..e3881ca88f5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScripts.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.client.Client; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.util.HashMap; +import java.util.Map; + +/** + * Create methods for the custom scripts that are run on Elasticsearch + */ +public final class ElasticsearchScripts { + + private static final String PAINLESS = "painless"; + + // Script names + private static final String UPDATE_AVERAGE_PROCESSING_TIME = "ctx._source.averageProcessingTimeMs = ctx._source.averageProcessingTimeMs" + + " * 0.9 + params.timeMs * 0.1"; + private static final String UPDATE_BUCKET_COUNT = "ctx._source.counts.bucketCount += params.count"; + private static final String UPDATE_USAGE = "ctx._source.inputBytes += params.bytes;ctx._source.inputFieldCount += params.fieldCount;" + + "ctx._source.inputRecordCount += params.recordCount;"; + + // Script parameters + private static final String COUNT_PARAM = "count"; + private static final String BYTES_PARAM = "bytes"; + private static final String FIELD_COUNT_PARAM = "fieldCount"; + private static final String RECORD_COUNT_PARAM = "recordCount"; + private static final String PROCESSING_TIME_PARAM = "timeMs"; + + public static final int UPDATE_JOB_RETRY_COUNT = 3; + + private ElasticsearchScripts() + { + // Do nothing + } + + public static Script newUpdateBucketCount(long count) + { + Map scriptParams = new HashMap<>(); + scriptParams.put(COUNT_PARAM, count); + return new Script(ScriptType.INLINE, PAINLESS, UPDATE_BUCKET_COUNT, scriptParams); + } + + public static Script newUpdateUsage(long additionalBytes, long additionalFields, + long additionalRecords) + { + Map scriptParams = new HashMap<>(); + scriptParams.put(BYTES_PARAM, additionalBytes); + scriptParams.put(FIELD_COUNT_PARAM, additionalFields); + scriptParams.put(RECORD_COUNT_PARAM, additionalRecords); + return new Script(ScriptType.INLINE, PAINLESS, UPDATE_USAGE, scriptParams); + } + + public static Script updateProcessingTime(Long processingTimeMs) + { + Map scriptParams = new HashMap<>(); + scriptParams.put(PROCESSING_TIME_PARAM, processingTimeMs); + return new Script(ScriptType.INLINE, PAINLESS, UPDATE_AVERAGE_PROCESSING_TIME, scriptParams); + } + + /** + * Updates the specified document via executing a script + * + * @param client + * the Elasticsearch client + * @param index + * the index + * @param type + * the document type + * @param docId + * the document id + * @param script + * the script the performs the update + * @return {@code} true if successful, {@code} false otherwise + */ + public static boolean updateViaScript(Client client, String index, String type, String docId, Script script) { + try { + client.prepareUpdate(index, type, docId) + .setScript(script) + .setRetryOnConflict(UPDATE_JOB_RETRY_COUNT).get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(index); + } + return true; + } + + /** + * Upserts the specified document via executing a script + * + * @param client + * the Elasticsearch client + * @param index + * the index + * @param type + * the document type + * @param docId + * the document id + * @param script + * the script the performs the update + * @param upsertMap + * the doc source of the update request to be used when the + * document does not exists + * @return {@code} true if successful, {@code} false otherwise + */ + public static boolean upsertViaScript(Client client, String index, String type, String docId, Script script, + Map upsertMap) { + try { + client.prepareUpdate(index, type, docId) + .setScript(script) + .setUpsert(upsertMap) + .setRetryOnConflict(UPDATE_JOB_RETRY_COUNT).get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(index); + } + return true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchUsagePersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchUsagePersister.java new file mode 100644 index 00000000000..79bc42ecb68 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchUsagePersister.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import static org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchJobProvider.PRELERT_USAGE_INDEX; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.engine.VersionConflictEngineException; + +import org.elasticsearch.xpack.prelert.job.usage.Usage; + +public class ElasticsearchUsagePersister implements UsagePersister { + private static final String USAGE_DOC_ID_PREFIX = "usage-"; + + private final Client client; + private final Logger logger; + private final DateTimeFormatter dateTimeFormatter; + private final Map upsertMap; + private String docId; + + public ElasticsearchUsagePersister(Client client, Logger logger) { + this.client = client; + this.logger = logger; + dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXX", Locale.ROOT); + upsertMap = new HashMap<>(); + + upsertMap.put(ElasticsearchMappings.ES_TIMESTAMP, ""); + upsertMap.put(Usage.INPUT_BYTES, null); + } + + @Override + public void persistUsage(String jobId, long bytesRead, long fieldsRead, long recordsRead) { + ZonedDateTime nowTruncatedToHour = ZonedDateTime.now().truncatedTo(ChronoUnit.HOURS); + String formattedNowTruncatedToHour = nowTruncatedToHour.format(dateTimeFormatter); + docId = USAGE_DOC_ID_PREFIX + formattedNowTruncatedToHour; + upsertMap.put(ElasticsearchMappings.ES_TIMESTAMP, formattedNowTruncatedToHour); + + // update global count + updateDocument(PRELERT_USAGE_INDEX, docId, bytesRead, fieldsRead, recordsRead); + updateDocument(ElasticsearchPersister.getJobIndexName(jobId), docId, bytesRead, + fieldsRead, recordsRead); + } + + + /** + * Update the metering document in the given index/id. + * Uses a script to update the volume field and 'upsert' + * to create the doc if it doesn't exist. + * + * @param index the index to persist to + * @param id Doc id is also its timestamp + * @param additionalBytes Add this value to the running total + * @param additionalFields Add this value to the running total + * @param additionalRecords Add this value to the running total + */ + private void updateDocument(String index, String id, long additionalBytes, long additionalFields, long additionalRecords) { + upsertMap.put(Usage.INPUT_BYTES, additionalBytes); + upsertMap.put(Usage.INPUT_FIELD_COUNT, additionalFields); + upsertMap.put(Usage.INPUT_RECORD_COUNT, additionalRecords); + + logger.trace("ES API CALL: upsert ID " + id + + " type " + Usage.TYPE + " in index " + index + + " by running Groovy script update-usage with arguments bytes=" + additionalBytes + + " fieldCount=" + additionalFields + " recordCount=" + additionalRecords); + + try { + ElasticsearchScripts.upsertViaScript(client, index, Usage.TYPE, id, + ElasticsearchScripts.newUpdateUsage(additionalBytes, additionalFields, + additionalRecords), + upsertMap); + } catch (VersionConflictEngineException e) { + logger.error("Failed to update the Usage document [" + id +"] in index [" + index + "]", e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/InfluencersQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/InfluencersQueryBuilder.java new file mode 100644 index 00000000000..87f09e0ba44 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/InfluencersQueryBuilder.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.xpack.prelert.job.results.Influencer; + +import java.util.Objects; + +/** + * One time query builder for buckets. + *
    + *
  • From- Skip the first N Buckets. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of Buckets. Default = + * {@value DEFAULT_SIZE}
    • + *
  • Expand- Include anomaly records. Default= false
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • normalizedProbabilityThreshold- Return only buckets with a + * maxNormalizedProbability >= this value. Default = 0.0
    • + *
  • epochStart- The start bucket time. A bucket with this timestamp will be + * included in the results. If 0 all buckets up to endEpochMs are + * returned. Default = -1
    • + *
  • epochEnd- The end bucket timestamp buckets up to but NOT including this + * timestamp are returned. If 0 all buckets from startEpochMs are + * returned. Default = -1
    • + *
  • partitionValue Set the bucket's max normalised probability to this + * partition field value's max normalised probability. Default = null
    • + *
+ */ +public final class InfluencersQueryBuilder { + public static final int DEFAULT_SIZE = 100; + + private InfluencersQuery influencersQuery = new InfluencersQuery(); + + public InfluencersQueryBuilder from(int from) { + influencersQuery.from = from; + return this; + } + + public InfluencersQueryBuilder size(int size) { + influencersQuery.size = size; + return this; + } + + public InfluencersQueryBuilder includeInterim(boolean include) { + influencersQuery.includeInterim = include; + return this; + } + + public InfluencersQueryBuilder anomalyScoreThreshold(Double anomalyScoreFilter) { + influencersQuery.anomalyScoreFilter = anomalyScoreFilter; + return this; + } + + public InfluencersQueryBuilder sortField(String sortField) { + influencersQuery.sortField = sortField; + return this; + } + + public InfluencersQueryBuilder sortDescending(boolean sortDescending) { + influencersQuery.sortDescending = sortDescending; + return this; + } + + /** + * If startTime >= 0 the parameter is not set + */ + public InfluencersQueryBuilder epochStart(String startTime) { + influencersQuery.epochStart = startTime; + return this; + } + + /** + * If endTime >= 0 the parameter is not set + */ + public InfluencersQueryBuilder epochEnd(String endTime) { + influencersQuery.epochEnd = endTime; + return this; + } + + public InfluencersQueryBuilder.InfluencersQuery build() { + return influencersQuery; + } + + public void clear() { + influencersQuery = new InfluencersQueryBuilder.InfluencersQuery(); + } + + + public class InfluencersQuery { + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean includeInterim = false; + private double anomalyScoreFilter = 0.0d; + private String epochStart; + private String epochEnd; + private String sortField = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean sortDescending = false; + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public String getEpochStart() { + return epochStart; + } + + public String getEpochEnd() { + return epochEnd; + } + + public String getSortField() { + return sortField; + } + + public boolean isSortDescending() { + return sortDescending; + } + + @Override + public int hashCode() { + return Objects.hash(from, size, includeInterim, anomalyScoreFilter, epochStart, epochEnd, + sortField, sortDescending); + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + InfluencersQuery other = (InfluencersQuery) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(epochStart, other.epochStart) && + Objects.equals(epochStart, other.epochStart) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(sortField, other.sortField) && + this.sortDescending == other.sortDescending; + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataCountsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataCountsPersister.java new file mode 100644 index 00000000000..71af291683a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataCountsPersister.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.xpack.prelert.job.DataCounts; + +/** + * Update a job's dataCounts + * i.e. the number of processed records, fields etc. + */ +public interface JobDataCountsPersister +{ + /** + * Update the job's data counts stats and figures. + * + * @param jobId Job to update + * @param counts The counts + */ + void persistDataCounts(String jobId, DataCounts counts); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleter.java new file mode 100644 index 00000000000..5c1c8eda2a1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleter.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; + +public interface JobDataDeleter { + /** + * Delete a {@code Bucket} and its records + * + * @param bucket the bucket to delete + */ + void deleteBucket(Bucket bucket); + + /** + * Delete the records of a {@code Bucket} + * + * @param bucket the bucket whose records to delete + */ + void deleteRecords(Bucket bucket); + + /** + * Delete an {@code Influencer} + * + * @param influencer the influencer to delete + */ + void deleteInfluencer(Influencer influencer); + + /** + * Delete a {@code ModelSnapshot} + * + * @param modelSnapshot the model snapshot to delete + */ + void deleteModelSnapshot(ModelSnapshot modelSnapshot); + + /** + * Delete a {@code ModelDebugOutput} record + * + * @param modelDebugOutput to delete + */ + void deleteModelDebugOutput(ModelDebugOutput modelDebugOutput); + + /** + * Delete a {@code ModelSizeStats} record + * + * @param modelSizeStats to delete + */ + void deleteModelSizeStats(ModelSizeStats modelSizeStats); + + /** + * Commit the deletions without enforcing the removal of data from disk + */ + void commit(ActionListener listener); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleterFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleterFactory.java new file mode 100644 index 00000000000..b3a8eb7aff4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobDataDeleterFactory.java @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +public interface JobDataDeleterFactory +{ + JobDataDeleter newDeleter(String jobId); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java new file mode 100644 index 00000000000..0b1dd5bf1a5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProvider.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.lists.ListDocument; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Optional; + +public interface JobProvider extends JobResultsProvider { + + /** + * Get the persisted quantiles state for the job + */ + Optional getQuantiles(String jobId); + + /** + * Get model snapshots for the job ordered by descending restore priority. + * + * @param jobId the job id + * @param from number of snapshots to from + * @param size number of snapshots to retrieve + * @return page of model snapshots + */ + QueryPage modelSnapshots(String jobId, int from, int size); + + /** + * Get model snapshots for the job ordered by descending restore priority. + * + * @param jobId the job id + * @param from number of snapshots to from + * @param size number of snapshots to retrieve + * @param startEpochMs earliest time to include (inclusive) + * @param endEpochMs latest time to include (exclusive) + * @param sortField optional sort field name (may be null) + * @param sortDescending Sort in descending order + * @param snapshotId optional snapshot ID to match (null for all) + * @param description optional description to match (null for all) + * @return page of model snapshots + */ + QueryPage modelSnapshots(String jobId, int from, int size, + String startEpochMs, String endEpochMs, String sortField, boolean sortDescending, + String snapshotId, String description); + + /** + * Update a persisted model snapshot metadata document to match the + * argument supplied. + * + * @param jobId the job id + * @param modelSnapshot the updated model snapshot object to be stored + * @param restoreModelSizeStats should the model size stats in this + * snapshot be made the current ones for this job? + */ + void updateModelSnapshot(String jobId, ModelSnapshot modelSnapshot, + boolean restoreModelSizeStats); + + /** + * Given a model snapshot, get the corresponding state and write it to the supplied + * stream. If there are multiple state documents they are separated using '\0' + * when written to the stream. + * @param jobId the job id + * @param modelSnapshot the model snapshot to be restored + * @param restoreStream the stream to write the state to + */ + void restoreStateToStream(String jobId, ModelSnapshot modelSnapshot, OutputStream restoreStream) throws IOException; + + /** + * Get the job's model size stats. + */ + Optional modelSizeStats(String jobId); + + /** + * Retrieves the list with the given {@code listId} from the datastore. + * + * @param listId the id of the requested list + * @return the matching list if it exists + */ + Optional getList(String listId); + + /** + * Get an auditor for the given job + * + * @param jobId the job id + * @return the {@code Auditor} + */ + Auditor audit(String jobId); + + /** + * Save the details of the new job to the datastore. + * Throws JobIdAlreadyExistsException if a job with the + * same Id already exists. + */ + // TODO: rename and move? + void createJobRelatedIndices(Job job, ActionListener listener); + + /** + * Delete all the job related documents from the database. + */ + // TODO: should live together with createJobRelatedIndices (in case it moves)? + void deleteJobRelatedIndices(String jobId, ActionListener listener); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderFactory.java new file mode 100644 index 00000000000..a49d63269e0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobProviderFactory.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +/** + * Get a {@linkplain JobProvider} + * This may create a new JobProvider or return an existing + * one if it is thread safe and shareable. + */ +public interface JobProviderFactory +{ + /** + * Get a {@linkplain JobProvider} + */ + JobProvider jobProvider(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobRenormaliser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobRenormaliser.java new file mode 100644 index 00000000000..a59435812de --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobRenormaliser.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.Influencer; + +import java.util.List; + + +/** + * Interface for classes that update {@linkplain Bucket Buckets} + * for a particular job with new normalised anomaly scores and + * unusual scores + */ +public interface JobRenormaliser +{ + /** + * Update the bucket with the changes that may result + * due to renormalisation. + * + * @param bucket the bucket to update + */ + void updateBucket(Bucket bucket); + + + /** + * Update the anomaly records for a particular bucket and job. + * The anomaly records are updated with the values in the + * records list. + * + * @param bucketId Id of the bucket to update + * @param records The new record values + */ + void updateRecords(String bucketId, List records); + + /** + * Update the influencer for a particular job + */ + void updateInfluencer(Influencer influencer); +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java new file mode 100644 index 00000000000..9ffe746d421 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsPersister.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; + +/** + * Interface for classes that persist {@linkplain Bucket Buckets} and + * {@linkplain Quantiles Quantiles} + */ +public interface JobResultsPersister +{ + /** + * Persist the result bucket + */ + void persistBucket(Bucket bucket); + + /** + * Persist the category definition + * @param category The category to be persisted + */ + void persistCategoryDefinition(CategoryDefinition category); + + /** + * Persist the quantiles + */ + void persistQuantiles(Quantiles quantiles); + + /** + * Persist a model snapshot description + */ + void persistModelSnapshot(ModelSnapshot modelSnapshot); + + /** + * Persist the memory usage data + */ + void persistModelSizeStats(ModelSizeStats modelSizeStats); + + /** + * Persist model debug output + */ + void persistModelDebugOutput(ModelDebugOutput modelDebugOutput); + + /** + * Persist the influencer + */ + void persistInfluencer(Influencer influencer); + + /** + * Persist state sent from the native process + */ + void persistBulkState(BytesReference bytesRef); + + /** + * Delete any existing interim results + */ + void deleteInterimResults(); + + /** + * Once all the job data has been written this function will be + * called to commit the data if the implementing persister requires + * it. + * + * @return True if successful + */ + boolean commitWrites(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsProvider.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsProvider.java new file mode 100644 index 00000000000..6cff79dabb8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/JobResultsProvider.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; + +import java.util.Optional; + +public interface JobResultsProvider +{ + /** + * Search for buckets with the parameters in the {@link BucketsQueryBuilder} + * @return QueryPage of Buckets + * @throws ResourceNotFoundException If the job id is no recognised + */ + QueryPage buckets(String jobId, BucketsQueryBuilder.BucketsQuery query) + throws ResourceNotFoundException; + + /** + * Get the bucket at time timestampMillis from the job. + * + * @param jobId the job id + * @param query The bucket query + * @return QueryPage Bucket + * @throws ResourceNotFoundException If the job id is not recognised + */ + QueryPage bucket(String jobId, BucketQueryBuilder.BucketQuery query) + throws ResourceNotFoundException; + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a large number of buckets of the given job + * + * @param jobId the id of the job for which buckets are requested + * @return a bucket {@link BatchedDocumentsIterator} + */ + BatchedDocumentsIterator newBatchedBucketsIterator(String jobId); + + /** + * Expand a bucket to include the associated records. + * + * @param jobId the job id + * @param includeInterim Include interim results + * @param bucket The bucket to be expanded + * @return The number of records added to the bucket + */ + int expandBucket(String jobId, boolean includeInterim, Bucket bucket); + + /** + * Get a page of {@linkplain CategoryDefinition}s for the given jobId. + * + * @param jobId the job id + * @param from Skip the first N categories. This parameter is for paging + * @param size Take only this number of categories + * @return QueryPage of CategoryDefinition + */ + QueryPage categoryDefinitions(String jobId, int from, int size); + + /** + * Get the specific CategoryDefinition for the given job and category id. + * + * @param jobId the job id + * @param categoryId Unique id + * @return QueryPage CategoryDefinition + */ + QueryPage categoryDefinition(String jobId, String categoryId); + + /** + * Search for anomaly records with the parameters in the + * {@link org.elasticsearch.xpack.prelert.job.persistence.RecordsQueryBuilder.RecordsQuery} + * @return QueryPage of AnomalyRecords + */ + QueryPage records(String jobId, RecordsQueryBuilder.RecordsQuery query); + + /** + * Return a page of influencers for the given job and within the given date + * range + * + * @param jobId + * The job ID for which influencers are requested + * @param query + * the query + * @return QueryPage of Influencer + */ + QueryPage influencers(String jobId, InfluencersQuery query) + throws ResourceNotFoundException; + + /** + * Get the influencer for the given job for id + * + * @param jobId the job id + * @param influencerId The unique influencer Id + * @return Optional Influencer + */ + Optional influencer(String jobId, String influencerId); + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a large number of influencers of the given job + * + * @param jobId the id of the job for which influencers are requested + * @return an influencer {@link BatchedDocumentsIterator} + */ + BatchedDocumentsIterator newBatchedInfluencersIterator(String jobId); + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a number of model snapshots of the given job + * + * @param jobId the id of the job for which model snapshots are requested + * @return a model snapshot {@link BatchedDocumentsIterator} + */ + BatchedDocumentsIterator newBatchedModelSnapshotIterator(String jobId); + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a number of ModelDebugOutputs of the given job + * + * @param jobId the id of the job for which model snapshots are requested + * @return a model snapshot {@link BatchedDocumentsIterator} + */ + BatchedDocumentsIterator newBatchedModelDebugOutputIterator(String jobId); + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a number of ModelSizeStats of the given job + * + * @param jobId the id of the job for which model snapshots are requested + * @return a model snapshot {@link BatchedDocumentsIterator} + */ + BatchedDocumentsIterator newBatchedModelSizeStatsIterator(String jobId); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/OldDataRemover.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/OldDataRemover.java new file mode 100644 index 00000000000..82d324bf1e4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/OldDataRemover.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkResponse; + +import java.util.Date; +import java.util.Deque; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A class that removes results from all the jobs that + * have expired their respected retention time. + */ +public class OldDataRemover { + + private final JobProvider jobProvider; + private final Function dataDeleterFactory; + + public OldDataRemover(JobProvider jobProvider, Function dataDeleterFactory) { + this.jobProvider = Objects.requireNonNull(jobProvider); + this.dataDeleterFactory = Objects.requireNonNull(dataDeleterFactory); + } + + /** + * Removes results between the time given and the current time + */ + public void deleteResultsAfter(ActionListener listener, String jobId, long cutoffEpochMs) { + Date now = new Date(); + JobDataDeleter deleter = dataDeleterFactory.apply(jobId); + deleteResultsWithinRange(jobId, deleter, cutoffEpochMs, now.getTime()); + deleter.commit(listener); + } + + private void deleteResultsWithinRange(String jobId, JobDataDeleter deleter, long start, long end) { + deleteBatchedData( + jobProvider.newBatchedInfluencersIterator(jobId).timeRange(start, end), + deleter::deleteInfluencer + ); + deleteBatchedData( + jobProvider.newBatchedBucketsIterator(jobId).timeRange(start, end), + deleter::deleteBucket + ); + } + + private void deleteBatchedData(BatchedDocumentsIterator resultsIterator, + Consumer deleteFunction) { + while (resultsIterator.hasNext()) { + Deque batch = resultsIterator.next(); + if (batch.isEmpty()) { + return; + } + for (T result : batch) { + deleteFunction.accept(result); + } + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/QueryPage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/QueryPage.java new file mode 100644 index 00000000000..5949e356ba6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/QueryPage.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Generic wrapper class for a page of query results and the total number of + * query hits.
+ * {@linkplain #hitCount()} is the total number of results but that value may + * not be equal to the actual length of the {@linkplain #hits()} list if from + * & take or some cursor was used in the database query. + */ +public final class QueryPage extends ToXContentToBytes implements Writeable { + + public static final ParseField HITS = new ParseField("hits"); + public static final ParseField HIT_COUNT = new ParseField("hitCount"); + + private final List hits; + private final long hitCount; + + public QueryPage(List hits, long hitCount) { + this.hits = hits; + this.hitCount = hitCount; + } + + public QueryPage(StreamInput in, Reader hitReader) throws IOException { + hits = in.readList(hitReader); + hitCount = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(hits); + out.writeLong(hitCount); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(HITS.getPreferredName(), hits); + builder.field(HIT_COUNT.getPreferredName(), hitCount); + return builder; + } + + public List hits() { + return hits; + } + + public long hitCount() { + return hitCount; + } + + @Override + public int hashCode() { + return Objects.hash(hits, hitCount); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + @SuppressWarnings("unchecked") + QueryPage other = (QueryPage) obj; + return Objects.equals(hits, other.hits) && + Objects.equals(hitCount, other.hitCount); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/RecordsQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/RecordsQueryBuilder.java new file mode 100644 index 00000000000..e9162c7ccff --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/RecordsQueryBuilder.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +/** + * One time query builder for records. Sets default values for the following + * parameters: + *
    + *
  • From- Skip the first N records. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of records. Default = + * {@value DEFAULT_SIZE}
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • SortField- The field to sort results by if null no sort is + * applied. Default = null
    • + *
  • SortDescending- Sort in descending order. Default = true
    • + *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • normalizedProbabilityThreshold. Return only buckets with a + * maxNormalizedProbability >= this value. Default = 0.0
    • + *
  • epochStart- The start bucket time. A bucket with this timestamp will be + * included in the results. If 0 all buckets up to endEpochMs are + * returned. Default = -1
    • + *
  • epochEnd- The end bucket timestamp buckets up to but NOT including this + * timestamp are returned. If 0 all buckets from startEpochMs are + * returned. Default = -1
    • + *
+ */ +public final class RecordsQueryBuilder +{ + public static final int DEFAULT_SIZE = 100; + + private RecordsQuery recordsQuery = new RecordsQuery(); + + public RecordsQueryBuilder from(int from) + { + recordsQuery.from = from; + return this; + } + + public RecordsQueryBuilder size(int size) + { + recordsQuery.size = size; + return this; + } + + public RecordsQueryBuilder epochStart(String startTime) + { + recordsQuery.epochStart = startTime; + return this; + } + + public RecordsQueryBuilder epochEnd(String endTime) + { + recordsQuery.epochEnd = endTime; + return this; + } + + public RecordsQueryBuilder includeInterim(boolean include) + { + recordsQuery.includeInterim = include; + return this; + } + + public RecordsQueryBuilder sortField(String fieldname) + { + recordsQuery.sortField = fieldname; + return this; + } + + public RecordsQueryBuilder sortDescending(boolean sortDescending) + { + recordsQuery.sortDescending = sortDescending; + return this; + } + + public RecordsQueryBuilder anomalyScoreThreshold(double anomalyScoreFilter) + { + recordsQuery.anomalyScoreFilter = anomalyScoreFilter; + return this; + } + + public RecordsQueryBuilder normalizedProbability(double normalizedProbability) + { + recordsQuery.normalizedProbability = normalizedProbability; + return this; + } + + public RecordsQueryBuilder partitionFieldValue(String partitionFieldValue) + { + recordsQuery.partitionFieldValue = partitionFieldValue; + return this; + } + + public RecordsQuery build() + { + return recordsQuery; + } + + public void clear() + { + recordsQuery = new RecordsQuery(); + } + + public class RecordsQuery + { + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean includeInterim = false; + private String sortField; + private boolean sortDescending = true; + private double anomalyScoreFilter = 0.0d; + private double normalizedProbability = 0.0d; + private String partitionFieldValue; + private String epochStart; + private String epochEnd; + + + public int getSize() + { + return size; + } + + public boolean isIncludeInterim() + { + return includeInterim; + } + + public String getSortField() + { + return sortField; + } + + public boolean isSortDescending() + { + return sortDescending; + } + + public double getAnomalyScoreThreshold() + { + return anomalyScoreFilter; + } + + public double getNormalizedProbabilityThreshold() + { + return normalizedProbability; + } + + public String getPartitionFieldValue() + { + return partitionFieldValue; + } + + public int getFrom() + { + return from; + } + + public String getEpochStart() + { + return epochStart; + } + + public String getEpochEnd() + { + return epochEnd; + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ResultsFilterBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ResultsFilterBuilder.java new file mode 100644 index 00000000000..34a376ff411 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/ResultsFilterBuilder.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import java.util.ArrayList; +import java.util.List; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; + +/** + * This builder facilitates the creation of a {@link QueryBuilder} with common + * characteristics to both buckets and records. + */ +class ResultsFilterBuilder { + private final List filters; + + ResultsFilterBuilder() { + filters = new ArrayList<>(); + } + + ResultsFilterBuilder(QueryBuilder filterBuilder) { + this(); + filters.add(filterBuilder); + } + + ResultsFilterBuilder timeRange(String field, Object start, Object end) { + if (start != null || end != null) { + RangeQueryBuilder timeRange = QueryBuilders.rangeQuery(field); + if (start != null) { + timeRange.gte(start); + } + if (end != null) { + timeRange.lt(end); + } + addFilter(timeRange); + } + return this; + } + + ResultsFilterBuilder score(String fieldName, double threshold) { + if (threshold > 0.0) { + RangeQueryBuilder scoreFilter = QueryBuilders.rangeQuery(fieldName); + scoreFilter.gte(threshold); + addFilter(scoreFilter); + } + return this; + } + + public ResultsFilterBuilder interim(String fieldName, boolean includeInterim) { + if (includeInterim) { + // Including interim results does not stop final results being + // shown, so including interim results means no filtering on the + // isInterim field + return this; + } + + // Implemented as "NOT isInterim == true" so that not present and null + // are equivalent to false. This improves backwards compatibility. + // Also, note how for a boolean field, unlike numeric term filters, the + // term value is supplied as a string. + TermQueryBuilder interimFilter = QueryBuilders.termQuery(fieldName, + Boolean.TRUE.toString()); + QueryBuilder notInterimFilter = QueryBuilders.boolQuery().mustNot(interimFilter); + addFilter(notInterimFilter); + return this; + } + + ResultsFilterBuilder term(String fieldName, String fieldValue) { + if (Strings.isNullOrEmpty(fieldName) || Strings.isNullOrEmpty(fieldValue)) { + return this; + } + + TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(fieldName, fieldValue); + addFilter(termQueryBuilder); + return this; + } + + private void addFilter(QueryBuilder fb) { + filters.add(fb); + } + + public QueryBuilder build() { + if (filters.isEmpty()) { + return QueryBuilders.matchAllQuery(); + } + if (filters.size() == 1) { + return filters.get(0); + } + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (QueryBuilder query : filters) { + boolQueryBuilder.filter(query); + } + return boolQueryBuilder; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/UsagePersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/UsagePersister.java new file mode 100644 index 00000000000..3725771397a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/persistence/UsagePersister.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +/** + * Interface for classes that persist usage information + */ +public interface UsagePersister +{ + /** + * Persist the usage info. + */ + void persistUsage(String jobId, long bytesRead, long fieldsRead, long recordsRead); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/NativeController.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/NativeController.java new file mode 100644 index 00000000000..a1dee37ec86 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/NativeController.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.job.logging.CppLogMessageHandler; +import org.elasticsearch.xpack.prelert.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; + + +/** + * Maintains the connection to the native controller daemon that can start other processes. + */ +public class NativeController { + private static final Logger LOGGER = Loggers.getLogger(NativeController.class); + + // TODO: this can be reduced once Elasticsearch is automatically starting the controller process - + // at the moment it has to be started manually, which could take a while + private static final Duration CONTROLLER_CONNECT_TIMEOUT = Duration.ofMinutes(1); + + private static final String START_COMMAND = "start"; + + private final CppLogMessageHandler cppLogHandler; + private final OutputStream commandStream; + private Thread logTailThread; + + public NativeController(Environment env, NamedPipeHelper namedPipeHelper) throws IOException { + ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, ProcessCtrl.CONTROLLER, null, + true, true, false, false, false, false); + processPipes.connectStreams(CONTROLLER_CONNECT_TIMEOUT); + cppLogHandler = new CppLogMessageHandler(null, processPipes.getLogStream().get()); + commandStream = processPipes.getCommandStream().get(); + } + + public void tailLogsInThread() { + logTailThread = new Thread(() -> { + try { + cppLogHandler.tailStream(); + cppLogHandler.close(); + } catch (IOException e) { + LOGGER.error("Error tailing C++ controller logs", e); + } + LOGGER.info("Native controller process has stopped - no new native processes can be started"); + }); + logTailThread.start(); + } + + public void startProcess(List command) throws IOException { + // Sanity check to avoid hard-to-debug errors - tabs and newlines will confuse the controller process + for (String arg : command) { + if (arg.contains("\t")) { + throw new IllegalArgumentException("argument contains a tab character: " + arg + " in " + command); + } + if (arg.contains("\n")) { + throw new IllegalArgumentException("argument contains a newline character: " + arg + " in " + command); + } + } + + synchronized (commandStream) { + LOGGER.info("Starting process with command: " + command); + commandStream.write(START_COMMAND.getBytes(StandardCharsets.UTF_8)); + for (String arg : command) { + commandStream.write('\t'); + commandStream.write(arg.getBytes(StandardCharsets.UTF_8)); + } + commandStream.write('\n'); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessCtrl.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessCtrl.java new file mode 100644 index 00000000000..c43c0d6dac7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessCtrl.java @@ -0,0 +1,304 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.IgnoreDowntime; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + + +/** + * Utility class for running a Prelert process
+ * The process runs in a clean environment. + */ +public class ProcessCtrl { + + /** + * Autodetect API native program name - always loaded from the same directory as the controller process + */ + public static final String AUTODETECT = "prelert_autodetect"; + static final String AUTODETECT_PATH = "./" + AUTODETECT; + + /** + * The normalisation native program name - always loaded from the same directory as the controller process + */ + public static final String NORMALIZE = "prelert_normalize"; + static final String NORMALIZE_PATH = "./" + NORMALIZE; + + /** + * Process controller native program name + */ + public static final String CONTROLLER = "prelert_controller"; + + /** + * Name of the config setting containing the path to the logs directory + */ + private static final int DEFAULT_MAX_NUM_RECORDS = 500; + /** + * The maximum number of anomaly records that will be written each bucket + */ + public static final Setting MAX_ANOMALY_RECORDS_SETTING = Setting.intSetting("max.anomaly.records", DEFAULT_MAX_NUM_RECORDS, + Property.NodeScope); + + // TODO: remove once the C++ logger no longer needs it + static final String LOG_ID_ARG = "--logid="; + + /* + * General arguments + */ + static final String JOB_ID_ARG = "--jobid="; + + + /* + * Arguments used by both prelert_autodetect and prelert_normalize + */ + static final String BUCKET_SPAN_ARG = "--bucketspan="; + public static final String DELETE_STATE_FILES_ARG = "--deleteStateFiles"; + static final String IGNORE_DOWNTIME_ARG = "--ignoreDowntime"; + static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput"; + static final String MODEL_CONFIG_ARG = "--modelconfig="; + public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState="; + static final String MULTIPLE_BUCKET_SPANS_ARG = "--multipleBucketspans="; + static final String PER_PARTITION_NORMALIZATION = "--perPartitionNormalization"; + + /* + * Arguments used by prelert_autodetect + */ + static final String BATCH_SPAN_ARG = "--batchspan="; + static final String LATENCY_ARG = "--latency="; + static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow="; + static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields"; + static final String PERIOD_ARG = "--period="; + static final String PERSIST_INTERVAL_ARG = "--persistInterval="; + static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval="; + static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield="; + static final String TIME_FIELD_ARG = "--timefield="; + + private static final int SECONDS_IN_HOUR = 3600; + + /** + * Roughly how often should the C++ process persist state? A staggering + * factor that varies by job is added to this. + */ + static final long DEFAULT_BASE_PERSIST_INTERVAL = 10800; // 3 hours + + /** + * Roughly how often should the C++ process output quantiles when no + * anomalies are being detected? A staggering factor that varies by job is + * added to this. + */ + static final int BASE_MAX_QUANTILE_INTERVAL = 21600; // 6 hours + + /** + * Name of the model config file + */ + static final String PRELERT_MODEL_CONF = "prelertmodel.conf"; + + /** + * Persisted quantiles are written to disk so they can be read by + * the autodetect program. All quantiles files have this extension. + */ + private static final String QUANTILES_FILE_EXTENSION = ".json"; + + /** + * Config setting storing the flag that disables model persistence + */ + public static final Setting DONT_PERSIST_MODEL_STATE_SETTING = Setting.boolSetting("no.model.state.persist", false, + Property.NodeScope); + + static String maxAnomalyRecordsArg(Settings settings) { + return "--maxAnomalyRecords=" + MAX_ANOMALY_RECORDS_SETTING.get(settings); + } + + private ProcessCtrl() { + + } + + /** + * This random time of up to 1 hour is added to intervals at which we + * tell the C++ process to perform periodic operations. This means that + * when there are many jobs there is a certain amount of staggering of + * their periodic operations. A given job will always be given the same + * staggering interval (for a given JVM implementation). + * + * @param jobId The ID of the job to calculate the staggering interval for + * @return The staggering interval + */ + static int calculateStaggeringInterval(String jobId) { + Random rng = new Random(jobId.hashCode()); + return rng.nextInt(SECONDS_IN_HOUR); + } + + public static List buildAutodetectCommand(Environment env, Settings settings, Job job, Logger logger, boolean ignoreDowntime) { + List command = new ArrayList<>(); + command.add(AUTODETECT_PATH); + + String jobId = JOB_ID_ARG + job.getId(); + command.add(jobId); + + // the logging id is the job id + String logId = LOG_ID_ARG + job.getId(); + command.add(logId); + + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + if (analysisConfig != null) { + addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command); + addIfNotNull(analysisConfig.getBatchSpan(), BATCH_SPAN_ARG, command); + addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command); + addIfNotNull(analysisConfig.getPeriod(), PERIOD_ARG, command); + addIfNotNull(analysisConfig.getSummaryCountFieldName(), + SUMMARY_COUNT_FIELD_ARG, command); + addIfNotNull(analysisConfig.getMultipleBucketSpans(), + MULTIPLE_BUCKET_SPANS_ARG, command); + if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) { + Long window = analysisConfig.getResultFinalizationWindow(); + if (window == null) { + window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW; + } + command.add(RESULT_FINALIZATION_WINDOW_ARG + window); + } + if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) { + command.add(MULTIVARIATE_BY_FIELDS_ARG); + } + + if (analysisConfig.getUsePerPartitionNormalization()) { + command.add(PER_PARTITION_NORMALIZATION); + } + } + + // Input is always length encoded + command.add(LENGTH_ENCODED_INPUT_ARG); + + // Limit the number of output records + command.add(maxAnomalyRecordsArg(settings)); + + // always set the time field + String timeFieldArg = TIME_FIELD_ARG + getTimeFieldOrDefault(job); + command.add(timeFieldArg); + + int intervalStagger = calculateStaggeringInterval(job.getId()); + logger.debug("Periodic operations staggered by " + intervalStagger +" seconds for job '" + job.getId() + "'"); + + // Supply a URL for persisting/restoring model state unless model + // persistence has been explicitly disabled. + if (DONT_PERSIST_MODEL_STATE_SETTING.get(settings)) { + logger.info("Will not persist model state - " + DONT_PERSIST_MODEL_STATE_SETTING + " setting was set"); + } else { + // Persist model state every few hours even if the job isn't closed + long persistInterval = (job.getBackgroundPersistInterval() == null) ? + (DEFAULT_BASE_PERSIST_INTERVAL + intervalStagger) : + job.getBackgroundPersistInterval(); + command.add(PERSIST_INTERVAL_ARG + persistInterval); + } + + int maxQuantileInterval = BASE_MAX_QUANTILE_INTERVAL + intervalStagger; + command.add(MAX_QUANTILE_INTERVAL_ARG + maxQuantileInterval); + + ignoreDowntime = ignoreDowntime + || job.getIgnoreDowntime() == IgnoreDowntime.ONCE + || job.getIgnoreDowntime() == IgnoreDowntime.ALWAYS; + + if (ignoreDowntime) { + command.add(IGNORE_DOWNTIME_ARG); + } + + if (ProcessCtrl.modelConfigFilePresent(env)) { + String modelConfigFile = PrelertPlugin.resolveConfigFile(env, PRELERT_MODEL_CONF).toString(); + command.add(MODEL_CONFIG_ARG + modelConfigFile); + } + + return command; + } + + private static String getTimeFieldOrDefault(Job job) { + DataDescription dataDescription = job.getDataDescription(); + boolean useDefault = dataDescription == null + || Strings.isNullOrEmpty(dataDescription.getTimeField()); + return useDefault ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField(); + } + + private static void addIfNotNull(T object, String argKey, List command) { + if (object != null) { + String param = argKey + object; + command.add(param); + } + } + + /** + * Return true if there is a file ES_HOME/config/prelertmodel.conf + */ + public static boolean modelConfigFilePresent(Environment env) { + Path modelConfPath = PrelertPlugin.resolveConfigFile(env, PRELERT_MODEL_CONF); + + return Files.isRegularFile(modelConfPath); + } + + /** + * Build the command to start the normalizer process. + */ + public static List buildNormaliserCommand(Environment env, String jobId, String quantilesState, Integer bucketSpan, + boolean perPartitionNormalization, Logger logger) throws IOException { + + List command = new ArrayList<>(); + command.add(NORMALIZE_PATH); + addIfNotNull(bucketSpan, BUCKET_SPAN_ARG, command); + command.add(LOG_ID_ARG + jobId); + command.add(LENGTH_ENCODED_INPUT_ARG); + if (perPartitionNormalization) { + command.add(PER_PARTITION_NORMALIZATION); + } + + if (quantilesState != null) { + Path quantilesStateFilePath = writeNormaliserInitState(jobId, quantilesState, env); + + String stateFileArg = QUANTILES_STATE_PATH_ARG + quantilesStateFilePath; + command.add(stateFileArg); + command.add(DELETE_STATE_FILES_ARG); + } + + if (modelConfigFilePresent(env)) { + Path modelConfPath = PrelertPlugin.resolveConfigFile(env, PRELERT_MODEL_CONF); + command.add(MODEL_CONFIG_ARG + modelConfPath.toAbsolutePath().getFileName()); + } + + return command; + } + + /** + * Write the normaliser init state to file. + */ + public static Path writeNormaliserInitState(String jobId, String state, Environment env) + throws IOException { + // createTempFile has a race condition where it may return the same + // temporary file name to different threads if called simultaneously + // from multiple threads, hence add the thread ID to avoid this + Path stateFile = Files.createTempFile(env.tmpFile(), jobId + "_quantiles_" + Thread.currentThread().getId(), + QUANTILES_FILE_EXTENSION); + + try (BufferedWriter osw = Files.newBufferedWriter(stateFile, StandardCharsets.UTF_8);) { + osw.write(state); + } + + return stateFile; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessPipes.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessPipes.java new file mode 100644 index 00000000000..1ed9c7a03c7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/ProcessPipes.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.xpack.prelert.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +/** + * Utility class for telling a Prelert C++ process which named pipes to use, + * and then waiting for them to connect once the C++ process is running. + */ +public class ProcessPipes { + + public static final String LOG_PIPE_ARG = "--logPipe="; + public static final String COMMAND_PIPE_ARG = "--commandPipe="; + public static final String INPUT_ARG = "--input="; + public static final String INPUT_IS_PIPE_ARG = "--inputIsPipe"; + public static final String OUTPUT_ARG = "--output="; + public static final String OUTPUT_IS_PIPE_ARG = "--outputIsPipe"; + public static final String RESTORE_ARG = "--restore="; + public static final String RESTORE_IS_PIPE_ARG = "--restoreIsPipe"; + public static final String PERSIST_ARG = "--persist="; + public static final String PERSIST_IS_PIPE_ARG = "--persistIsPipe"; + + private final NamedPipeHelper namedPipeHelper; + + /** + * null indicates a pipe won't be used + */ + private final String logPipeName; + private final String commandPipeName; + private final String processInPipeName; + private final String processOutPipeName; + private final String restorePipeName; + private final String persistPipeName; + + private InputStream logStream; + private OutputStream commandStream; + private OutputStream processInStream; + private InputStream processOutStream; + private OutputStream restoreStream; + private InputStream persistStream; + + /** + * Construct, stating which pipes are expected to be created. The corresponding C++ process creates the named pipes, so + * only one combination of wanted pipes will work with any given C++ process. The arguments to this constructor + * must be carefully chosen with reference to the corresponding C++ code. + * @param processName The name of the process that pipes are to be opened to. + * Must not be a full path, nor have the .exe extension on Windows. + * @param jobId The job ID of the process to which pipes are to be opened, if the process is associated with a specific job. + * May be null or empty for processes not associated with a specific job. + */ + public ProcessPipes(Environment env, NamedPipeHelper namedPipeHelper, String processName, String jobId, + boolean wantLogPipe, boolean wantCommandPipe, boolean wantProcessInPipe, boolean wantProcessOutPipe, + boolean wantRestorePipe, boolean wantPersistPipe) { + this.namedPipeHelper = namedPipeHelper; + + // The way the pipe names are formed MUST match what is done in the prelert_controller main() + // function, as it does not get any command line arguments when started as a daemon. If + // you change the code here then you MUST also change the C++ code in prelert_controller's + // main() function. + StringBuilder prefixBuilder = new StringBuilder(); + prefixBuilder.append(namedPipeHelper.getDefaultPipeDirectoryPrefix(env)).append(Objects.requireNonNull(processName)).append('_'); + if (!Strings.isNullOrEmpty(jobId)) { + prefixBuilder.append(jobId).append('_'); + } + String prefix = prefixBuilder.toString(); + String suffix = String.format(Locale.ROOT, "_%d", JvmInfo.jvmInfo().getPid()); + logPipeName = wantLogPipe ? String.format(Locale.ROOT, "%slog%s", prefix, suffix) : null; + commandPipeName = wantCommandPipe ? String.format(Locale.ROOT, "%scommand%s", prefix, suffix) : null; + processInPipeName = wantProcessInPipe ? String.format(Locale.ROOT, "%sinput%s", prefix, suffix) : null; + processOutPipeName = wantProcessOutPipe ? String.format(Locale.ROOT, "%soutput%s", prefix, suffix) : null; + restorePipeName = wantRestorePipe ? String.format(Locale.ROOT, "%srestore%s", prefix, suffix) : null; + persistPipeName = wantPersistPipe ? String.format(Locale.ROOT, "%spersist%s", prefix, suffix) : null; + } + + /** + * Augments a list of command line arguments, for example that built up by the AutodetectBuilder class. + */ + public void addArgs(List command) { + if (logPipeName != null) { + command.add(LOG_PIPE_ARG + logPipeName); + } + if (commandPipeName != null) { + command.add(COMMAND_PIPE_ARG + commandPipeName); + } + // The following are specified using two arguments, as the C++ processes could already accept input from files on disk + if (processInPipeName != null) { + command.add(INPUT_ARG + processInPipeName); + command.add(INPUT_IS_PIPE_ARG); + } + if (processOutPipeName != null) { + command.add(OUTPUT_ARG + processOutPipeName); + command.add(OUTPUT_IS_PIPE_ARG); + } + if (restorePipeName != null) { + command.add(RESTORE_ARG + restorePipeName); + command.add(RESTORE_IS_PIPE_ARG); + } + if (persistPipeName != null) { + command.add(PERSIST_ARG + persistPipeName); + command.add(PERSIST_IS_PIPE_ARG); + } + } + + /** + * Connect the pipes created by the C++ process. This must be called after the corresponding C++ process has been started. + * @param timeout Needs to be long enough for the C++ process perform all startup tasks that precede creation of named pipes. + * There should not be very many of these, so a short timeout should be fine. However, at least a couple of + * seconds is recommended due to the vagaries of process scheduling and the way VMs can completely stall for + * some hypervisor actions. + */ + public void connectStreams(Duration timeout) throws IOException { + // The order here is important. It must match the order that the C++ process tries to connect to the pipes, otherwise + // a timeout is guaranteed. Also change api::CIoManager in the C++ code if changing the order here. + if (logPipeName != null) { + logStream = namedPipeHelper.openNamedPipeInputStream(logPipeName, timeout); + } + if (commandPipeName != null) { + commandStream = namedPipeHelper.openNamedPipeOutputStream(commandPipeName, timeout); + } + if (processInPipeName != null) { + processInStream = namedPipeHelper.openNamedPipeOutputStream(processInPipeName, timeout); + } + if (processOutPipeName != null) { + processOutStream = namedPipeHelper.openNamedPipeInputStream(processOutPipeName, timeout); + } + if (restorePipeName != null) { + restoreStream = namedPipeHelper.openNamedPipeOutputStream(restorePipeName, timeout); + } + if (persistPipeName != null) { + persistStream = namedPipeHelper.openNamedPipeInputStream(persistPipeName, timeout); + } + } + + public Optional getLogStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (logPipeName == null) { + return Optional.empty(); + } + if (logStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(logStream); + } + + public Optional getCommandStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (commandPipeName == null) { + return Optional.empty(); + } + if (commandStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(commandStream); + } + + public Optional getProcessInStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (processInPipeName == null) { + return Optional.empty(); + } + if (processInStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(processInStream); + } + + public Optional getProcessOutStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (processOutPipeName == null) { + return Optional.empty(); + } + if (processOutStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(processOutStream); + } + + public Optional getRestoreStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (restorePipeName == null) { + return Optional.empty(); + } + if (restoreStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(restoreStream); + } + + public Optional getPersistStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (persistPipeName == null) { + return Optional.empty(); + } + if (persistStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(persistStream); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectBuilder.java new file mode 100644 index 00000000000..131d73ea441 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectBuilder.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.job.AnalysisLimits; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelDebugConfig; +import org.elasticsearch.xpack.prelert.job.process.NativeController; +import org.elasticsearch.xpack.prelert.job.process.ProcessCtrl; +import org.elasticsearch.xpack.prelert.job.process.ProcessPipes; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.AnalysisLimitsWriter; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.FieldConfigWriter; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.ModelDebugConfigWriter; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.lists.ListDocument; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * The autodetect process builder. + */ +public class AutodetectBuilder { + private static final String CONF_EXTENSION = ".conf"; + private static final String LIMIT_CONFIG_ARG = "--limitconfig="; + private static final String MODEL_DEBUG_CONFIG_ARG = "--modeldebugconfig="; + private static final String FIELD_CONFIG_ARG = "--fieldconfig="; + + private Job job; + private List filesToDelete; + private Logger logger; + private boolean ignoreDowntime; + private Set referencedLists; + private Optional quantiles; + private Environment env; + private Settings settings; + private NativeController controller; + private ProcessPipes processPipes; + + /** + * Constructs an autodetect process builder + * + * @param job The job configuration + * @param filesToDelete This method will append File objects that need to be + * deleted when the process completes + * @param logger The job's logger + */ + public AutodetectBuilder(Job job, List filesToDelete, Logger logger, Environment env, Settings settings, + NativeController controller, ProcessPipes processPipes) { + this.env = env; + this.settings = settings; + this.controller = controller; + this.processPipes = processPipes; + this.job = Objects.requireNonNull(job); + this.filesToDelete = Objects.requireNonNull(filesToDelete); + this.logger = Objects.requireNonNull(logger); + ignoreDowntime = false; + referencedLists = new HashSet<>(); + quantiles = Optional.empty(); + } + + /** + * Set ignoreDowntime + * + * @param ignoreDowntime If true set the ignore downtime flag overriding the + * setting in the job configuration + */ + public AutodetectBuilder ignoreDowntime(boolean ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + return this; + } + + public AutodetectBuilder referencedLists(Set lists) { + referencedLists = lists; + return this; + } + + /** + * Set quantiles to restore the normaliser state if any. + * + * @param quantiles the non-null quantiles + */ + public AutodetectBuilder quantiles(Optional quantiles) { + this.quantiles = quantiles; + return this; + } + + /** + * Requests that the controller daemon start an autodetect process. + */ + public void build() throws IOException { + + List command = ProcessCtrl.buildAutodetectCommand(env, settings, job, logger, ignoreDowntime); + + buildLimits(command); + buildModelDebugConfig(command); + + buildQuantiles(command); + buildFieldConfig(command); + processPipes.addArgs(command); + controller.startProcess(command); + } + + private void buildLimits(List command) throws IOException { + if (job.getAnalysisLimits() != null) { + Path limitConfigFile = Files.createTempFile(env.tmpFile(), "limitconfig", CONF_EXTENSION); + filesToDelete.add(limitConfigFile); + writeLimits(job.getAnalysisLimits(), limitConfigFile); + String limits = LIMIT_CONFIG_ARG + limitConfigFile.toString(); + command.add(limits); + } + } + + /** + * Write the Prelert autodetect model options to emptyConfFile. + */ + private static void writeLimits(AnalysisLimits options, Path emptyConfFile) throws IOException { + + try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(emptyConfFile), StandardCharsets.UTF_8)) { + new AnalysisLimitsWriter(options, osw).write(); + } + } + + private void buildModelDebugConfig(List command) throws IOException { + if (job.getModelDebugConfig() != null) { + Path modelDebugConfigFile = Files.createTempFile(env.tmpFile(), "modeldebugconfig", CONF_EXTENSION); + filesToDelete.add(modelDebugConfigFile); + writeModelDebugConfig(job.getModelDebugConfig(), modelDebugConfigFile); + String modelDebugConfig = MODEL_DEBUG_CONFIG_ARG + modelDebugConfigFile.toString(); + command.add(modelDebugConfig); + } + } + + private static void writeModelDebugConfig(ModelDebugConfig config, Path emptyConfFile) + throws IOException { + try (OutputStreamWriter osw = new OutputStreamWriter( + Files.newOutputStream(emptyConfFile), + StandardCharsets.UTF_8)) { + new ModelDebugConfigWriter(config, osw).write(); + } + } + + private void buildQuantiles(List command) throws IOException { + if (quantiles.isPresent() && !quantiles.get().getQuantileState().isEmpty()) { + Quantiles quantiles = this.quantiles.get(); + logger.info("Restoring quantiles for job '" + job.getId() + "'"); + + Path normalisersStateFilePath = ProcessCtrl.writeNormaliserInitState( + job.getId(), quantiles.getQuantileState(), env); + + String quantilesStateFileArg = ProcessCtrl.QUANTILES_STATE_PATH_ARG + normalisersStateFilePath; + command.add(quantilesStateFileArg); + command.add(ProcessCtrl.DELETE_STATE_FILES_ARG); + } + } + + private void buildFieldConfig(List command) throws IOException, FileNotFoundException { + if (job.getAnalysisConfig() != null) { + // write to a temporary field config file + Path fieldConfigFile = Files.createTempFile(env.tmpFile(), "fieldconfig", CONF_EXTENSION); + filesToDelete.add(fieldConfigFile); + try (OutputStreamWriter osw = new OutputStreamWriter( + Files.newOutputStream(fieldConfigFile), + StandardCharsets.UTF_8)) { + new FieldConfigWriter(job.getAnalysisConfig(), referencedLists, osw, logger).write(); + } + + String fieldConfig = FIELD_CONFIG_ARG + fieldConfigFile.toString(); + command.add(fieldConfig); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectCommunicator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectCommunicator.java new file mode 100644 index 00000000000..f1aac15ab1e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectCommunicator.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutoDetectResultProcessor; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.StateReader; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.DataToProcessWriter; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.DataToProcessWriterFactory; +import org.elasticsearch.xpack.prelert.job.status.CountingInputStream; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Optional; + +public class AutodetectCommunicator implements Closeable { + + private static final int DEFAULT_TRY_COUNT = 5; + private static final int DEFAULT_TRY_TIMEOUT_SECS = 6; + + private final Logger jobLogger; + private final StatusReporter statusReporter; + private final AutodetectProcess autodetectProcess; + private final DataToProcessWriter autoDetectWriter; + private final AutoDetectResultProcessor autoDetectResultProcessor; + + private final StateReader stateReader; + private final Thread stateParserThread; + + + public AutodetectCommunicator(ThreadPool threadPool, Job job, AutodetectProcess process, Logger jobLogger, + JobResultsPersister persister, StatusReporter statusReporter, + AutoDetectResultProcessor autoDetectResultProcessor) { + this.autodetectProcess = process; + this.jobLogger = jobLogger; + this.statusReporter = statusReporter; + this.autoDetectResultProcessor = autoDetectResultProcessor; + this.stateReader = new StateReader(persister, process.getPersistStream(), this.jobLogger); + + // TODO norelease: prevent that we fail to start any of the required threads for interacting with analytical process: + // We should before we start the analytical process (and scheduler) verify that have enough threads. + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + boolean usePerPartitionNormalization = analysisConfig.getUsePerPartitionNormalization(); + threadPool.executor(PrelertPlugin.THREAD_POOL_NAME).execute(() -> { + this.autoDetectResultProcessor.process(jobLogger, process.getProcessOutStream(), usePerPartitionNormalization); + }); + // NORELEASE - use ES ThreadPool + stateParserThread = new Thread(stateReader, job.getId() + "-State-Parser"); + stateParserThread.start(); + + this.autoDetectWriter = createProcessWriter(job, process, statusReporter); + } + + private DataToProcessWriter createProcessWriter(Job job, AutodetectProcess process, StatusReporter statusReporter) { + return DataToProcessWriterFactory.create(true, process, job.getDataDescription(), job.getAnalysisConfig(), + job.getSchedulerConfig(), new TransformConfigs(job.getTransforms()) , statusReporter, jobLogger); + } + + public DataCounts writeToJob(InputStream inputStream) throws IOException { + checkProcessIsAlive(); + CountingInputStream countingStream = new CountingInputStream(inputStream, statusReporter); + DataCounts results = autoDetectWriter.write(countingStream); + autoDetectWriter.flush(); + return results; + } + + @Override + public void close() throws IOException { + checkProcessIsAlive(); + autodetectProcess.close(); + autoDetectResultProcessor.awaitCompletion(); + } + + public void writeResetBucketsControlMessage(DataLoadParams params) throws IOException { + checkProcessIsAlive(); + autodetectProcess.writeResetBucketsControlMessage(params); + } + + public void writeUpdateConfigMessage(String config) throws IOException { + checkProcessIsAlive(); + autodetectProcess.writeUpdateConfigMessage(config); + } + + public void flushJob(InterimResultsParams params) throws IOException { + flushJob(params, DEFAULT_TRY_COUNT, DEFAULT_TRY_TIMEOUT_SECS); + } + + void flushJob(InterimResultsParams params, int tryCount, int tryTimeoutSecs) throws IOException { + String flushId = autodetectProcess.flushJob(params); + + // TODO: norelease: I think waiting once 30 seconds will have the same effect as 5 * 6 seconds. + // So we may want to remove this retry logic here + Duration intermittentTimeout = Duration.ofSeconds(tryTimeoutSecs); + boolean isFlushComplete = false; + while (isFlushComplete == false && --tryCount >= 0) { + // Check there wasn't an error in the flush + if (!autodetectProcess.isProcessAlive()) { + + String msg = Messages.getMessage(Messages.AUTODETECT_FLUSH_UNEXPTECTED_DEATH) + " " + autodetectProcess.readError(); + jobLogger.error(msg); + throw ExceptionsHelper.serverError(msg); + } + isFlushComplete = autoDetectResultProcessor.waitForFlushAcknowledgement(flushId, intermittentTimeout); + jobLogger.info("isFlushComplete={}", isFlushComplete); + } + + if (!isFlushComplete) { + String msg = Messages.getMessage(Messages.AUTODETECT_FLUSH_TIMEOUT) + " " + autodetectProcess.readError(); + jobLogger.error(msg); + throw ExceptionsHelper.serverError(msg); + } + + // We also have to wait for the normaliser to become idle so that we block + // clients from querying results in the middle of normalisation. + autoDetectResultProcessor.waitUntilRenormaliserIsIdle(); + } + + /** + * Throws an exception if the process has exited + */ + private void checkProcessIsAlive() { + if (!autodetectProcess.isProcessAlive()) { + String errorMsg = "Unexpected death of autodetect: " + autodetectProcess.readError(); + jobLogger.error(errorMsg); + throw ExceptionsHelper.serverError(errorMsg); + } + } + + public ZonedDateTime getProcessStartTime() { + return autodetectProcess.getProcessStartTime(); + } + + public Optional getModelSizeStats() { + return autoDetectResultProcessor.modelSizeStats(); + } + + public Optional getDataCounts() { + return Optional.ofNullable(statusReporter.runningTotalStats()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcess.java new file mode 100644 index 00000000000..a4bd7a5b334 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcess.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; + +import java.io.IOException; +import java.io.InputStream; +import java.time.ZonedDateTime; + +/** + * Interface representing the native C++ autodetect process + */ +public interface AutodetectProcess { + + /** + * Write the record to autodetect. The record parameter should not be encoded + * (i.e. length encoded) the implementation will appy the corrrect encoding. + * + * @param record Plain array of strings, implementors of this class should + * encode the record appropriately + * @throws IOException If the write failed + */ + void writeRecord(String [] record) throws IOException; + + /** + * Write the reset buckets control message + * @param params Reset bucket options + * @throws IOException If write reset mesage fails + */ + void writeResetBucketsControlMessage(DataLoadParams params) throws IOException; + + /** + * Write an update configuration message + * @param config Config message + * @throws IOException If the write config message fails + */ + void writeUpdateConfigMessage(String config) throws IOException; + + /** + * Flush the job pushing any stale data into autodetect. + * Every flush command generates a unique flush Id which will be output + * in a flush acknowledgment by the autodetect process once the flush has + * been processed. + * + * @param params Should interim results be generated + * @return The flush Id + * @throws IOException If the flush failed + */ + String flushJob(InterimResultsParams params) throws IOException; + + /** + * Flush the output data stream + */ + void flushStream() throws IOException; + + /** + * Close + */ + void close() throws IOException; + + /** + * Autodetect's output stream + * @return output stream + */ + InputStream getProcessOutStream(); + + /** + * Autodetect's state persistence stream + * @return persist stream + */ + InputStream getPersistStream(); + + /** + * The time the process was started + * @return Process start time + */ + ZonedDateTime getProcessStartTime(); + + /** + * Returns true if the process still running. + * Methods such as {@link #flushJob(InterimResultsParams)} are essentially + * asynchronous the command will be continue to execute in the process after + * the call has returned. This method tests whether something catastrophic + * occurred in the process during its execution. + * @return True if the process is still running + */ + boolean isProcessAlive(); + + /** + * Read any content in the error output buffer. + * @return An error message or empty String if no error. + */ + String readError(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcessFactory.java new file mode 100644 index 00000000000..4a7336d0a1b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/AutodetectProcessFactory.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.elasticsearch.xpack.prelert.job.Job; + +/** + * Factory interface for creating implementations of {@link AutodetectProcess} + */ +public interface AutodetectProcessFactory { + /** + * Create an implementation of {@link AutodetectProcess} + * + * @param job Job configuration for the analysis process + * @param ignoreDowntime Should gaps in data be treated as anomalous or as a maintenance window after a job re-start + * @return The process + */ + AutodetectProcess createAutodetectProcess(Job job, boolean ignoreDowntime); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/BlackHoleAutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/BlackHoleAutodetectProcess.java new file mode 100644 index 00000000000..a558b183019 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/BlackHoleAutodetectProcess.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.results.AutodetectResult; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.ZonedDateTime; + +/** + * A placeholder class simulating the actions of the native Autodetect process. + * Most methods consume data without performing any action however, after a call to + * {@link #flushJob(InterimResultsParams)} a {@link org.elasticsearch.xpack.prelert.job.process.autodetect.output.FlushAcknowledgement} + * message is expected on the {@link #getProcessOutStream()} stream. This class writes the flush + * acknowledgement immediately. + */ +public class BlackHoleAutodetectProcess implements AutodetectProcess, Closeable { + + private static final Logger LOGGER = Loggers.getLogger(BlackHoleAutodetectProcess.class); + private static final String FLUSH_ID = "flush-1"; + + private final PipedInputStream processOutStream; + private final PipedInputStream persistStream; + private PipedOutputStream pipedProcessOutStream; + private PipedOutputStream pipedPersistStream; + private final ZonedDateTime startTime; + + public BlackHoleAutodetectProcess() { + processOutStream = new PipedInputStream(); + persistStream = new PipedInputStream(); + try { + pipedProcessOutStream = new PipedOutputStream(processOutStream); + pipedPersistStream = new PipedOutputStream(persistStream); + } catch (IOException e) { + LOGGER.error("Error connecting PipedOutputStream", e); + } + startTime = ZonedDateTime.now(); + } + + @Override + public void writeRecord(String[] record) throws IOException { + } + + @Override + public void writeResetBucketsControlMessage(DataLoadParams params) throws IOException { + } + + @Override + public void writeUpdateConfigMessage(String config) throws IOException { + } + + /** + * Accept the request do nothing with it but write the flush acknowledgement to {@link #getProcessOutStream()} + * @param params Should interim results be generated + * @return {@link #FLUSH_ID} + */ + @Override + public String flushJob(InterimResultsParams params) throws IOException { + FlushAcknowledgement flushAcknowledgement = new FlushAcknowledgement(FLUSH_ID); + AutodetectResult result = new AutodetectResult(null, null, null, null, null, null, flushAcknowledgement); + XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent()); + builder.startArray(); + builder.value(result); + builder.endArray(); + pipedProcessOutStream.write(builder.string().getBytes(StandardCharsets.UTF_8)); + pipedProcessOutStream.flush(); + return FLUSH_ID; + } + + @Override + public void flushStream() throws IOException { + } + + @Override + public void close() throws IOException { + pipedProcessOutStream.close(); + pipedPersistStream.close(); + } + + @Override + public InputStream getProcessOutStream() { + return processOutStream; + } + + @Override + public InputStream getPersistStream() { + return persistStream; + } + + @Override + public ZonedDateTime getProcessStartTime() { + return startTime; + } + + @Override + public boolean isProcessAlive() { + return true; + } + + @Override + public String readError() { + return ""; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcess.java new file mode 100644 index 00000000000..db87fa35ce1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcess.java @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.prelert.job.logging.CppLogMessageHandler; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.ControlMsgToProcessWriter; +import org.elasticsearch.xpack.prelert.job.process.autodetect.writer.LengthEncodedWriter; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.util.List; + +/** + * Autodetect process using native code. + */ +public class NativeAutodetectProcess implements AutodetectProcess { + private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcess.class); + + private final CppLogMessageHandler cppLogHandler; + private final OutputStream processInStream; + private final InputStream processOutStream; + private final InputStream persistStream; + private final LengthEncodedWriter recordWriter; + private final ZonedDateTime startTime; + private final int numberOfAnalysisFields; + private final List filesToDelete; + private Thread logTailThread; + + public NativeAutodetectProcess(String jobId, InputStream logStream, OutputStream processInStream, + InputStream processOutStream, InputStream persistStream, + int numberOfAnalysisFields, List filesToDelete) { + cppLogHandler = new CppLogMessageHandler(jobId, logStream); + this.processInStream = new BufferedOutputStream(processInStream); + this.processOutStream = processOutStream; + this.persistStream = persistStream; + this.recordWriter = new LengthEncodedWriter(this.processInStream); + startTime = ZonedDateTime.now(); + this.numberOfAnalysisFields = numberOfAnalysisFields; + this.filesToDelete = filesToDelete; + } + + void tailLogsInThread() { + logTailThread = new Thread(() -> { + try { + cppLogHandler.tailStream(); + cppLogHandler.close(); + } catch (IOException e) { + LOGGER.error("Error tailing C++ process logs", e); + } + }); + logTailThread.start(); + } + + @Override + public void writeRecord(String[] record) throws IOException { + recordWriter.writeRecord(record); + } + + @Override + public void writeResetBucketsControlMessage(DataLoadParams params) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeResetBucketsMessage(params); + } + + @Override + public void writeUpdateConfigMessage(String config) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeUpdateConfigMessage(config); + } + + @Override + public String flushJob(InterimResultsParams params) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeCalcInterimMessage(params); + return writer.writeFlushMessage(); + } + + @Override + public void flushStream() throws IOException { + recordWriter.flush(); + } + + @Override + public void close() throws IOException { + try { + // closing its input causes the process to exit + processInStream.close(); + + // wait for the process to exit by waiting for end-of-file on the named pipe connected to its logger + if (logTailThread != null) { + logTailThread.join(); + } + + if (cppLogHandler.seenFatalError()) { + throw ExceptionsHelper.serverError(cppLogHandler.getErrors()); + } + LOGGER.info("Process exited"); + } catch (InterruptedException e) { + LOGGER.warn("Exception closing the running native process"); + Thread.currentThread().interrupt(); + } finally { + deleteAssociatedFiles(); + } + } + + void deleteAssociatedFiles() throws IOException { + if (filesToDelete == null) { + return; + } + + for (Path fileToDelete : filesToDelete) { + if (Files.deleteIfExists(fileToDelete)) { + LOGGER.debug("Deleted file {}", fileToDelete::toString); + } else { + LOGGER.warn("Failed to delete file {}", fileToDelete::toString); + } + } + } + + @Override + public InputStream getProcessOutStream() { + return processOutStream; + } + + @Override + public InputStream getPersistStream() { + return persistStream; + } + + @Override + public ZonedDateTime getProcessStartTime() { + return startTime; + } + + @Override + public boolean isProcessAlive() { + // Sanity check: make sure the process hasn't terminated already + return !cppLogHandler.hasLogStreamEnded(); + } + + @Override + public String readError() { + return cppLogHandler.getErrors(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcessFactory.java new file mode 100644 index 00000000000..98b094488d5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/NativeAutodetectProcessFactory.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.process.NativeController; +import org.elasticsearch.xpack.prelert.job.process.ProcessCtrl; +import org.elasticsearch.xpack.prelert.job.process.ProcessPipes; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.elasticsearch.xpack.prelert.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class NativeAutodetectProcessFactory implements AutodetectProcessFactory { + + private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcessFactory.class); + private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper(); + private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(2); + private final JobProvider jobProvider; + private Environment env; + private Settings settings; + private NativeController nativeController; + + public NativeAutodetectProcessFactory(JobProvider jobProvider, Environment env, Settings settings, NativeController nativeController) { + this.env = Objects.requireNonNull(env); + this.settings = Objects.requireNonNull(settings); + this.jobProvider = Objects.requireNonNull(jobProvider); + this.nativeController = Objects.requireNonNull(nativeController); + } + + @Override + public AutodetectProcess createAutodetectProcess(Job job, boolean ignoreDowntime) { + List filesToDelete = new ArrayList<>(); + List modelSnapshots = jobProvider.modelSnapshots(job.getId(), 0, 1).hits(); + ModelSnapshot modelSnapshot = (modelSnapshots != null && !modelSnapshots.isEmpty()) ? modelSnapshots.get(0) : null; + + ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, ProcessCtrl.AUTODETECT, job.getId(), + true, false, true, true, modelSnapshot != null, !ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING.get(settings)); + createNativeProcess(job, processPipes, ignoreDowntime, filesToDelete); + int numberOfAnalysisFields = job.getAnalysisConfig().analysisFields().size(); + NativeAutodetectProcess autodetect = new NativeAutodetectProcess(job.getId(), processPipes.getLogStream().get(), + processPipes.getProcessInStream().get(), processPipes.getProcessOutStream().get(), + processPipes.getPersistStream().get(), numberOfAnalysisFields, filesToDelete); + autodetect.tailLogsInThread(); + if (modelSnapshot != null) { + restoreStateInThread(job.getId(), modelSnapshot, processPipes.getRestoreStream().get()); + } + return autodetect; + } + + private void restoreStateInThread(String jobId, ModelSnapshot modelSnapshot, OutputStream restoreStream) { + new Thread(() -> { + try { + jobProvider.restoreStateToStream(jobId, modelSnapshot, restoreStream); + } catch (Exception e) { + LOGGER.error("Error restoring model state for job " + jobId, e); + } + // The restore stream will not be needed again. If an error occurred getting state to restore then + // it's critical to close the restore stream so that the C++ code can realise that it will never + // receive any state to restore. If restoration went smoothly then this is just good practice. + try { + restoreStream.close(); + } catch (IOException e) { + LOGGER.error("Error closing restore stream for job " + jobId, e); + } + }).start(); + } + + private void createNativeProcess(Job job, ProcessPipes processPipes, boolean ignoreDowntime, List filesToDelete) { + + String jobId = job.getId(); + Optional quantiles = jobProvider.getQuantiles(jobId); + + try { + AutodetectBuilder autodetectBuilder = new AutodetectBuilder(job, filesToDelete, LOGGER, env, + settings, nativeController, processPipes) + .ignoreDowntime(ignoreDowntime) + .referencedLists(resolveLists(job.getAnalysisConfig().extractReferencedLists())); + + // if state is null or empty it will be ignored + // else it is used to restore the quantiles + if (quantiles != null) { + autodetectBuilder.quantiles(quantiles); + } + + autodetectBuilder.build(); + processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); + } catch (IOException e) { + String msg = "Failed to launch process for job " + job.getId(); + LOGGER.error(msg); + throw ExceptionsHelper.serverError(msg, e); + } + } + + private Set resolveLists(Set listIds) { + Set resolved = new HashSet<>(); + for (String listId : listIds) { + Optional list = jobProvider.getList(listId); + if (list.isPresent()) { + resolved.add(list.get()); + } else { + LOGGER.warn("List '" + listId + "' could not be retrieved."); + } + } + return resolved; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/FlushAcknowledgement.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/FlushAcknowledgement.java new file mode 100644 index 00000000000..51a1cce47c0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/FlushAcknowledgement.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.output; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Simple class to parse and store a flush ID. + */ +public class FlushAcknowledgement extends ToXContentToBytes implements Writeable { + /** + * Field Names + */ + public static final ParseField TYPE = new ParseField("flush"); + public static final ParseField ID = new ParseField("id"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new FlushAcknowledgement((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); + } + + private String id; + + public FlushAcknowledgement(String id) { + this.id = id; + } + + public FlushAcknowledgement(StreamInput in) throws IOException { + id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + public String getId() { + return id; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + FlushAcknowledgement other = (FlushAcknowledgement) obj; + return Objects.equals(id, other.id); + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutoDetectResultProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutoDetectResultProcessor.java new file mode 100644 index 00000000000..94a993d8c24 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutoDetectResultProcessor.java @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.prelert.job.process.normalizer.Renormaliser; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AutodetectResult; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.prelert.utils.CloseableIterator; + +import java.io.InputStream; +import java.time.Duration; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; + +/** + * A runnable class that reads the autodetect process output + * and writes the results via the {@linkplain JobResultsPersister} + * passed in the constructor. + *

+ * Has methods to register and remove alert observers. + * Also has a method to wait for a flush to be complete. + */ +public class AutoDetectResultProcessor { + + private final Renormaliser renormaliser; + private final JobResultsPersister persister; + private final AutodetectResultsParser parser; + + private final CountDownLatch latch = new CountDownLatch(1); + private final FlushListener flushListener; + + private volatile ModelSizeStats latestModelSizeStats; + + public AutoDetectResultProcessor(Renormaliser renormaliser, JobResultsPersister persister, AutodetectResultsParser parser) { + this.renormaliser = renormaliser; + this.persister = persister; + this.parser = parser; + this.flushListener = new FlushListener(); + } + + AutoDetectResultProcessor(Renormaliser renormaliser, JobResultsPersister persister, AutodetectResultsParser parser, + FlushListener flushListener) { + this.renormaliser = renormaliser; + this.persister = persister; + this.parser = parser; + this.flushListener = flushListener; + } + + public void process(Logger jobLogger, InputStream in, boolean isPerPartitionNormalisation) { + try (CloseableIterator iterator = parser.parseResults(in)) { + int bucketCount = 0; + Context context = new Context(jobLogger, isPerPartitionNormalisation); + while (iterator.hasNext()) { + AutodetectResult result = iterator.next(); + processResult(context, result); + bucketCount++; + jobLogger.trace("Bucket number {} parsed from output", bucketCount); + } + jobLogger.info(bucketCount + " buckets parsed from autodetect output - about to refresh indexes"); + jobLogger.info("Parse results Complete"); + } catch (Exception e) { + jobLogger.info("Error parsing autodetect output", e); + } finally { + latch.countDown(); + flushListener.clear(); + renormaliser.shutdown(jobLogger); + } + } + + void processResult(Context context, AutodetectResult result) { + Bucket bucket = result.getBucket(); + if (bucket != null) { + if (context.deleteInterimRequired) { + // Delete any existing interim results at the start + // of a job upload: + // these are generated by a Flush command, and will + // be replaced or + // superseded by new results + context.jobLogger.trace("Deleting interim results"); + + // NOCOMMIT: This feels like an odd side-effect to + // have in a parser, + // especially since it has to wire up to + // actionlisteners. Feels like it should + // be refactored out somewhere, after parsing? + persister.deleteInterimResults(); + context.deleteInterimRequired = false; + } + if (context.isPerPartitionNormalization) { + bucket.calcMaxNormalizedProbabilityPerPartition(); + } + persister.persistBucket(bucket); + } + CategoryDefinition categoryDefinition = result.getCategoryDefinition(); + if (categoryDefinition != null) { + persister.persistCategoryDefinition(categoryDefinition); + } + ModelDebugOutput modelDebugOutput = result.getModelDebugOutput(); + if (modelDebugOutput != null) { + persister.persistModelDebugOutput(modelDebugOutput); + } + ModelSizeStats modelSizeStats = result.getModelSizeStats(); + if (modelSizeStats != null) { + context.jobLogger.trace(String.format(Locale.ROOT, "Parsed ModelSizeStats: %d / %d / %d / %d / %d / %s", + modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(), modelSizeStats.getTotalOverFieldCount(), + modelSizeStats.getTotalPartitionFieldCount(), modelSizeStats.getBucketAllocationFailuresCount(), + modelSizeStats.getMemoryStatus())); + + latestModelSizeStats = modelSizeStats; + persister.persistModelSizeStats(modelSizeStats); + } + ModelSnapshot modelSnapshot = result.getModelSnapshot(); + if (modelSnapshot != null) { + persister.persistModelSnapshot(modelSnapshot); + } + Quantiles quantiles = result.getQuantiles(); + if (quantiles != null) { + persister.persistQuantiles(quantiles); + + context.jobLogger.debug("Quantiles parsed from output - will " + "trigger renormalisation of scores"); + if (context.isPerPartitionNormalization) { + renormaliser.renormaliseWithPartition(quantiles, context.jobLogger); + } else { + renormaliser.renormalise(quantiles, context.jobLogger); + } + } + FlushAcknowledgement flushAcknowledgement = result.getFlushAcknowledgement(); + if (flushAcknowledgement != null) { + context.jobLogger.debug("Flush acknowledgement parsed from output for ID " + flushAcknowledgement.getId()); + // Commit previous writes here, effectively continuing + // the flush from the C++ autodetect process right + // through to the data store + persister.commitWrites(); + flushListener.acknowledgeFlush(flushAcknowledgement.getId()); + // Interim results may have been produced by the flush, + // which need to be + // deleted when the next finalized results come through + context.deleteInterimRequired = true; + } + } + + public void awaitCompletion() { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + /** + * Blocks until a flush is acknowledged or the timeout expires, whichever happens first. + * + * @param flushId the id of the flush request to wait for + * @param timeout the timeout + * @return {@code true} if the flush has completed or the parsing finished; {@code false} if the timeout expired + */ + public boolean waitForFlushAcknowledgement(String flushId, Duration timeout) { + return flushListener.waitForFlush(flushId, timeout.toMillis()); + } + + public void waitUntilRenormaliserIsIdle() { + renormaliser.waitUntilIdle(); + } + + static class Context { + + private final Logger jobLogger; + private final boolean isPerPartitionNormalization; + + boolean deleteInterimRequired; + + Context(Logger jobLogger, boolean isPerPartitionNormalization) { + this.jobLogger = jobLogger; + this.isPerPartitionNormalization = isPerPartitionNormalization; + this.deleteInterimRequired = true; + } + } + + public Optional modelSizeStats() { + return Optional.ofNullable(latestModelSizeStats); + } + +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutodetectResultsParser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutodetectResultsParser.java new file mode 100644 index 00000000000..38c4307c75c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/AutodetectResultsParser.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.prelert.job.results.AutodetectResult; +import org.elasticsearch.xpack.prelert.utils.CloseableIterator; + +import java.io.IOException; +import java.io.InputStream; + + +/** + * Parses the JSON output of the autodetect program. + *

+ * Expects an array of buckets so the first element will always be the + * start array symbol and the data must be terminated with the end array symbol. + */ +public class AutodetectResultsParser extends AbstractComponent { + + private final ParseFieldMatcherSupplier parseFieldMatcherSupplier; + + public AutodetectResultsParser(Settings settings, ParseFieldMatcherSupplier parseFieldMatcherSupplier) { + super(settings); + this.parseFieldMatcherSupplier = parseFieldMatcherSupplier; + } + + CloseableIterator parseResults(InputStream in) throws ElasticsearchParseException { + try { + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(in); + XContentParser.Token token = parser.nextToken(); + // if start of an array ignore it, we expect an array of buckets + if (token != XContentParser.Token.START_ARRAY) { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + return new AutodetectResultIterator(in, parser); + } catch (IOException e) { + consumeAndCloseStream(in); + throw new ElasticsearchParseException(e.getMessage(), e); + } + } + + private void consumeAndCloseStream(InputStream in) { + try { + // read anything left in the stream before + // closing the stream otherwise if the process + // tries to write more after the close it gets + // a SIGPIPE + byte[] buff = new byte[512]; + while (in.read(buff) >= 0) { + // Do nothing + } + in.close(); + } catch (IOException e) { + logger.warn("Error closing result parser input stream", e); + } + } + + private class AutodetectResultIterator implements CloseableIterator { + + private final InputStream in; + private final XContentParser parser; + + private XContentParser.Token token; + + private AutodetectResultIterator(InputStream in, XContentParser parser) { + this.in = in; + this.parser = parser; + token = parser.currentToken(); + } + + @Override + public boolean hasNext() { + try { + token = parser.nextToken(); + } catch (IOException e) { + throw new ElasticsearchParseException(e.getMessage(), e); + } + if (token == XContentParser.Token.END_ARRAY) { + return false; + } else if (token != XContentParser.Token.START_OBJECT) { + logger.error("Expecting Json Field name token after the Start Object token"); + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + return true; + } + + @Override + public AutodetectResult next() { + return AutodetectResult.PARSER.apply(parser, parseFieldMatcherSupplier); + } + + @Override + public void close() throws IOException { + consumeAndCloseStream(in); + } + + } + +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/FlushListener.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/FlushListener.java new file mode 100644 index 00000000000..398602c02d0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/FlushListener.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing; + +import java.util.Iterator; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +class FlushListener { + + final ConcurrentMap awaitingFlushed = new ConcurrentHashMap<>(); + final AtomicBoolean cleared = new AtomicBoolean(false); + + boolean waitForFlush(String flushId, long timeout) { + if (cleared.get()) { + return false; + } + + CountDownLatch latch = new CountDownLatch(1); + CountDownLatch previous = awaitingFlushed.putIfAbsent(flushId, latch); + if (previous != null) { + latch = previous; + } + try { + return latch.await(timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + void acknowledgeFlush(String flushId) { + CountDownLatch latch = awaitingFlushed.get(flushId); + if (latch == null) { + return; + } + + latch.countDown(); + } + + void clear() { + if (cleared.compareAndSet(false, true)) { + Iterator> latches = awaitingFlushed.entrySet().iterator(); + while (latches.hasNext()) { + latches.next().getValue().countDown(); + latches.remove(); + } + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/StateReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/StateReader.java new file mode 100644 index 00000000000..b03d101569a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/output/parsing/StateReader.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.xpack.prelert.job.persistence.JobResultsPersister; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A runnable class that reads the autodetect persisted state + * and writes the results via the {@linkplain JobResultsPersister} + * passed in the constructor. + */ +public class StateReader implements Runnable { + + private static final int READ_BUF_SIZE = 8192; + + private final InputStream stream; + private final Logger logger; + private final JobResultsPersister persister; + + public StateReader(JobResultsPersister persister, InputStream stream, Logger logger) { + this.stream = stream; + this.logger = logger; + this.persister = persister; + } + + @Override + public void run() { + try { + BytesReference bytesRef = null; + byte[] readBuf = new byte[READ_BUF_SIZE]; + for (int bytesRead = stream.read(readBuf); bytesRead != -1; bytesRead = stream.read(readBuf)) { + if (bytesRef == null) { + bytesRef = new BytesArray(readBuf, 0, bytesRead); + } else { + bytesRef = new CompositeBytesReference(bytesRef, new BytesArray(readBuf, 0, bytesRead)); + } + bytesRef = splitAndPersist(bytesRef); + readBuf = new byte[READ_BUF_SIZE]; + } + } catch (IOException e) { + logger.info("Error reading autodetect state output", e); + } + + logger.info("State output finished"); + } + + /** + * Splits bulk data streamed from the C++ process on '\0' characters. The + * data is expected to be a series of Elasticsearch bulk requests in UTF-8 JSON + * (as would be uploaded to the public REST API) separated by zero bytes ('\0'). + */ + private BytesReference splitAndPersist(BytesReference bytesRef) { + int from = 0; + while (true) { + int nextZeroByte = findNextZeroByte(bytesRef, from); + if (nextZeroByte == -1) { + // No more zero bytes in this block + break; + } + persister.persistBulkState(bytesRef.slice(from, nextZeroByte - from)); + from = nextZeroByte + 1; + } + return bytesRef.slice(from, bytesRef.length() - from); + } + + private static int findNextZeroByte(BytesReference bytesRef, int from) { + for (int i = from; i < bytesRef.length(); ++i) { + if (bytesRef.get(i) == 0) { + return i; + } + } + return -1; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/DataLoadParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/DataLoadParams.java new file mode 100644 index 00000000000..27904fc3e51 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/DataLoadParams.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.params; + +import java.util.Objects; + +public class DataLoadParams { + private final TimeRange resetTimeRange; + private final boolean ignoreDowntime; + + public DataLoadParams(TimeRange resetTimeRange) { + this(resetTimeRange, false); + } + + public DataLoadParams(TimeRange resetTimeRange, boolean ignoreDowntime) { + this.resetTimeRange = Objects.requireNonNull(resetTimeRange); + this.ignoreDowntime = ignoreDowntime; + } + + public boolean isResettingBuckets() { + return !getStart().isEmpty(); + } + + public String getStart() { + return resetTimeRange.getStart(); + } + + public String getEnd() { + return resetTimeRange.getEnd(); + } + + public boolean isIgnoreDowntime() { + return ignoreDowntime; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/InterimResultsParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/InterimResultsParams.java new file mode 100644 index 00000000000..cdcea7ba419 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/InterimResultsParams.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.util.Objects; + +public class InterimResultsParams { + private final boolean calcInterim; + private final TimeRange timeRange; + private final Long advanceTimeSeconds; + + private InterimResultsParams(boolean calcInterim, TimeRange timeRange, Long advanceTimeSeconds) { + this.calcInterim = calcInterim; + this.timeRange = Objects.requireNonNull(timeRange); + this.advanceTimeSeconds = advanceTimeSeconds; + } + + public boolean shouldCalculateInterim() { + return calcInterim; + } + + public boolean shouldAdvanceTime() { + return advanceTimeSeconds != null; + } + + public String getStart() { + return timeRange.getStart(); + } + + public String getEnd() { + return timeRange.getEnd(); + } + + public long getAdvanceTime() { + if (!shouldAdvanceTime()) { + throw new IllegalStateException(); + } + return advanceTimeSeconds; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InterimResultsParams that = (InterimResultsParams) o; + return calcInterim == that.calcInterim && + Objects.equals(timeRange, that.timeRange) && + Objects.equals(advanceTimeSeconds, that.advanceTimeSeconds); + } + + @Override + public int hashCode() { + return Objects.hash(calcInterim, timeRange, advanceTimeSeconds); + } + + public static class Builder { + private boolean calcInterim = false; + private TimeRange timeRange; + private String advanceTime; + + private Builder() { + calcInterim = false; + timeRange = TimeRange.builder().build(); + advanceTime = ""; + } + + public Builder calcInterim(boolean value) { + calcInterim = value; + return this; + } + + public Builder forTimeRange(TimeRange timeRange) { + this.timeRange = timeRange; + return this; + } + + public Builder advanceTime(String timestamp) { + advanceTime = timestamp; + return this; + } + + public InterimResultsParams build() { + checkValidFlushArgumentsCombination(); + Long advanceTimeSeconds = checkAdvanceTimeParam(); + return new InterimResultsParams(calcInterim, timeRange, advanceTimeSeconds); + } + + private void checkValidFlushArgumentsCombination() { + if (!calcInterim) { + checkFlushParamIsEmpty(TimeRange.START_PARAM, timeRange.getStart()); + checkFlushParamIsEmpty(TimeRange.END_PARAM, timeRange.getEnd()); + } else if (!isValidTimeRange(timeRange)) { + String msg = Messages.getMessage(Messages.REST_INVALID_FLUSH_PARAMS_MISSING, "start"); + throw new IllegalArgumentException(msg); + } + } + + private Long checkAdvanceTimeParam() { + if (advanceTime != null && !advanceTime.isEmpty()) { + return paramToEpochIfValidOrThrow("advanceTime", advanceTime) / TimeRange.MILLISECONDS_IN_SECOND; + } + return null; + } + + private long paramToEpochIfValidOrThrow(String paramName, String date) { + if (TimeRange.NOW.equals(date)) { + return System.currentTimeMillis(); + } + long epoch = 0; + if (date.isEmpty() == false) { + epoch = TimeUtils.dateStringToEpoch(date); + if (epoch < 0) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg); + } + } + return epoch; + } + + private void checkFlushParamIsEmpty(String paramName, String paramValue) { + if (!paramValue.isEmpty()) { + String msg = Messages.getMessage(Messages.REST_INVALID_FLUSH_PARAMS_UNEXPECTED, paramName); + throw new IllegalArgumentException(msg); + } + } + + private boolean isValidTimeRange(TimeRange timeRange) { + return !timeRange.getStart().isEmpty() || (timeRange.getStart().isEmpty() && timeRange.getEnd().isEmpty()); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/TimeRange.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/TimeRange.java new file mode 100644 index 00000000000..25658548eb6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/params/TimeRange.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.util.Objects; + +public class TimeRange { + + public static final String START_PARAM = "start"; + public static final String END_PARAM = "end"; + public static final String NOW = "now"; + public static final int MILLISECONDS_IN_SECOND = 1000; + + private final Long start; + private final Long end; + + private TimeRange(Long start, Long end) { + this.start = start; + this.end = end; + } + + public String getStart() { + return start == null ? "" : String.valueOf(start); + } + + public String getEnd() { + return end == null ? "" : String.valueOf(end); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeRange timeRange = (TimeRange) o; + return Objects.equals(start, timeRange.start) && + Objects.equals(end, timeRange.end); + } + + @Override + public int hashCode() { + return Objects.hash(start, end); + } + + public static class Builder { + + private String start = ""; + private String end = ""; + + private Builder() { + } + + public Builder startTime(String start) { + this.start = start; + return this; + } + + public Builder endTime(String end) { + this.end = end; + return this; + } + + /** + * Create a new TimeRange instance after validating the start and end params. + * Throws {@link ElasticsearchStatusException} if the validation fails + * @return The time range + */ + public TimeRange build() { + return createTimeRange(start, end); + } + + private TimeRange createTimeRange(String start, String end) { + Long epochStart = null; + Long epochEnd = null; + if (!start.isEmpty()) { + epochStart = paramToEpochIfValidOrThrow(START_PARAM, start) / MILLISECONDS_IN_SECOND; + epochEnd = paramToEpochIfValidOrThrow(END_PARAM, end) / MILLISECONDS_IN_SECOND; + if (end.isEmpty() || epochEnd.equals(epochStart)) { + epochEnd = epochStart + 1; + } + if (epochEnd < epochStart) { + String msg = Messages.getMessage(Messages.REST_START_AFTER_END, end, start); + throw new IllegalArgumentException(msg); + } + } else { + if (!end.isEmpty()) { + epochEnd = paramToEpochIfValidOrThrow(END_PARAM, end) / MILLISECONDS_IN_SECOND; + } + } + return new TimeRange(epochStart, epochEnd); + } + + /** + * Returns epoch milli seconds + */ + private long paramToEpochIfValidOrThrow(String paramName, String date) { + if (NOW.equals(date)) { + return System.currentTimeMillis(); + } + long epoch = 0; + if (date.isEmpty() == false) { + epoch = TimeUtils.dateStringToEpoch(date); + if (epoch < 0) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg); + } + } + + return epoch; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractDataToProcessWriter.java new file mode 100644 index 00000000000..a8634ac8ca4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractDataToProcessWriter.java @@ -0,0 +1,486 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.DataDescription.DataFormat; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; +import org.elasticsearch.xpack.prelert.transforms.DependencySorter; +import org.elasticsearch.xpack.prelert.transforms.Transform; +import org.elasticsearch.xpack.prelert.transforms.Transform.TransformIndex; +import org.elasticsearch.xpack.prelert.transforms.Transform.TransformResult; +import org.elasticsearch.xpack.prelert.transforms.TransformException; +import org.elasticsearch.xpack.prelert.transforms.TransformFactory; +import org.elasticsearch.xpack.prelert.transforms.date.DateFormatTransform; +import org.elasticsearch.xpack.prelert.transforms.date.DateTransform; +import org.elasticsearch.xpack.prelert.transforms.date.DoubleDateTransform; + +public abstract class AbstractDataToProcessWriter implements DataToProcessWriter { + protected static final int TIME_FIELD_OUT_INDEX = 0; + private static final int MS_IN_SECOND = 1000; + + protected final boolean includeControlField; + + protected final AutodetectProcess autodetectProcess; + protected final DataDescription dataDescription; + protected final AnalysisConfig analysisConfig; + protected final StatusReporter statusReporter; + protected final Logger logger; + protected final TransformConfigs transformConfigs; + + protected List dateInputTransforms; + protected DateTransform dateTransform; + protected List postDateTransforms; + + protected Map inFieldIndexes; + protected List inputOutputMap; + + private String[] scratchArea; + private String[][] readWriteArea; + + // epoch in seconds + private long latestEpochMs; + private long latestEpochMsThisUpload; + + + protected AbstractDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + TransformConfigs transformConfigs, StatusReporter statusReporter, Logger logger) { + this.includeControlField = includeControlField; + this.autodetectProcess = Objects.requireNonNull(autodetectProcess); + this.dataDescription = Objects.requireNonNull(dataDescription); + this.analysisConfig = Objects.requireNonNull(analysisConfig); + this.statusReporter = Objects.requireNonNull(statusReporter); + this.logger = Objects.requireNonNull(logger); + this.transformConfigs = Objects.requireNonNull(transformConfigs); + + postDateTransforms = new ArrayList<>(); + dateInputTransforms = new ArrayList<>(); + Date date = statusReporter.getLatestRecordTime(); + latestEpochMsThisUpload = 0; + latestEpochMs = 0; + if (date != null) { + latestEpochMs = date.getTime(); + } + + readWriteArea = new String[3][]; + } + + + /** + * Create the transforms. This must be called before {@linkplain #write(java.io.InputStream)} + * even if no transforms are configured as it creates the + * date transform and sets up the field mappings.
+ *

+ * Finds the required input indexes in the header + * and sets the mappings for the transforms so they know where + * to read their inputs and write outputs. + *

+ * Transforms can be chained so some write their outputs to + * a scratch area which is input to another transform + *

+ * Writes the header. + */ + public void buildTransformsAndWriteHeader(String[] header) throws IOException { + Collection inputFields = inputFields(); + inFieldIndexes = inputFieldIndexes(header, inputFields); + checkForMissingFields(inputFields, inFieldIndexes, header); + + Map outFieldIndexes = outputFieldIndexes(); + inputOutputMap = createInputOutputMap(inFieldIndexes); + statusReporter.setAnalysedFieldsPerRecord(analysisConfig.analysisFields().size()); + + Map scratchAreaIndexes = scratchAreaIndexes(inputFields, outputFields(), + dataDescription.getTimeField()); + scratchArea = new String[scratchAreaIndexes.size()]; + readWriteArea[TransformFactory.SCRATCH_ARRAY_INDEX] = scratchArea; + + + buildDateTransform(scratchAreaIndexes, outFieldIndexes); + + List dateInputTransforms = DependencySorter.findDependencies( + dataDescription.getTimeField(), transformConfigs.getTransforms()); + + + TransformFactory transformFactory = new TransformFactory(); + for (TransformConfig config : dateInputTransforms) { + Transform tr = transformFactory.create(config, inFieldIndexes, scratchAreaIndexes, + outFieldIndexes, logger); + this.dateInputTransforms.add(tr); + } + + // get the transforms that don't input into the date + List postDateTransforms = new ArrayList<>(); + for (TransformConfig tc : transformConfigs.getTransforms()) { + if (dateInputTransforms.contains(tc) == false) { + postDateTransforms.add(tc); + } + } + + postDateTransforms = DependencySorter.sortByDependency(postDateTransforms); + for (TransformConfig config : postDateTransforms) { + Transform tr = transformFactory.create(config, inFieldIndexes, scratchAreaIndexes, + outFieldIndexes, logger); + this.postDateTransforms.add(tr); + } + + writeHeader(outFieldIndexes); + } + + protected void buildDateTransform(Map scratchAreaIndexes, Map outFieldIndexes) { + boolean isDateFormatString = dataDescription.isTransformTime() + && !dataDescription.isEpochMs(); + + List readIndexes = new ArrayList<>(); + + Integer index = inFieldIndexes.get(dataDescription.getTimeField()); + if (index != null) { + readIndexes.add(new TransformIndex(TransformFactory.INPUT_ARRAY_INDEX, index)); + } else { + index = outFieldIndexes.get(dataDescription.getTimeField()); + if (index != null) { + // date field could also be an output field + readIndexes.add(new TransformIndex(TransformFactory.OUTPUT_ARRAY_INDEX, index)); + } else if (scratchAreaIndexes.containsKey(dataDescription.getTimeField())) { + index = scratchAreaIndexes.get(dataDescription.getTimeField()); + readIndexes.add(new TransformIndex(TransformFactory.SCRATCH_ARRAY_INDEX, index)); + } else { + throw new IllegalStateException( + String.format(Locale.ROOT, "Transform input date field '%s' not found", + dataDescription.getTimeField())); + } + } + + + List writeIndexes = new ArrayList<>(); + writeIndexes.add(new TransformIndex(TransformFactory.OUTPUT_ARRAY_INDEX, + outFieldIndexes.get(dataDescription.getTimeField()))); + + if (isDateFormatString) { + // Elasticsearch assumes UTC for dates without timezone information. + ZoneId defaultTimezone = dataDescription.getFormat() == DataFormat.ELASTICSEARCH + ? ZoneOffset.UTC : ZoneOffset.systemDefault(); + dateTransform = new DateFormatTransform(dataDescription.getTimeFormat(), + defaultTimezone, readIndexes, writeIndexes, logger); + } else { + dateTransform = new DoubleDateTransform(dataDescription.isEpochMs(), + readIndexes, writeIndexes, logger); + } + + } + + /** + * Transform the input data and write to length encoded writer.
+ *

+ * Fields that aren't transformed i.e. those in inputOutputMap must be + * copied from input to output before this function is called. + *

+ * First all the transforms whose outputs the Date transform relies + * on are executed then the date transform then the remaining transforms. + * + * @param input The record the transforms should read their input from. The contents should + * align with the header parameter passed to {@linkplain #buildTransformsAndWriteHeader(String[])} + * @param output The record that will be written to the length encoded writer. + * This should be the same size as the number of output (analysis fields) i.e. + * the size of the map returned by {@linkplain #outputFieldIndexes()} + * @param numberOfFieldsRead The total number read not just those included in the analysis + */ + protected boolean applyTransformsAndWrite(String[] input, String[] output, long numberOfFieldsRead) + throws IOException { + readWriteArea[TransformFactory.INPUT_ARRAY_INDEX] = input; + readWriteArea[TransformFactory.OUTPUT_ARRAY_INDEX] = output; + Arrays.fill(readWriteArea[TransformFactory.SCRATCH_ARRAY_INDEX], ""); + + if (!applyTransforms(dateInputTransforms, numberOfFieldsRead)) { + return false; + } + + try { + dateTransform.transform(readWriteArea); + } catch (TransformException e) { + statusReporter.reportDateParseError(numberOfFieldsRead); + logger.error(e.getMessage()); + return false; + } + + long epochMs = dateTransform.epochMs(); + + // Records have epoch seconds timestamp so compare for out of order in seconds + if (epochMs / MS_IN_SECOND < latestEpochMs / MS_IN_SECOND - analysisConfig.getLatency()) { + // out of order + statusReporter.reportOutOfOrderRecord(inFieldIndexes.size()); + + if (epochMs > latestEpochMsThisUpload) { + // record this timestamp even if the record won't be processed + latestEpochMsThisUpload = epochMs; + statusReporter.reportLatestTimeIncrementalStats(latestEpochMsThisUpload); + } + return false; + } + + // Now do the rest of the transforms + if (!applyTransforms(postDateTransforms, numberOfFieldsRead)) { + return false; + } + + latestEpochMs = Math.max(latestEpochMs, epochMs); + latestEpochMsThisUpload = latestEpochMs; + + autodetectProcess.writeRecord(output); + statusReporter.reportRecordWritten(numberOfFieldsRead, latestEpochMs); + + return true; + } + + /** + * If false then the transform is excluded + */ + private boolean applyTransforms(List transforms, long inputFieldCount) { + for (Transform tr : transforms) { + try { + TransformResult result = tr.transform(readWriteArea); + if (result == TransformResult.EXCLUDE) { + return false; + } + } catch (TransformException e) { + logger.warn(e); + } + } + + return true; + } + + + /** + * Write the header. + * The header is created from the list of analysis input fields, + * the time field and the control field + */ + protected void writeHeader(Map outFieldIndexes) throws IOException { + // header is all the analysis input fields + the time field + control field + int numFields = outFieldIndexes.size(); + String[] record = new String[numFields]; + + Iterator> itr = outFieldIndexes.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry entry = itr.next(); + record[entry.getValue()] = entry.getKey(); + } + + // Write the header + autodetectProcess.writeRecord(record); + } + + @Override + public void flush() throws IOException { + autodetectProcess.flushStream(); + } + + /** + * Get all the expected input fields i.e. all the fields we + * must see in the csv header. + * = transform input fields + analysis fields that aren't a transform output + * + the date field - the transform output field names + */ + public final Collection inputFields() { + Set requiredFields = new HashSet<>(analysisConfig.analysisFields()); + requiredFields.add(dataDescription.getTimeField()); + requiredFields.addAll(transformConfigs.inputFieldNames()); + + requiredFields.removeAll(transformConfigs.outputFieldNames()); // inputs not in a transform + + return requiredFields; + } + + /** + * Find the indexes of the input fields from the header + */ + protected final Map inputFieldIndexes(String[] header, Collection inputFields) { + List headerList = Arrays.asList(header); // TODO header could be empty + + Map fieldIndexes = new HashMap(); + + for (String field : inputFields) { + int index = headerList.indexOf(field); + if (index >= 0) { + fieldIndexes.put(field, index); + } + } + + return fieldIndexes; + } + + public Map getInputFieldIndexes() { + return inFieldIndexes; + } + + /** + * This output fields are the time field and all the fields + * configured for analysis + */ + public final Collection outputFields() { + List outputFields = new ArrayList<>(analysisConfig.analysisFields()); + outputFields.add(dataDescription.getTimeField()); + + return outputFields; + } + + /** + * Create indexes of the output fields. + * This is the time field and all the fields configured for analysis + * and the control field. + * Time is the first field and the last is the control field + */ + protected final Map outputFieldIndexes() { + Map fieldIndexes = new HashMap(); + + // time field + fieldIndexes.put(dataDescription.getTimeField(), TIME_FIELD_OUT_INDEX); + + int index = TIME_FIELD_OUT_INDEX + 1; + List analysisFields = analysisConfig.analysisFields(); + Collections.sort(analysisFields); + + for (String field : analysisConfig.analysisFields()) { + fieldIndexes.put(field, index++); + } + + // control field + if (includeControlField) { + fieldIndexes.put(LengthEncodedWriter.CONTROL_FIELD_NAME, index); + } + + return fieldIndexes; + } + + /** + * The number of fields used in the analysis field, + * the time field and (sometimes) the control field + */ + public int outputFieldCount() { + return analysisConfig.analysisFields().size() + (includeControlField ? 2 : 1); + } + + protected Map getOutputFieldIndexes() { + return outputFieldIndexes(); + } + + + /** + * Find all the scratch area fields. These are those that are input to a + * transform but are not written to the output or read from input. i.e. for + * the case where a transforms output is used exclusively by another + * transform + * + * @param inputFields + * Fields we expect in the header + * @param outputFields + * Fields that are written to the analytics + * @param dateTimeField date field + */ + protected final Map scratchAreaIndexes(Collection inputFields, Collection outputFields, + String dateTimeField) { + Set requiredFields = new HashSet<>(transformConfigs.outputFieldNames()); + boolean dateTimeFieldIsTransformOutput = requiredFields.contains(dateTimeField); + + requiredFields.addAll(transformConfigs.inputFieldNames()); + + requiredFields.removeAll(inputFields); + requiredFields.removeAll(outputFields); + + // date time is a output of a transform AND the input to the date time transform + // so add it back into the scratch area + if (dateTimeFieldIsTransformOutput) { + requiredFields.add(dateTimeField); + } + + int index = 0; + Map result = new HashMap(); + for (String field : requiredFields) { + result.put(field, new Integer(index++)); + } + + return result; + } + + + /** + * For inputs that aren't transformed create a map of input index + * to output index. This does not include the time or control fields + * + * @param inFieldIndexes Map of field name -> index in the input array + */ + protected final List createInputOutputMap(Map inFieldIndexes) { + // where no transform + List inputOutputMap = new ArrayList<>(); + + int outIndex = TIME_FIELD_OUT_INDEX + 1; + for (String field : analysisConfig.analysisFields()) { + Integer inIndex = inFieldIndexes.get(field); + if (inIndex != null) { + inputOutputMap.add(new InputOutputMap(inIndex, outIndex)); + } + + ++outIndex; + } + + return inputOutputMap; + } + + protected List getInputOutputMap() { + return inputOutputMap; + } + + + /** + * Check that all the fields are present in the header. + * Either return true or throw a MissingFieldException + *

+ * Every input field should have an entry in inputFieldIndexes + * otherwise the field cannnot be found. + */ + protected abstract boolean checkForMissingFields(Collection inputFields, Map inputFieldIndexes, + String[] header); + + + /** + * Input and output array indexes map + */ + protected class InputOutputMap { + int inputIndex; + int outputIndex; + + public InputOutputMap(int in, int out) { + inputIndex = in; + outputIndex = out; + } + } + + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractJsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractJsonRecordReader.java new file mode 100644 index 00000000000..cbab00cd3e2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AbstractJsonRecordReader.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; + +abstract class AbstractJsonRecordReader implements JsonRecordReader { + static final int PARSE_ERRORS_LIMIT = 100; + + // NORELEASE - Remove direct dependency on Jackson + protected final JsonParser parser; + protected final Map fieldMap; + protected final String recordHoldingField; + protected final Logger logger; + protected int nestedLevel; + protected long fieldCount; + protected int errorCounter; + + /** + * Create a reader that parses the mapped fields from JSON. + * + * @param parser + * The JSON parser + * @param fieldMap + * Map to field name to record array index position + * @param recordHoldingField + * record holding field + * @param logger + * the logger + */ + AbstractJsonRecordReader(JsonParser parser, Map fieldMap, String recordHoldingField, Logger logger) { + this.parser = Objects.requireNonNull(parser); + this.fieldMap = Objects.requireNonNull(fieldMap); + this.recordHoldingField = Objects.requireNonNull(recordHoldingField); + this.logger = Objects.requireNonNull(logger); + } + + protected void consumeToField(String field) throws IOException { + if (field == null || field.isEmpty()) { + return; + } + JsonToken token = null; + while ((token = tryNextTokenOrReadToEndOnError()) != null) { + if (token == JsonToken.FIELD_NAME + && parser.getCurrentName().equals(field)) { + tryNextTokenOrReadToEndOnError(); + return; + } + } + } + + protected void consumeToRecordHoldingField() throws IOException { + consumeToField(recordHoldingField); + } + + protected void initArrays(String[] record, boolean[] gotFields) { + Arrays.fill(gotFields, false); + Arrays.fill(record, ""); + } + + /** + * Returns null at the EOF or the next token + */ + protected JsonToken tryNextTokenOrReadToEndOnError() throws IOException { + try { + return parser.nextToken(); + } catch (JsonParseException e) { + logger.warn("Attempting to recover from malformed JSON data.", e); + for (int i = 0; i <= nestedLevel; ++i) { + readToEndOfObject(); + } + clearNestedLevel(); + } + + return parser.getCurrentToken(); + } + + protected abstract void clearNestedLevel(); + + /** + * In some cases the parser doesn't recognise the '}' of a badly formed + * JSON document and so may skip to the end of the second document. In this + * case we lose an extra record. + */ + protected void readToEndOfObject() throws IOException { + JsonToken token = null; + do { + try { + token = parser.nextToken(); + } catch (JsonParseException e) { + ++errorCounter; + if (errorCounter >= PARSE_ERRORS_LIMIT) { + logger.error("Failed to recover from malformed JSON data.", e); + throw new ElasticsearchParseException("The input JSON data is malformed."); + } + } + } + while (token != JsonToken.END_OBJECT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AggregatedJsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AggregatedJsonRecordReader.java new file mode 100644 index 00000000000..51dd68c69c8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AggregatedJsonRecordReader.java @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; + +/** + * Reads highly hierarchical JSON structures. Currently very much geared to the + * outputIndex of Elasticsearch's aggregations. Could be made more generic in the + * future if another slightly different hierarchical JSON structure needs to be + * parsed. + */ +class AggregatedJsonRecordReader extends AbstractJsonRecordReader { + private static final String AGG_KEY = "key"; + private static final String AGG_VALUE = "value"; + + private boolean isFirstTime = true; + + private final List nestingOrder; + private List nestedValues; + private String latestDocCount; + + /** + * Create a reader that simulates simple records given a hierarchical JSON + * structure where each field is at a progressively deeper level of nesting. + */ + AggregatedJsonRecordReader(JsonParser parser, Map fieldMap, String recordHoldingField, Logger logger, + List nestingOrder) { + super(parser, fieldMap, recordHoldingField, logger); + this.nestingOrder = Objects.requireNonNull(nestingOrder); + if (this.nestingOrder.isEmpty()) { + throw new IllegalArgumentException( + "Expected nesting order for aggregated JSON must not be empty"); + } + nestedValues = new ArrayList<>(); + } + + /** + * Read forwards in the JSON until enough information has been gathered to + * write to the record array. + * + * @param record Read fields are written to this array. This array is first filled with empty + * strings and will never contain a null + * @param gotFields boolean array each element is true if that field + * was read + * @return The number of fields in the aggregated hierarchy, or -1 if nothing was read + * because the end of the stream was reached + */ + @Override + public long read(String[] record, boolean[] gotFields) throws IOException { + initArrays(record, gotFields); + latestDocCount = null; + fieldCount = 0; + if (isFirstTime) { + clearNestedLevel(); + consumeToRecordHoldingField(); + isFirstTime = false; + } + + boolean gotInnerValue = false; + JsonToken token = tryNextTokenOrReadToEndOnError(); + while (!(token == JsonToken.END_OBJECT && nestedLevel == 0)) { + if (token == null) { + break; + } + + if (token == JsonToken.START_OBJECT) { + ++nestedLevel; + } else if (token == JsonToken.END_OBJECT) { + if (gotInnerValue) { + completeRecord(record, gotFields); + } + --nestedLevel; + if (nestedLevel % 2 == 0 && !nestedValues.isEmpty()) { + nestedValues.remove(nestedValues.size() - 1); + } + if (gotInnerValue) { + break; + } + } else if (token == JsonToken.FIELD_NAME) { + if (((nestedLevel + 1) / 2) == nestingOrder.size()) { + gotInnerValue = parseFieldValuePair(record, gotFields) || gotInnerValue; + } + // Alternate nesting levels are arbitrary labels that can be + // ignored. + else if (nestedLevel > 0 && nestedLevel % 2 == 0) { + String fieldName = parser.getCurrentName(); + if (fieldName.equals(AGG_KEY)) { + token = tryNextTokenOrReadToEndOnError(); + if (token == null) { + break; + } + nestedValues.add(parser.getText()); + } else if (fieldName.equals(SchedulerConfig.DOC_COUNT)) { + token = tryNextTokenOrReadToEndOnError(); + if (token == null) { + break; + } + latestDocCount = parser.getText(); + } + } + } + + token = tryNextTokenOrReadToEndOnError(); + } + + // null token means EOF; nestedLevel 0 means we've reached the end of + // the aggregations object + if (token == null || nestedLevel == 0) { + return -1; + } + return fieldCount; + } + + @Override + protected void clearNestedLevel() { + nestedLevel = 0; + } + + private boolean parseFieldValuePair(String[] record, boolean[] gotFields) throws IOException { + String fieldName = parser.getCurrentName(); + JsonToken token = tryNextTokenOrReadToEndOnError(); + + if (token == null) { + return false; + } + + if (token == JsonToken.START_OBJECT) { + ++nestedLevel; + return false; + } + + if (token == JsonToken.START_ARRAY) { + // We don't expect arrays at this level of aggregated inputIndex + // (although we do expect arrays at higher levels). Consume the + // whole array but do nothing with it. + while (token != JsonToken.END_ARRAY) { + token = tryNextTokenOrReadToEndOnError(); + } + return false; + } + + ++fieldCount; + + if (AGG_VALUE.equals(fieldName)) { + fieldName = nestingOrder.get(nestingOrder.size() - 1); + } + + Integer index = fieldMap.get(fieldName); + if (index == null) { + return false; + } + + String fieldValue = parser.getText(); + record[index] = fieldValue; + gotFields[index] = true; + + return true; + } + + private void completeRecord(String[] record, boolean[] gotFields) throws IOException { + // This loop should do time plus the by/over/partition/influencer fields + int numberOfFields = Math.min(nestingOrder.size() - 1, nestedValues.size()); + if (nestingOrder.size() - 1 != nestedValues.size()) { + logger.warn("Aggregation inputIndex does not match expectation: expected field order: " + + nestingOrder + " actual values: " + nestedValues); + } + fieldCount += numberOfFields; + for (int i = 0; i < numberOfFields; ++i) { + String fieldName = nestingOrder.get(i); + Integer index = fieldMap.get(fieldName); + if (index == null) { + continue; + } + + String fieldValue = nestedValues.get(i); + record[index] = fieldValue; + gotFields[index] = true; + } + + // This adds the summary count field + if (latestDocCount != null) { + ++fieldCount; + Integer index = fieldMap.get(SchedulerConfig.DOC_COUNT); + if (index != null) { + record[index] = latestDocCount; + gotFields[index] = true; + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AnalysisLimitsWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AnalysisLimitsWriter.java new file mode 100644 index 00000000000..55d933fe1cd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/AnalysisLimitsWriter.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Objects; + +import org.elasticsearch.xpack.prelert.job.AnalysisLimits; + +import static org.elasticsearch.xpack.prelert.job.process.autodetect.writer.WriterConstants.EQUALS; +import static org.elasticsearch.xpack.prelert.job.process.autodetect.writer.WriterConstants.NEW_LINE; + +public class AnalysisLimitsWriter { + /* + * The configuration fields used in limits.conf + */ + private static final String MEMORY_STANZA_STR = "[memory]"; + private static final String RESULTS_STANZA_STR = "[results]"; + private static final String MODEL_MEMORY_LIMIT_CONFIG_STR = "modelmemorylimit"; + private static final String MAX_EXAMPLES_LIMIT_CONFIG_STR = "maxexamples"; + + private final AnalysisLimits limits; + private final OutputStreamWriter writer; + + public AnalysisLimitsWriter(AnalysisLimits limits, OutputStreamWriter writer) { + this.limits = Objects.requireNonNull(limits); + this.writer = Objects.requireNonNull(writer); + } + + public void write() throws IOException { + StringBuilder contents = new StringBuilder(MEMORY_STANZA_STR).append(NEW_LINE); + if (limits.getModelMemoryLimit() != null && limits.getModelMemoryLimit() != 0L) { + contents.append(MODEL_MEMORY_LIMIT_CONFIG_STR + EQUALS) + .append(limits.getModelMemoryLimit()).append(NEW_LINE); + } + + contents.append(RESULTS_STANZA_STR).append(NEW_LINE); + if (limits.getCategorizationExamplesLimit() != null) { + contents.append(MAX_EXAMPLES_LIMIT_CONFIG_STR + EQUALS) + .append(limits.getCategorizationExamplesLimit()) + .append(NEW_LINE); + } + + writer.write(contents.toString()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ControlMsgToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ControlMsgToProcessWriter.java new file mode 100644 index 00000000000..92e57c8cebc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ControlMsgToProcessWriter.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; + +/** + * A writer for sending control messages to the C++ autodetect process. + * The data written to outputIndex is length encoded. + */ +public class ControlMsgToProcessWriter { + /** + * This should be the same size as the buffer in the C++ autodetect process. + */ + public static final int FLUSH_SPACES_LENGTH = 8192; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String FLUSH_MESSAGE_CODE = "f"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String INTERIM_MESSAGE_CODE = "i"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + public static final String RESET_BUCKETS_MESSAGE_CODE = "r"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String ADVANCE_TIME_MESSAGE_CODE = "t"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + public static final String UPDATE_MESSAGE_CODE = "u"; + + /** + * An number to uniquely identify each flush so that subsequent code can + * wait for acknowledgement of the correct flush. + */ + private static AtomicLong ms_FlushNumber = new AtomicLong(1); + + private final LengthEncodedWriter lengthEncodedWriter; + private final int numberOfAnalysisFields; + + /** + * Construct the control message writer with a LengthEncodedWriter + * + * @param lengthEncodedWriter + * the writer + * @param numberOfAnalysisFields + * The number of fields configured for analysis not including the + * time field + */ + public ControlMsgToProcessWriter(LengthEncodedWriter lengthEncodedWriter, int numberOfAnalysisFields) { + this.lengthEncodedWriter = Objects.requireNonNull(lengthEncodedWriter); + this.numberOfAnalysisFields= numberOfAnalysisFields; + } + + /** + * Create the control message writer with a OutputStream. A + * LengthEncodedWriter is created on the OutputStream parameter + * + * @param os + * the output stream + * @param numberOfAnalysisFields + * The number of fields configured for analysis not including the + * time field + */ + public static ControlMsgToProcessWriter create(OutputStream os, int numberOfAnalysisFields) { + return new ControlMsgToProcessWriter(new LengthEncodedWriter(os), numberOfAnalysisFields); + } + + /** + * Send an instruction to calculate interim results to the C++ autodetect process. + * + * @param params Parameters indicating whether interim results should be written + * and for which buckets + */ + public void writeCalcInterimMessage(InterimResultsParams params) throws IOException { + if (params.shouldAdvanceTime()) { + writeMessage(ADVANCE_TIME_MESSAGE_CODE + params.getAdvanceTime()); + } + if (params.shouldCalculateInterim()) { + writeControlCodeFollowedByTimeRange(INTERIM_MESSAGE_CODE, params.getStart(), params.getEnd()); + } + } + + /** + * Send a flush message to the C++ autodetect process. + * This actually consists of two messages: one to carry the flush ID and the + * other (which might not be processed until much later) to fill the buffers + * and force prior messages through. + * + * @return an ID for this flush that will be echoed back by the C++ + * autodetect process once it is complete. + */ + public String writeFlushMessage() throws IOException { + String flushId = Long.toString(ms_FlushNumber.getAndIncrement()); + writeMessage(FLUSH_MESSAGE_CODE + flushId); + + char[] spaces = new char[FLUSH_SPACES_LENGTH]; + Arrays.fill(spaces, ' '); + writeMessage(new String(spaces)); + + lengthEncodedWriter.flush(); + return flushId; + } + + public void writeUpdateConfigMessage(String config) throws IOException { + writeMessage(UPDATE_MESSAGE_CODE + config); + } + + public void writeResetBucketsMessage(DataLoadParams params) throws IOException { + writeControlCodeFollowedByTimeRange(RESET_BUCKETS_MESSAGE_CODE, params.getStart(), params.getEnd()); + } + + private void writeControlCodeFollowedByTimeRange(String code, String start, String end) + throws IOException { + StringBuilder message = new StringBuilder(code); + if (start.isEmpty() == false) { + message.append(start); + message.append(' '); + message.append(end); + } + writeMessage(message.toString()); + } + + /** + * Transform the supplied control message to length encoded values and + * write to the OutputStream. + * The number of blank fields to make up a full record is deduced from + * analysisConfig. + * + * @param message The control message to write. + */ + private void writeMessage(String message) throws IOException { + + // The fields consist of all the analysis fields plus the time and the + // control field, hence + 2 + lengthEncodedWriter.writeNumFields(numberOfAnalysisFields + 2); + + // Write blank values for all analysis fields and the time + for (int i = -1; i < numberOfAnalysisFields; ++i) { + lengthEncodedWriter.writeField(""); + } + + // The control field comes last + lengthEncodedWriter.writeField(message); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvDataToProcessWriter.java new file mode 100644 index 00000000000..24705070e08 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvDataToProcessWriter.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; +import org.supercsv.io.CsvListReader; +import org.supercsv.prefs.CsvPreference; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * A writer for transforming and piping CSV data from an + * inputstream to outputstream. + * The data written to outputIndex is length encoded each record + * consists of number of fields followed by length/value pairs. + * See CLengthEncodedInputParser.h in the C++ code for a more + * detailed description. + * A control field is added to the end of each length encoded + * line. + */ +class CsvDataToProcessWriter extends AbstractDataToProcessWriter { + /** + * Maximum number of lines allowed within a single CSV record. + *

+ * In the scenario where there is a misplaced quote, there is + * the possibility that it results to a single record expanding + * over many lines. Supercsv will eventually deplete all memory + * from the JVM. We set a limit to an arbitrary large number + * to prevent that from happening. Unfortunately, supercsv + * throws an exception which means we cannot recover and continue + * reading new records from the next line. + */ + private static final int MAX_LINES_PER_RECORD = 10000; + + public CsvDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + TransformConfigs transforms, StatusReporter statusReporter, Logger logger) { + super(includeControlField, autodetectProcess, dataDescription, analysisConfig, transforms, statusReporter, logger); + } + + /** + * Read the csv inputIndex, transform to length encoded values and pipe to + * the OutputStream. If any of the expected fields in the transform inputs, + * analysis inputIndex or if the expected time field is missing from the CSV + * header a exception is thrown + */ + @Override + public DataCounts write(InputStream inputStream) throws IOException { + CsvPreference csvPref = new CsvPreference.Builder( + dataDescription.getQuoteCharacter(), + dataDescription.getFieldDelimiter(), + new String(new char[]{DataDescription.LINE_ENDING})) + .maxLinesPerRow(MAX_LINES_PER_RECORD).build(); + + statusReporter.startNewIncrementalCount(); + + try (CsvListReader csvReader = new CsvListReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), csvPref)) { + String[] header = csvReader.getHeader(true); + if (header == null) // null if EoF + { + return statusReporter.incrementalStats(); + } + + long inputFieldCount = Math.max(header.length - 1, 0); // time field doesn't count + + buildTransformsAndWriteHeader(header); + + //backing array for the inputIndex + String[] inputRecord = new String[header.length]; + + int maxIndex = 0; + for (Integer index : inFieldIndexes.values()) { + maxIndex = Math.max(index, maxIndex); + } + + int numFields = outputFieldCount(); + String[] record = new String[numFields]; + + List line; + while ((line = csvReader.read()) != null) { + Arrays.fill(record, ""); + + if (maxIndex >= line.size()) { + logger.warn("Not enough fields in csv record, expected at least " + maxIndex + ". " + line); + + for (InputOutputMap inOut : inputOutputMap) { + if (inOut.inputIndex >= line.size()) { + statusReporter.reportMissingField(); + continue; + } + + String field = line.get(inOut.inputIndex); + record[inOut.outputIndex] = (field == null) ? "" : field; + } + } else { + for (InputOutputMap inOut : inputOutputMap) { + String field = line.get(inOut.inputIndex); + record[inOut.outputIndex] = (field == null) ? "" : field; + } + } + + fillRecordFromLine(line, inputRecord); + applyTransformsAndWrite(inputRecord, record, inputFieldCount); + } + + // This function can throw + statusReporter.finishReporting(); + } + + return statusReporter.incrementalStats(); + } + + private static void fillRecordFromLine(List line, String[] record) { + Arrays.fill(record, ""); + for (int i = 0; i < Math.min(line.size(), record.length); i++) { + String value = line.get(i); + if (value != null) { + record[i] = value; + } + } + } + + @Override + protected boolean checkForMissingFields(Collection inputFields, Map inputFieldIndexes, String[] header) { + for (String field : inputFields) { + if (AnalysisConfig.AUTO_CREATED_FIELDS.contains(field)) { + continue; + } + Integer index = inputFieldIndexes.get(field); + if (index == null) { + String msg = String.format(Locale.ROOT, "Field configured for analysis " + + "'%s' is not in the CSV header '%s'", + field, Arrays.toString(header)); + + logger.error(msg); + throw new IllegalArgumentException(msg); + } + } + + return true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvRecordWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvRecordWriter.java new file mode 100644 index 00000000000..dd54eb0068f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/CsvRecordWriter.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; + +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.supercsv.io.CsvListWriter; +import org.supercsv.prefs.CsvPreference; + +/** + * Write the records to the outputIndex stream as UTF 8 encoded CSV + */ +public class CsvRecordWriter implements RecordWriter { + private final CsvListWriter writer; + + /** + * Create the writer on the OutputStream os. + * This object will never close os. + */ + public CsvRecordWriter(OutputStream os) { + writer = new CsvListWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8), CsvPreference.STANDARD_PREFERENCE); + } + + @Override + public void writeRecord(String[] record) throws IOException { + writer.write(record); + } + + @Override + public void writeRecord(List record) throws IOException { + writer.write(record); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriter.java new file mode 100644 index 00000000000..f636e8980f1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriter.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.InputStream; + +import org.elasticsearch.xpack.prelert.job.DataCounts; + +/** + * A writer for transforming and piping data from an + * inputstream to outputstream as the process expects. + */ +public interface DataToProcessWriter { + /** + * Reads the inputIndex, transform to length encoded values and pipe + * to the OutputStream. + * If any of the fields in analysisFields or the + * DataDescriptions timeField is missing from the CSV header + * a MissingFieldException is thrown + * + * @return Counts of the records processed, bytes read etc + */ + DataCounts write(InputStream inputStream) throws IOException; + + /** + * Flush the outputstream + */ + void flush() throws IOException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriterFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriterFactory.java new file mode 100644 index 00000000000..d2d66cdee4c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/DataToProcessWriterFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; + +/** + * Factory for creating the suitable writer depending on + * whether the data format is JSON or not, and on the kind + * of date transformation that should occur. + */ +public final class DataToProcessWriterFactory { + + private DataToProcessWriterFactory() { + + } + + /** + * Constructs a {@link DataToProcessWriter} depending on + * the data format and the time transformation. + * + * @return A {@link JsonDataToProcessWriter} if the data + * format is JSON or otherwise a {@link CsvDataToProcessWriter} + */ + public static DataToProcessWriter create(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + SchedulerConfig schedulerConfig, TransformConfigs transforms, + StatusReporter statusReporter, Logger logger) { + switch (dataDescription.getFormat()) { + case JSON: + case ELASTICSEARCH: + return new JsonDataToProcessWriter(includeControlField, autodetectProcess, dataDescription, analysisConfig, + schedulerConfig, transforms, statusReporter, logger); + case DELIMITED: + return new CsvDataToProcessWriter(includeControlField, autodetectProcess, dataDescription, analysisConfig, + transforms, statusReporter, logger); + case SINGLE_LINE: + return new SingleLineDataToProcessWriter(includeControlField, autodetectProcess, dataDescription, analysisConfig, + transforms, statusReporter, logger); + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/FieldConfigWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/FieldConfigWriter.java new file mode 100644 index 00000000000..1d7ee712093 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/FieldConfigWriter.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import static org.elasticsearch.xpack.prelert.job.process.autodetect.writer.WriterConstants.EQUALS; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.job.config.DefaultDetectorDescription; +import org.elasticsearch.xpack.prelert.job.detectionrules.DetectionRule; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.utils.PrelertStrings; + +public class FieldConfigWriter { + private static final String DETECTOR_PREFIX = "detector."; + private static final String DETECTOR_CLAUSE_SUFFIX = ".clause"; + private static final String DETECTOR_RULES_SUFFIX = ".rules"; + private static final String INFLUENCER_PREFIX = "influencer."; + private static final String CATEGORIZATION_FIELD_OPTION = " categorizationfield="; + private static final String CATEGORIZATION_FILTER_PREFIX = "categorizationfilter."; + private static final String LIST_PREFIX = "list."; + + // Note: for the Engine API summarycountfield is currently passed as a + // command line option to prelert_autodetect rather than in the field + // config file + + private static final char NEW_LINE = '\n'; + + private final AnalysisConfig config; + private final Set lists; + private final OutputStreamWriter writer; + private final Logger logger; + + public FieldConfigWriter(AnalysisConfig config, Set lists, + OutputStreamWriter writer, Logger logger) { + this.config = Objects.requireNonNull(config); + this.lists = Objects.requireNonNull(lists); + this.writer = Objects.requireNonNull(writer); + this.logger = Objects.requireNonNull(logger); + } + + /** + * Write the Prelert autodetect field options to the outputIndex stream. + */ + public void write() throws IOException { + StringBuilder contents = new StringBuilder(); + + writeDetectors(contents); + writeLists(contents); + writeAsEnumeratedSettings(CATEGORIZATION_FILTER_PREFIX, config.getCategorizationFilters(), + contents, true); + + // As values are written as entire settings rather than part of a + // clause no quoting is needed + writeAsEnumeratedSettings(INFLUENCER_PREFIX, config.getInfluencers(), contents, false); + + logger.debug("FieldConfig:\n" + contents.toString()); + + writer.write(contents.toString()); + } + + private void writeDetectors(StringBuilder contents) throws IOException { + int counter = 0; + for (Detector detector : config.getDetectors()) { + int detectorId = counter++; + writeDetectorClause(detectorId, detector, contents); + writeDetectorRules(detectorId, detector, contents); + } + } + + private void writeDetectorClause(int detectorId, Detector detector, StringBuilder contents) { + contents.append(DETECTOR_PREFIX).append(detectorId) + .append(DETECTOR_CLAUSE_SUFFIX).append(EQUALS); + + DefaultDetectorDescription.appendOn(detector, contents); + + if (Strings.isNullOrEmpty(config.getCategorizationFieldName()) == false) { + contents.append(CATEGORIZATION_FIELD_OPTION) + .append(quoteField(config.getCategorizationFieldName())); + } + + contents.append(NEW_LINE); + } + + private void writeDetectorRules(int detectorId, Detector detector, StringBuilder contents) throws IOException { + List rules = detector.getDetectorRules(); + if (rules == null || rules.isEmpty()) { + return; + } + + contents.append(DETECTOR_PREFIX).append(detectorId) + .append(DETECTOR_RULES_SUFFIX).append(EQUALS); + + contents.append('['); + boolean first = true; + for (DetectionRule rule : detector.getDetectorRules()) { + if (first) { + first = false; + } else { + contents.append(','); + } + contents.append(rule.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string()); + } + contents.append(']'); + + contents.append(NEW_LINE); + } + + private void writeLists(StringBuilder buffer) throws IOException { + for (ListDocument list : lists) { + + StringBuilder listAsJson = new StringBuilder(); + listAsJson.append('['); + boolean first = true; + for (String item : list.getItems()) { + if (first) { + first = false; + } else { + listAsJson.append(','); + } + listAsJson.append('"'); + listAsJson.append(item); + listAsJson.append('"'); + } + listAsJson.append(']'); + buffer.append(LIST_PREFIX).append(list.getId()).append(EQUALS).append(listAsJson) + .append(NEW_LINE); + } + } + + private static void writeAsEnumeratedSettings(String settingName, List values, StringBuilder buffer, boolean quote) { + if (values == null) { + return; + } + + int counter = 0; + for (String value : values) { + buffer.append(settingName).append(counter++).append(EQUALS) + .append(quote ? quoteField(value) : value).append(NEW_LINE); + } + } + + private static String quoteField(String field) { + return PrelertStrings.doubleQuoteIfNotAlphaNumeric(field); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonDataToProcessWriter.java new file mode 100644 index 00000000000..2295115883a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonDataToProcessWriter.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.DataDescription.DataFormat; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; + +/** + * A writer for transforming and piping JSON data from an + * inputstream to outputstream. + * The data written to outputIndex is length encoded each record + * consists of number of fields followed by length/value pairs. + * See CLengthEncodedInputParser.h in the C++ code for a more + * detailed description. + */ +class JsonDataToProcessWriter extends AbstractDataToProcessWriter { + private static final String ELASTICSEARCH_SOURCE_FIELD = "_source"; + + /** + * Scheduler config. May be null. + */ + private SchedulerConfig schedulerConfig; + + public JsonDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + SchedulerConfig schedulerConfig, TransformConfigs transforms, + StatusReporter statusReporter, Logger logger) { + super(includeControlField, autodetectProcess, dataDescription, analysisConfig, transforms, + statusReporter, logger); + this.schedulerConfig = schedulerConfig; + } + + /** + * Read the JSON inputIndex, transform to length encoded values and pipe to + * the OutputStream. No transformation is applied to the data the timestamp + * is expected in seconds from the epoch. If any of the fields in + * analysisFields or the DataDescriptions + * timeField is missing from the JOSN inputIndex an exception is thrown + */ + @Override + public DataCounts write(InputStream inputStream) throws IOException { + statusReporter.startNewIncrementalCount(); + + try (JsonParser parser = new JsonFactory().createParser(inputStream)) { + writeJson(parser); + + // this line can throw and will be propagated + statusReporter.finishReporting(); + } + + return statusReporter.incrementalStats(); + } + + private void writeJson(JsonParser parser) throws IOException { + Collection analysisFields = inputFields(); + + buildTransformsAndWriteHeader(analysisFields.toArray(new String[0])); + + int numFields = outputFieldCount(); + String[] input = new String[numFields]; + String[] record = new String[numFields]; + + // We never expect to get the control field + boolean[] gotFields = new boolean[analysisFields.size()]; + + JsonRecordReader recordReader = makeRecordReader(parser); + long inputFieldCount = recordReader.read(input, gotFields); + while (inputFieldCount >= 0) { + Arrays.fill(record, ""); + + inputFieldCount = Math.max(inputFieldCount - 1, 0); // time field doesn't count + + long missing = missingFieldCount(gotFields); + if (missing > 0) { + statusReporter.reportMissingFields(missing); + } + + for (InputOutputMap inOut : inputOutputMap) { + String field = input[inOut.inputIndex]; + record[inOut.outputIndex] = (field == null) ? "" : field; + } + + applyTransformsAndWrite(input, record, inputFieldCount); + + inputFieldCount = recordReader.read(input, gotFields); + } + } + + private String getRecordHoldingField() { + if (dataDescription.getFormat().equals(DataFormat.ELASTICSEARCH)) { + if (schedulerConfig != null) { + if (schedulerConfig.getAggregationsOrAggs() != null) { + return SchedulerConfig.AGGREGATIONS.getPreferredName(); + } + } + return ELASTICSEARCH_SOURCE_FIELD; + } + return ""; + } + + // TODO norelease: Feels like this is checked in the wrong place. The fact that there is a different format, should + // be specified to this class and this class shouldn't know about the existence of SchedulerConfig + private JsonRecordReader makeRecordReader(JsonParser parser) { + List nestingOrder = (schedulerConfig != null) ? + schedulerConfig.buildAggregatedFieldList() : Collections.emptyList(); + return nestingOrder.isEmpty() ? new SimpleJsonRecordReader(parser, inFieldIndexes, getRecordHoldingField(), logger) + : new AggregatedJsonRecordReader(parser, inFieldIndexes, getRecordHoldingField(), logger, nestingOrder); + } + + /** + * Don't enforce the check that all the fields are present in JSON docs. + * Always returns true + */ + @Override + protected boolean checkForMissingFields(Collection inputFields, + Map inputFieldIndexes, + String[] header) { + return true; + } + + /** + * Return the number of missing fields + */ + private static long missingFieldCount(boolean[] gotFieldFlags) { + long count = 0; + + for (int i = 0; i < gotFieldFlags.length; i++) { + if (gotFieldFlags[i] == false) { + ++count; + } + } + + return count; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonRecordReader.java new file mode 100644 index 00000000000..ffb99619cea --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/JsonRecordReader.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; + +/** + * Interface for classes that read the various styles of JSON inputIndex. + */ +interface JsonRecordReader { + /** + * Read some JSON and write to the record array. + * + * @param record Read fields are written to this array. This array is first filled with empty + * strings and will never contain a null + * @param gotFields boolean array each element is true if that field + * was read + * @return The number of fields in the JSON doc or -1 if nothing was read + * because the end of the stream was reached + */ + long read(String[] record, boolean[] gotFields) throws IOException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/LengthEncodedWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/LengthEncodedWriter.java new file mode 100644 index 00000000000..6be3cf1d83a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/LengthEncodedWriter.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * Writes the data records to the outputIndex stream as length encoded pairs. + * Each record consists of number of fields followed by length/value pairs. The + * first call to one the of the writeRecord() methods should be + * with the header fields, once the headers are written records can be written + * sequentially. + *

+ * See CLengthEncodedInputParser.h in the C++ code for a more detailed + * description. + *

+ */ +public class LengthEncodedWriter implements RecordWriter { + private OutputStream outputStream; + private ByteBuffer lengthBuffer; + + /** + * Create the writer on the OutputStream os. + * This object will never close os. + */ + public LengthEncodedWriter(OutputStream os) { + outputStream = os; + // This will be used to convert 32 bit integers to network byte order + lengthBuffer = ByteBuffer.allocate(4); // 4 == sizeof(int) + } + + + /** + * Convert each String in the record array to a length/value encoded pair + * and write to the outputstream. + */ + @Override + public void writeRecord(String[] record) throws IOException { + writeNumFields(record.length); + + for (String field : record) { + writeField(field); + } + } + + /** + * Convert each String in the record list to a length/value encoded + * pair and write to the outputstream. + */ + @Override + public void writeRecord(List record) throws IOException { + writeNumFields(record.size()); + + for (String field : record) { + writeField(field); + } + } + + + /** + * Lower level functions to write records individually. + * After this function is called {@link #writeField(String)} + * must be called numFields times. + */ + public void writeNumFields(int numFields) throws IOException { + // number fields + lengthBuffer.clear(); + lengthBuffer.putInt(numFields); + outputStream.write(lengthBuffer.array()); + } + + + /** + * Lower level functions to write record fields individually. + * {@linkplain #writeNumFields(int)} must be called first + */ + public void writeField(String field) throws IOException { + byte[] utf8Bytes = field.getBytes(StandardCharsets.UTF_8); + lengthBuffer.clear(); + lengthBuffer.putInt(utf8Bytes.length); + outputStream.write(lengthBuffer.array()); + outputStream.write(utf8Bytes); + } + + @Override + public void flush() throws IOException { + outputStream.flush(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ModelDebugConfigWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ModelDebugConfigWriter.java new file mode 100644 index 00000000000..613c5dbb812 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/ModelDebugConfigWriter.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import org.elasticsearch.xpack.prelert.job.ModelDebugConfig; + +import java.io.IOException; +import java.io.Writer; +import java.util.Objects; + +import static org.elasticsearch.xpack.prelert.job.process.autodetect.writer.WriterConstants.EQUALS; +import static org.elasticsearch.xpack.prelert.job.process.autodetect.writer.WriterConstants.NEW_LINE; + +public class ModelDebugConfigWriter { + private static final String WRITE_TO_STR = "writeto"; + private static final String BOUNDS_PERCENTILE_STR = "boundspercentile"; + private static final String TERMS_STR = "terms"; + + private final ModelDebugConfig modelDebugConfig; + private final Writer writer; + + public ModelDebugConfigWriter(ModelDebugConfig modelDebugConfig, Writer writer) { + this.modelDebugConfig = Objects.requireNonNull(modelDebugConfig); + this.writer = Objects.requireNonNull(writer); + } + + public void write() throws IOException { + StringBuilder contents = new StringBuilder(); + if (modelDebugConfig.getWriteTo() != null) { + contents.append(WRITE_TO_STR) + .append(EQUALS) + .append(modelDebugConfig.getWriteTo()) + .append(NEW_LINE); + } + + contents.append(BOUNDS_PERCENTILE_STR) + .append(EQUALS) + .append(modelDebugConfig.getBoundsPercentile()) + .append(NEW_LINE); + + String terms = modelDebugConfig.getTerms(); + contents.append(TERMS_STR) + .append(EQUALS) + .append(terms == null ? "" : terms) + .append(NEW_LINE); + + writer.write(contents.toString()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/RecordWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/RecordWriter.java new file mode 100644 index 00000000000..e544c364745 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/RecordWriter.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.util.List; + +/** + * Interface for classes that write arrays of strings to the + * Prelert analytics processes. + */ +public interface RecordWriter { + /** + * Value must match api::CAnomalyDetector::CONTROL_FIELD_NAME in the C++ + * code. + */ + String CONTROL_FIELD_NAME = "."; + + /** + * Write each String in the record array + */ + void writeRecord(String[] record) throws IOException; + + /** + * Write each String in the record list + */ + void writeRecord(List record) throws IOException; + + /** + * Flush the outputIndex stream. + */ + void flush() throws IOException; + +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SimpleJsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SimpleJsonRecordReader.java new file mode 100644 index 00000000000..05c5e109c33 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SimpleJsonRecordReader.java @@ -0,0 +1,177 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +class SimpleJsonRecordReader extends AbstractJsonRecordReader { + private Deque nestedFields; + private String nestedPrefix; + + /** + * Create a reader that parses the mapped fields from JSON. + * + * @param parser + * The JSON parser + * @param fieldMap + * Map to field name to record array index position + * @param recordHoldingField + * record field + * @param logger + * logger + */ + SimpleJsonRecordReader(JsonParser parser, Map fieldMap, String recordHoldingField, Logger logger) { + super(parser, fieldMap, recordHoldingField, logger); + } + + /** + * Read the JSON object and write to the record array. + * Nested objects are flattened with the field names separated by + * a '.'. + * e.g. for a record with a nested 'tags' object: + * "{"name":"my.test.metric1","tags":{"tag1":"blah","tag2":"boo"},"time":1350824400,"value":12345.678}" + * use 'tags.tag1' to reference the tag1 field in the nested object + *

+ * Array fields in the JSON are ignored + * + * @param record Read fields are written to this array. This array is first filled with empty + * strings and will never contain a null + * @param gotFields boolean array each element is true if that field + * was read + * @return The number of fields in the JSON doc or -1 if nothing was read + * because the end of the stream was reached + */ + @Override + public long read(String[] record, boolean[] gotFields) throws IOException { + initArrays(record, gotFields); + fieldCount = 0; + clearNestedLevel(); + consumeToRecordHoldingField(); + + JsonToken token = tryNextTokenOrReadToEndOnError(); + while (!(token == JsonToken.END_OBJECT && nestedLevel == 0)) { + if (token == null) { + break; + } + + if (token == JsonToken.END_OBJECT) { + --nestedLevel; + String objectFieldName = nestedFields.pop(); + + int lastIndex = nestedPrefix.length() - objectFieldName.length() - 1; + nestedPrefix = nestedPrefix.substring(0, lastIndex); + } else if (token == JsonToken.FIELD_NAME) { + parseFieldValuePair(record, gotFields); + } + + token = tryNextTokenOrReadToEndOnError(); + } + + // null token means EOF + if (token == null) { + return -1; + } + return fieldCount; + } + + @Override + protected void clearNestedLevel() { + nestedLevel = 0; + nestedFields = new ArrayDeque(); + nestedPrefix = ""; + } + + private void parseFieldValuePair(String[] record, boolean[] gotFields) throws IOException { + String fieldName = parser.getCurrentName(); + JsonToken token = tryNextTokenOrReadToEndOnError(); + + if (token == null) { + return; + } + + if (token == JsonToken.START_OBJECT) { + ++nestedLevel; + nestedFields.push(fieldName); + nestedPrefix = nestedPrefix + fieldName + "."; + } else { + if (token == JsonToken.START_ARRAY || token.isScalarValue()) { + ++fieldCount; + + // Only do the donkey work of converting the field value to a + // string if we need it + Integer index = fieldMap.get(nestedPrefix + fieldName); + if (index != null) { + record[index] = parseSingleFieldValue(token); + gotFields[index] = true; + } else { + skipSingleFieldValue(token); + } + } + } + } + + private String parseSingleFieldValue(JsonToken token) throws IOException { + if (token == JsonToken.START_ARRAY) { + // Convert any scalar values in the array to a comma delimited + // string. (Arrays of more complex objects are ignored.) + StringBuilder strBuilder = new StringBuilder(); + boolean needComma = false; + while (token != JsonToken.END_ARRAY) { + token = tryNextTokenOrReadToEndOnError(); + if (token.isScalarValue()) { + if (needComma) { + strBuilder.append(','); + } else { + needComma = true; + } + strBuilder.append(tokenToString(token)); + } + } + + return strBuilder.toString(); + } + + return tokenToString(token); + } + + private void skipSingleFieldValue(JsonToken token) throws IOException { + // Scalar values don't need any extra skip code + if (token == JsonToken.START_ARRAY) { + // Consume the whole array but do nothing with it + int arrayDepth = 1; + do { + token = tryNextTokenOrReadToEndOnError(); + if (token == JsonToken.END_ARRAY) { + --arrayDepth; + } else if (token == JsonToken.START_ARRAY) { + ++arrayDepth; + } + } + while (token != null && arrayDepth > 0); + } + } + + /** + * Get the text representation of the current token unless it's a null. + * Nulls are replaced with empty strings to match the way the rest of the + * product treats them (which in turn is shaped by the fact that CSV + * cannot distinguish empty string and null). + */ + private String tokenToString(JsonToken token) throws IOException { + if (token == null || token == JsonToken.VALUE_NULL) { + return ""; + } + return parser.getText(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SingleLineDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SingleLineDataToProcessWriter.java new file mode 100644 index 00000000000..357382f94bb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/SingleLineDataToProcessWriter.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.prelert.job.status.StatusReporter; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfigs; + +/** + * This writer is used for reading inputIndex data that are unstructured and + * each record is a single line. The writer applies transforms and pipes + * the records into length encoded outputIndex. + *

+ * This writer is expected only to be used in combination of transforms + * that will extract the time and the other fields used in the analysis. + *

+ * Records for which no time can be extracted will be ignored. + */ +public class SingleLineDataToProcessWriter extends AbstractDataToProcessWriter { + private static final String RAW = "raw"; + + protected SingleLineDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + TransformConfigs transformConfigs, StatusReporter statusReporter, Logger logger) { + super(includeControlField, autodetectProcess, dataDescription, analysisConfig, transformConfigs, statusReporter, logger); + } + + @Override + public DataCounts write(InputStream inputStream) throws IOException { + statusReporter.startNewIncrementalCount(); + + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + String[] header = {RAW}; + buildTransformsAndWriteHeader(header); + + int numFields = outputFieldCount(); + String[] record = new String[numFields]; + + for (String line = bufferedReader.readLine(); line != null; + line = bufferedReader.readLine()) { + Arrays.fill(record, ""); + applyTransformsAndWrite(new String[]{line}, record, 1); + } + statusReporter.finishReporting(); + } + + return statusReporter.incrementalStats(); + } + + @Override + protected boolean checkForMissingFields(Collection inputFields, + Map inputFieldIndexes, String[] header) { + return true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/WriterConstants.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/WriterConstants.java new file mode 100644 index 00000000000..b8ad94b2748 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/autodetect/writer/WriterConstants.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.autodetect.writer; + +/** + * Common constants for process writer classes + */ +public final class WriterConstants { + public static final char NEW_LINE = '\n'; + public static final String EQUALS = " = "; + + private WriterConstants() { + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/Renormaliser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/Renormaliser.java new file mode 100644 index 00000000000..c7b95bf5d29 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/Renormaliser.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; + +public interface Renormaliser { + /** + * Update the anomaly score field on all previously persisted buckets + * and all contained records + */ + void renormalise(Quantiles quantiles, Logger logger); + + /** + * Update the anomaly score field on all previously persisted buckets + * and all contained records and aggregate records to the partition + * level + */ + void renormaliseWithPartition(Quantiles quantiles, Logger logger); + + + /** + * Blocks until the renormaliser is idle and no further normalisation tasks are pending. + */ + void waitUntilIdle(); + + /** + * Shut down the renormaliser + */ + boolean shutdown(Logger logger); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/RenormaliserFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/RenormaliserFactory.java new file mode 100644 index 00000000000..8bac84d5084 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/RenormaliserFactory.java @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.normalizer; + + +public interface RenormaliserFactory { + Renormaliser create(String jobId); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/noop/NoOpRenormaliser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/noop/NoOpRenormaliser.java new file mode 100644 index 00000000000..374c4449d04 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/process/normalizer/noop/NoOpRenormaliser.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.process.normalizer.noop; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.prelert.job.process.normalizer.Renormaliser; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; + +/** + * A {@link Renormaliser} implementation that does absolutely nothing + * This should be removed when the normaliser code is ported + */ +public class NoOpRenormaliser implements Renormaliser { + // NORELEASE Remove once the normaliser code is ported + @Override + public void renormalise(Quantiles quantiles, Logger logger) { + + } + + @Override + public void renormaliseWithPartition(Quantiles quantiles, Logger logger) { + + } + + @Override + public void waitUntilIdle() { + + } + + @Override + public boolean shutdown(Logger logger) { + return true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java new file mode 100644 index 00000000000..17c8f1a9b40 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/quantiles/Quantiles.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.quantiles; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Quantiles Result POJO + */ +public class Quantiles extends ToXContentToBytes implements Writeable { + public static final String QUANTILES_ID = "hierarchical"; + + /** + * Field Names + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField QUANTILE_STATE = new ParseField("quantileState"); + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("quantiles"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG); + PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); + } + + private String jobId; + private Date timestamp; + private String quantileState; + + public Quantiles(String jobId, Date timestamp, String quantilesState) { + this.jobId = jobId; + this.timestamp = timestamp; + quantileState = quantilesState == null ? "" : quantilesState; + } + + public Quantiles(StreamInput in) throws IOException { + jobId = in.readString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + quantileState = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeOptionalString(quantileState); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (quantileState != null) { + builder.field(QUANTILE_STATE.getPreferredName(), quantileState); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getQuantileState() { + return quantileState; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, quantileState); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Quantiles == false) { + return false; + } + + Quantiles that = (Quantiles) other; + + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.quantileState, that.quantileState); + + + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyCause.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyCause.java new file mode 100644 index 00000000000..baa5cba88b5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyCause.java @@ -0,0 +1,383 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Anomaly Cause POJO. + * Used as a nested level inside population anomaly records. + */ +public class AnomalyCause extends ToXContentToBytes implements Writeable +{ + public static final ParseField ANOMALY_CAUSE = new ParseField("anomalyCause"); + /** + * Result fields + */ + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField OVER_FIELD_NAME = new ParseField("overFieldName"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("overFieldValue"); + public static final ParseField BY_FIELD_NAME = new ParseField("byFieldName"); + public static final ParseField BY_FIELD_VALUE = new ParseField("byFieldValue"); + public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlatedByFieldValue"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partitionFieldName"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partitionFieldValue"); + public static final ParseField FUNCTION = new ParseField("function"); + public static final ParseField FUNCTION_DESCRIPTION = new ParseField("functionDescription"); + public static final ParseField TYPICAL = new ParseField("typical"); + public static final ParseField ACTUAL = new ParseField("actual"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + + /** + * Metric Results + */ + public static final ParseField FIELD_NAME = new ParseField("fieldName"); + + public static final ObjectParser PARSER = new ObjectParser<>(ANOMALY_CAUSE.getPreferredName(), + AnomalyCause::new); + static { + PARSER.declareDouble(AnomalyCause::setProbability, PROBABILITY); + PARSER.declareString(AnomalyCause::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(AnomalyCause::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(AnomalyCause::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setFunction, FUNCTION); + PARSER.declareString(AnomalyCause::setFunctionDescription, FUNCTION_DESCRIPTION); + PARSER.declareDoubleArray(AnomalyCause::setTypical, TYPICAL); + PARSER.declareDoubleArray(AnomalyCause::setActual, ACTUAL); + PARSER.declareString(AnomalyCause::setFieldName, FIELD_NAME); + PARSER.declareString(AnomalyCause::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(AnomalyCause::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareObjectArray(AnomalyCause::setInfluencers, Influence.PARSER, INFLUENCERS); + } + + private double probability; + private String byFieldName; + private String byFieldValue; + private String correlatedByFieldValue; + private String partitionFieldName; + private String partitionFieldValue; + private String function; + private String functionDescription; + private List typical; + private List actual; + + private String fieldName; + + private String overFieldName; + private String overFieldValue; + + private List influencers; + + public AnomalyCause() { + } + + @SuppressWarnings("unchecked") + public AnomalyCause(StreamInput in) throws IOException { + probability = in.readDouble(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + correlatedByFieldValue = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + function = in.readOptionalString(); + functionDescription = in.readOptionalString(); + if (in.readBoolean()) { + typical = (List) in.readGenericValue(); + } + if (in.readBoolean()) { + actual = (List) in.readGenericValue(); + } + fieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + if (in.readBoolean()) { + influencers = in.readList(Influence::new); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(probability); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(correlatedByFieldValue); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(function); + out.writeOptionalString(functionDescription); + boolean hasTypical = typical != null; + out.writeBoolean(hasTypical); + if (hasTypical) { + out.writeGenericValue(typical); + } + boolean hasActual = actual != null; + out.writeBoolean(hasActual); + if (hasActual) { + out.writeGenericValue(actual); + } + out.writeOptionalString(fieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + boolean hasInfluencers = influencers != null; + out.writeBoolean(hasInfluencers); + if (hasInfluencers) { + out.writeList(influencers); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(PROBABILITY.getPreferredName(), probability); + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (correlatedByFieldValue != null) { + builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (function != null) { + builder.field(FUNCTION.getPreferredName(), function); + } + if (functionDescription != null) { + builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription); + } + if (typical != null) { + builder.field(TYPICAL.getPreferredName(), typical); + } + if (actual != null) { + builder.field(ACTUAL.getPreferredName(), actual); + } + if (fieldName != null) { + builder.field(FIELD_NAME.getPreferredName(), fieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (influencers != null) { + builder.field(INFLUENCERS.getPreferredName(), influencers); + } + builder.endObject(); + return builder; + } + + + public double getProbability() + { + return probability; + } + + public void setProbability(double value) + { + probability = value; + } + + + public String getByFieldName() + { + return byFieldName; + } + + public void setByFieldName(String value) + { + byFieldName = value.intern(); + } + + public String getByFieldValue() + { + return byFieldValue; + } + + public void setByFieldValue(String value) + { + byFieldValue = value.intern(); + } + + public String getCorrelatedByFieldValue() + { + return correlatedByFieldValue; + } + + public void setCorrelatedByFieldValue(String value) + { + correlatedByFieldValue = value.intern(); + } + + public String getPartitionFieldName() + { + return partitionFieldName; + } + + public void setPartitionFieldName(String field) + { + partitionFieldName = field.intern(); + } + + public String getPartitionFieldValue() + { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String value) + { + partitionFieldValue = value.intern(); + } + + public String getFunction() + { + return function; + } + + public void setFunction(String name) + { + function = name.intern(); + } + + public String getFunctionDescription() + { + return functionDescription; + } + + public void setFunctionDescription(String functionDescription) + { + this.functionDescription = functionDescription.intern(); + } + + public List getTypical() + { + return typical; + } + + public void setTypical(List typical) + { + this.typical = typical; + } + + public List getActual() + { + return actual; + } + + public void setActual(List actual) + { + this.actual = actual; + } + + public String getFieldName() + { + return fieldName; + } + + public void setFieldName(String field) + { + fieldName = field.intern(); + } + + public String getOverFieldName() + { + return overFieldName; + } + + public void setOverFieldName(String name) + { + overFieldName = name.intern(); + } + + public String getOverFieldValue() + { + return overFieldValue; + } + + public void setOverFieldValue(String value) + { + overFieldValue = value.intern(); + } + + public List getInfluencers() + { + return influencers; + } + + public void setInfluencers(List influencers) + { + this.influencers = influencers; + } + + @Override + public int hashCode() + { + return Objects.hash(probability, + actual, + typical, + byFieldName, + byFieldValue, + correlatedByFieldValue, + fieldName, + function, + functionDescription, + overFieldName, + overFieldValue, + partitionFieldName, + partitionFieldValue, + influencers); + } + + @Override + public boolean equals(Object other) + { + if (this == other) + { + return true; + } + + if (other instanceof AnomalyCause == false) + { + return false; + } + + AnomalyCause that = (AnomalyCause)other; + + return this.probability == that.probability && + Objects.deepEquals(this.typical, that.typical) && + Objects.deepEquals(this.actual, that.actual) && + Objects.equals(this.function, that.function) && + Objects.equals(this.functionDescription, that.functionDescription) && + Objects.equals(this.fieldName, that.fieldName) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.byFieldValue, that.byFieldValue) && + Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.overFieldValue, that.overFieldValue) && + Objects.equals(this.influencers, that.influencers); + } + + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyRecord.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyRecord.java new file mode 100644 index 00000000000..3c1a6aa0a46 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AnomalyRecord.java @@ -0,0 +1,578 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; + +/** + * Anomaly Record POJO. + * Uses the object wrappers Boolean and Double so null values + * can be returned if the members have not been set. + */ +public class AnomalyRecord extends ToXContentToBytes implements Writeable { + /** + * Serialisation fields + */ + public static final ParseField TYPE = new ParseField("results"); + + /** + * Result fields (all detector types) + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField RESULT_TYPE = new ParseField("result_type"); + public static final ParseField DETECTOR_INDEX = new ParseField("detectorIndex"); + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField BY_FIELD_NAME = new ParseField("byFieldName"); + public static final ParseField BY_FIELD_VALUE = new ParseField("byFieldValue"); + public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlatedByFieldValue"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partitionFieldName"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partitionFieldValue"); + public static final ParseField FUNCTION = new ParseField("function"); + public static final ParseField FUNCTION_DESCRIPTION = new ParseField("functionDescription"); + public static final ParseField TYPICAL = new ParseField("typical"); + public static final ParseField ACTUAL = new ParseField("actual"); + public static final ParseField IS_INTERIM = new ParseField("isInterim"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + public static final ParseField BUCKET_SPAN = new ParseField("bucketSpan"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + /** + * Metric Results (including population metrics) + */ + public static final ParseField FIELD_NAME = new ParseField("fieldName"); + + /** + * Population results + */ + public static final ParseField OVER_FIELD_NAME = new ParseField("overFieldName"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("overFieldValue"); + public static final ParseField CAUSES = new ParseField("causes"); + + /** + * Normalisation + */ + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + public static final ParseField NORMALIZED_PROBABILITY = new ParseField("normalizedProbability"); + public static final ParseField INITIAL_NORMALIZED_PROBABILITY = new ParseField("initialNormalizedProbability"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new AnomalyRecord((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareString((anomalyRecord, s) -> {}, RESULT_TYPE); + PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY); + PARSER.declareDouble(AnomalyRecord::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(AnomalyRecord::setNormalizedProbability, NORMALIZED_PROBABILITY); + PARSER.declareDouble(AnomalyRecord::setInitialNormalizedProbability, INITIAL_NORMALIZED_PROBABILITY); + PARSER.declareLong(AnomalyRecord::setBucketSpan, BUCKET_SPAN); + PARSER.declareInt(AnomalyRecord::setDetectorIndex, DETECTOR_INDEX); + PARSER.declareBoolean(AnomalyRecord::setInterim, IS_INTERIM); + PARSER.declareField(AnomalyRecord::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setFunction, FUNCTION); + PARSER.declareString(AnomalyRecord::setFunctionDescription, FUNCTION_DESCRIPTION); + PARSER.declareDoubleArray(AnomalyRecord::setTypical, TYPICAL); + PARSER.declareDoubleArray(AnomalyRecord::setActual, ACTUAL); + PARSER.declareString(AnomalyRecord::setFieldName, FIELD_NAME); + PARSER.declareString(AnomalyRecord::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareObjectArray(AnomalyRecord::setCauses, AnomalyCause.PARSER, CAUSES); + PARSER.declareObjectArray(AnomalyRecord::setInfluencers, Influence.PARSER, INFLUENCERS); + } + + public static final String RESULT_TYPE_VALUE = "record"; + + private final String jobId; + private String id; + private int detectorIndex; + private double probability; + private String byFieldName; + private String byFieldValue; + private String correlatedByFieldValue; + private String partitionFieldName; + private String partitionFieldValue; + private String function; + private String functionDescription; + private List typical; + private List actual; + private boolean isInterim; + + private String fieldName; + + private String overFieldName; + private String overFieldValue; + private List causes; + + private double anomalyScore; + private double normalizedProbability; + + private double initialNormalizedProbability; + + private Date timestamp; + private long bucketSpan; + + private List influencers; + + private boolean hadBigNormalisedUpdate; + + public AnomalyRecord(String jobId) { + this.jobId = jobId; + } + + @SuppressWarnings("unchecked") + public AnomalyRecord(StreamInput in) throws IOException { + jobId = in.readString(); + id = in.readOptionalString(); + detectorIndex = in.readInt(); + probability = in.readDouble(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + correlatedByFieldValue = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + function = in.readOptionalString(); + functionDescription = in.readOptionalString(); + fieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + if (in.readBoolean()) { + typical = (List) in.readGenericValue(); + } + if (in.readBoolean()) { + actual = (List) in.readGenericValue(); + } + isInterim = in.readBoolean(); + if (in.readBoolean()) { + causes = in.readList(AnomalyCause::new); + } + anomalyScore = in.readDouble(); + normalizedProbability = in.readDouble(); + initialNormalizedProbability = in.readDouble(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + bucketSpan = in.readLong(); + if (in.readBoolean()) { + influencers = in.readList(Influence::new); + } + hadBigNormalisedUpdate = in.readBoolean(); + + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(id); + out.writeInt(detectorIndex); + out.writeDouble(probability); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(correlatedByFieldValue); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(function); + out.writeOptionalString(functionDescription); + out.writeOptionalString(fieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + boolean hasTypical = typical != null; + out.writeBoolean(hasTypical); + if (hasTypical) { + out.writeGenericValue(typical); + } + boolean hasActual = actual != null; + out.writeBoolean(hasActual); + if (hasActual) { + out.writeGenericValue(actual); + } + out.writeBoolean(isInterim); + boolean hasCauses = causes != null; + out.writeBoolean(hasCauses); + if (hasCauses) { + out.writeList(causes); + } + out.writeDouble(anomalyScore); + out.writeDouble(normalizedProbability); + out.writeDouble(initialNormalizedProbability); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeLong(bucketSpan); + boolean hasInfluencers = influencers != null; + out.writeBoolean(hasInfluencers); + if (hasInfluencers) { + out.writeList(influencers); + } + out.writeBoolean(hadBigNormalisedUpdate); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + builder.field(RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(PROBABILITY.getPreferredName(), probability); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(NORMALIZED_PROBABILITY.getPreferredName(), normalizedProbability); + builder.field(INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), initialNormalizedProbability); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); + builder.field(IS_INTERIM.getPreferredName(), isInterim); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (correlatedByFieldValue != null) { + builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (function != null) { + builder.field(FUNCTION.getPreferredName(), function); + } + if (functionDescription != null) { + builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription); + } + if (typical != null) { + builder.field(TYPICAL.getPreferredName(), typical); + } + if (actual != null) { + builder.field(ACTUAL.getPreferredName(), actual); + } + if (fieldName != null) { + builder.field(FIELD_NAME.getPreferredName(), fieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (causes != null) { + builder.field(CAUSES.getPreferredName(), causes); + } + if (influencers != null) { + builder.field(INFLUENCERS.getPreferredName(), influencers); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return this.jobId; + } + + /** + * Data store ID of this record. May be null for records that have not been + * read from the data store. + */ + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public int getDetectorIndex() { + return detectorIndex; + } + + public void setDetectorIndex(int detectorIndex) { + this.detectorIndex = detectorIndex; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public double getNormalizedProbability() { + return normalizedProbability; + } + + public void setNormalizedProbability(double normalizedProbability) { + this.normalizedProbability = normalizedProbability; + } + + public double getInitialNormalizedProbability() { + return initialNormalizedProbability; + } + + public void setInitialNormalizedProbability(double initialNormalizedProbability) { + this.initialNormalizedProbability = initialNormalizedProbability; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + /** + * Bucketspan expressed in seconds + */ + public long getBucketSpan() { + return bucketSpan; + } + + /** + * Bucketspan expressed in seconds + */ + public void setBucketSpan(long bucketSpan) { + this.bucketSpan = bucketSpan; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double value) { + probability = value; + } + + + public String getByFieldName() { + return byFieldName; + } + + public void setByFieldName(String value) { + byFieldName = value.intern(); + } + + public String getByFieldValue() { + return byFieldValue; + } + + public void setByFieldValue(String value) { + byFieldValue = value.intern(); + } + + public String getCorrelatedByFieldValue() { + return correlatedByFieldValue; + } + + public void setCorrelatedByFieldValue(String value) { + correlatedByFieldValue = value.intern(); + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String field) { + partitionFieldName = field.intern(); + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String value) { + partitionFieldValue = value.intern(); + } + + public String getFunction() { + return function; + } + + public void setFunction(String name) { + function = name.intern(); + } + + public String getFunctionDescription() { + return functionDescription; + } + + public void setFunctionDescription(String functionDescription) { + this.functionDescription = functionDescription.intern(); + } + + public List getTypical() { + return typical; + } + + public void setTypical(List typical) { + this.typical = typical; + } + + public List getActual() { + return actual; + } + + public void setActual(List actual) { + this.actual = actual; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String field) { + fieldName = field.intern(); + } + + public String getOverFieldName() { + return overFieldName; + } + + public void setOverFieldName(String name) { + overFieldName = name.intern(); + } + + public String getOverFieldValue() { + return overFieldValue; + } + + public void setOverFieldValue(String value) { + overFieldValue = value.intern(); + } + + public List getCauses() { + return causes; + } + + public void setCauses(List causes) { + this.causes = causes; + } + + public void addCause(AnomalyCause cause) { + if (causes == null) { + causes = new ArrayList<>(); + } + causes.add(cause); + } + + public List getInfluencers() { + return influencers; + } + + public void setInfluencers(List influencers) { + this.influencers = influencers; + } + + + @Override + public int hashCode() { + // ID is NOT included in the hash, so that a record from the data store + // will hash the same as a record representing the same anomaly that did + // not come from the data store + + // hadBigNormalisedUpdate is also deliberately excluded from the hash + + return Objects.hash(detectorIndex, probability, anomalyScore, initialNormalizedProbability, + normalizedProbability, typical, actual, + function, functionDescription, fieldName, byFieldName, byFieldValue, correlatedByFieldValue, + partitionFieldName, partitionFieldValue, overFieldName, overFieldValue, + timestamp, isInterim, causes, influencers, jobId, RESULT_TYPE_VALUE); + } + + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof AnomalyRecord == false) { + return false; + } + + AnomalyRecord that = (AnomalyRecord) other; + + // ID is NOT compared, so that a record from the data store will compare + // equal to a record representing the same anomaly that did not come + // from the data store + + // hadBigNormalisedUpdate is also deliberately excluded from the test + return Objects.equals(this.jobId, that.jobId) + && this.detectorIndex == that.detectorIndex + && this.probability == that.probability + && this.anomalyScore == that.anomalyScore + && this.normalizedProbability == that.normalizedProbability + && this.initialNormalizedProbability == that.initialNormalizedProbability + && Objects.deepEquals(this.typical, that.typical) + && Objects.deepEquals(this.actual, that.actual) + && Objects.equals(this.function, that.function) + && Objects.equals(this.functionDescription, that.functionDescription) + && Objects.equals(this.fieldName, that.fieldName) + && Objects.equals(this.byFieldName, that.byFieldName) + && Objects.equals(this.byFieldValue, that.byFieldValue) + && Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue) + && Objects.equals(this.partitionFieldName, that.partitionFieldName) + && Objects.equals(this.partitionFieldValue, that.partitionFieldValue) + && Objects.equals(this.overFieldName, that.overFieldName) + && Objects.equals(this.overFieldValue, that.overFieldValue) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.isInterim, that.isInterim) + && Objects.equals(this.causes, that.causes) + && Objects.equals(this.influencers, that.influencers); + } + + public boolean hadBigNormalisedUpdate() { + return this.hadBigNormalisedUpdate; + } + + public void resetBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = false; + } + + public void raiseBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AutodetectResult.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AutodetectResult.java new file mode 100644 index 00000000000..f81b70bc659 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/AutodetectResult.java @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; + +import java.io.IOException; +import java.util.Objects; + +public class AutodetectResult extends ToXContentToBytes implements Writeable { + + public static final ParseField TYPE = new ParseField("autodetect_result"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new AutodetectResult((Bucket) a[0], (Quantiles) a[1], (ModelSnapshot) a[2], + a[3] == null ? null : ((ModelSizeStats.Builder) a[3]).build(), (ModelDebugOutput) a[4], (CategoryDefinition) a[5], + (FlushAcknowledgement) a[6])); + + static { + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Bucket.PARSER, Bucket.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Quantiles.PARSER, Quantiles.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSnapshot.PARSER, ModelSnapshot.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.PARSER, ModelSizeStats.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelDebugOutput.PARSER, ModelDebugOutput.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), CategoryDefinition.PARSER, CategoryDefinition.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), FlushAcknowledgement.PARSER, FlushAcknowledgement.TYPE); + } + + private final Bucket bucket; + private final Quantiles quantiles; + private final ModelSnapshot modelSnapshot; + private final ModelSizeStats modelSizeStats; + private final ModelDebugOutput modelDebugOutput; + private final CategoryDefinition categoryDefinition; + private final FlushAcknowledgement flushAcknowledgement; + + public AutodetectResult(Bucket bucket, Quantiles quantiles, ModelSnapshot modelSnapshot, ModelSizeStats modelSizeStats, + ModelDebugOutput modelDebugOutput, CategoryDefinition categoryDefinition, FlushAcknowledgement flushAcknowledgement) { + this.bucket = bucket; + this.quantiles = quantiles; + this.modelSnapshot = modelSnapshot; + this.modelSizeStats = modelSizeStats; + this.modelDebugOutput = modelDebugOutput; + this.categoryDefinition = categoryDefinition; + this.flushAcknowledgement = flushAcknowledgement; + } + + public AutodetectResult(StreamInput in) throws IOException { + if (in.readBoolean()) { + this.bucket = new Bucket(in); + } else { + this.bucket = null; + } + if (in.readBoolean()) { + this.quantiles = new Quantiles(in); + } else { + this.quantiles = null; + } + if (in.readBoolean()) { + this.modelSnapshot = new ModelSnapshot(in); + } else { + this.modelSnapshot = null; + } + if (in.readBoolean()) { + this.modelSizeStats = new ModelSizeStats(in); + } else { + this.modelSizeStats = null; + } + if (in.readBoolean()) { + this.modelDebugOutput = new ModelDebugOutput(in); + } else { + this.modelDebugOutput = null; + } + if (in.readBoolean()) { + this.categoryDefinition = new CategoryDefinition(in); + } else { + this.categoryDefinition = null; + } + if (in.readBoolean()) { + this.flushAcknowledgement = new FlushAcknowledgement(in); + } else { + this.flushAcknowledgement = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + boolean hasBucket = bucket != null; + out.writeBoolean(hasBucket); + if (hasBucket) { + bucket.writeTo(out); + } + boolean hasQuantiles = quantiles != null; + out.writeBoolean(hasQuantiles); + if (hasQuantiles) { + quantiles.writeTo(out); + } + boolean hasModelSnapshot = modelSnapshot != null; + out.writeBoolean(hasModelSnapshot); + if (hasModelSnapshot) { + modelSnapshot.writeTo(out); + } + boolean hasModelSizeStats = modelSizeStats != null; + out.writeBoolean(hasModelSizeStats); + if (hasModelSizeStats) { + modelSizeStats.writeTo(out); + } + boolean hasModelDebugOutput = modelDebugOutput != null; + out.writeBoolean(hasModelDebugOutput); + if (hasModelDebugOutput) { + modelDebugOutput.writeTo(out); + } + boolean hasCategoryDefinition = categoryDefinition != null; + out.writeBoolean(hasCategoryDefinition); + if (hasCategoryDefinition) { + categoryDefinition.writeTo(out); + } + boolean hasFlushAcknowledgement = flushAcknowledgement != null; + out.writeBoolean(hasFlushAcknowledgement); + if (hasFlushAcknowledgement) { + flushAcknowledgement.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (bucket != null) { + builder.field(Bucket.TYPE.getPreferredName(), bucket); + } + if (quantiles != null) { + builder.field(Quantiles.TYPE.getPreferredName(), quantiles); + } + if (modelSnapshot != null) { + builder.field(ModelSnapshot.TYPE.getPreferredName(), modelSnapshot); + } + if (modelSizeStats != null) { + builder.field(ModelSizeStats.TYPE.getPreferredName(), modelSizeStats); + } + if (modelDebugOutput != null) { + builder.field(ModelDebugOutput.TYPE.getPreferredName(), modelDebugOutput); + } + if (categoryDefinition != null) { + builder.field(CategoryDefinition.TYPE.getPreferredName(), categoryDefinition); + } + if (flushAcknowledgement != null) { + builder.field(FlushAcknowledgement.TYPE.getPreferredName(), flushAcknowledgement); + } + builder.endObject(); + return builder; + } + + public Bucket getBucket() { + return bucket; + } + + public Quantiles getQuantiles() { + return quantiles; + } + + public ModelSnapshot getModelSnapshot() { + return modelSnapshot; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public ModelDebugOutput getModelDebugOutput() { + return modelDebugOutput; + } + + public CategoryDefinition getCategoryDefinition() { + return categoryDefinition; + } + + public FlushAcknowledgement getFlushAcknowledgement() { + return flushAcknowledgement; + } + + @Override + public int hashCode() { + return Objects.hash(bucket, categoryDefinition, flushAcknowledgement, modelDebugOutput, modelSizeStats, modelSnapshot, quantiles); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AutodetectResult other = (AutodetectResult) obj; + return Objects.equals(bucket, other.bucket) && + Objects.equals(categoryDefinition, other.categoryDefinition) && + Objects.equals(flushAcknowledgement, other.flushAcknowledgement) && + Objects.equals(modelDebugOutput, other.modelDebugOutput) && + Objects.equals(modelSizeStats, other.modelSizeStats) && + Objects.equals(modelSnapshot, other.modelSnapshot) && + Objects.equals(quantiles, other.quantiles); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Bucket.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Bucket.java new file mode 100644 index 00000000000..8379a6eaa7d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Bucket.java @@ -0,0 +1,439 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * Bucket Result POJO + */ +public class Bucket extends ToXContentToBytes implements Writeable { + /* + * Field Names + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initialAnomalyScore"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("maxNormalizedProbability"); + public static final ParseField IS_INTERIM = new ParseField("isInterim"); + public static final ParseField RECORD_COUNT = new ParseField("recordCount"); + public static final ParseField EVENT_COUNT = new ParseField("eventCount"); + public static final ParseField RECORDS = new ParseField("records"); + public static final ParseField BUCKET_INFLUENCERS = new ParseField("bucketInfluencers"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + public static final ParseField BUCKET_SPAN = new ParseField("bucketSpan"); + public static final ParseField PROCESSING_TIME_MS = new ParseField("processingTimeMs"); + public static final ParseField PARTITION_SCORES = new ParseField("partitionScores"); + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("bucket"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new Bucket((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareField(Bucket::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(Bucket::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + PARSER.declareBoolean(Bucket::setInterim, IS_INTERIM); + PARSER.declareInt(Bucket::setRecordCount, RECORD_COUNT); + PARSER.declareLong(Bucket::setEventCount, EVENT_COUNT); + PARSER.declareObjectArray(Bucket::setRecords, AnomalyRecord.PARSER, RECORDS); + PARSER.declareObjectArray(Bucket::setBucketInfluencers, BucketInfluencer.PARSER, BUCKET_INFLUENCERS); + PARSER.declareObjectArray(Bucket::setInfluencers, Influencer.PARSER, INFLUENCERS); + PARSER.declareLong(Bucket::setBucketSpan, BUCKET_SPAN); + PARSER.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS); + PARSER.declareObjectArray(Bucket::setPartitionScores, PartitionScore.PARSER, PARTITION_SCORES); + } + + private final String jobId; + private String id; + private Date timestamp; + private double anomalyScore; + private long bucketSpan; + + private double initialAnomalyScore; + + private double maxNormalizedProbability; + private int recordCount; + private List records = Collections.emptyList(); + private long eventCount; + private boolean isInterim; + private boolean hadBigNormalisedUpdate; + private List bucketInfluencers = new ArrayList<>(); + private List influencers = Collections.emptyList(); + private long processingTimeMs; + private Map perPartitionMaxProbability = Collections.emptyMap(); + private List partitionScores = Collections.emptyList(); + + public Bucket(String jobId) { + this.jobId = jobId; + } + + @SuppressWarnings("unchecked") + public Bucket(StreamInput in) throws IOException { + jobId = in.readString(); + id = in.readOptionalString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + anomalyScore = in.readDouble(); + bucketSpan = in.readLong(); + initialAnomalyScore = in.readDouble(); + maxNormalizedProbability = in.readDouble(); + recordCount = in.readInt(); + records = in.readList(AnomalyRecord::new); + eventCount = in.readLong(); + isInterim = in.readBoolean(); + hadBigNormalisedUpdate = in.readBoolean(); + bucketInfluencers = in.readList(BucketInfluencer::new); + influencers = in.readList(Influencer::new); + processingTimeMs = in.readLong(); + perPartitionMaxProbability = (Map) in.readGenericValue(); + partitionScores = in.readList(PartitionScore::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(id); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeDouble(anomalyScore); + out.writeLong(bucketSpan); + out.writeDouble(initialAnomalyScore); + out.writeDouble(maxNormalizedProbability); + out.writeInt(recordCount); + out.writeList(records); + out.writeLong(eventCount); + out.writeBoolean(isInterim); + out.writeBoolean(hadBigNormalisedUpdate); + out.writeList(bucketInfluencers); + out.writeList(influencers); + out.writeLong(processingTimeMs); + out.writeGenericValue(perPartitionMaxProbability); + out.writeList(partitionScores); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + builder.field(RECORD_COUNT.getPreferredName(), recordCount); + builder.field(RECORDS.getPreferredName(), records); + builder.field(EVENT_COUNT.getPreferredName(), eventCount); + builder.field(IS_INTERIM.getPreferredName(), isInterim); + builder.field(BUCKET_INFLUENCERS.getPreferredName(), bucketInfluencers); + builder.field(INFLUENCERS.getPreferredName(), influencers); + builder.field(PROCESSING_TIME_MS.getPreferredName(), processingTimeMs); + builder.field(PARTITION_SCORES.getPreferredName(), partitionScores); + builder.endObject(); + return builder; + } + + + public String getJobId() { + return jobId; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + /** + * Timestamp expressed in seconds since the epoch (rather than Java's + * convention of milliseconds). + */ + public long getEpoch() { + return timestamp.getTime() / 1000; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + /** + * Bucketspan expressed in seconds + */ + public long getBucketSpan() { + return bucketSpan; + } + + /** + * Bucketspan expressed in seconds + */ + public void setBucketSpan(long bucketSpan) { + this.bucketSpan = bucketSpan; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double influenceScore) { + this.initialAnomalyScore = influenceScore; + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + this.maxNormalizedProbability = maxNormalizedProbability; + } + + public int getRecordCount() { + return recordCount; + } + + public void setRecordCount(int recordCount) { + this.recordCount = recordCount; + } + + /** + * Get all the anomaly records associated with this bucket + * + * @return All the anomaly records + */ + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + /** + * The number of records (events) actually processed in this bucket. + */ + public long getEventCount() { + return eventCount; + } + + public void setEventCount(long value) { + eventCount = value; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public long getProcessingTimeMs() { + return processingTimeMs; + } + + public void setProcessingTimeMs(long timeMs) { + processingTimeMs = timeMs; + } + + public List getInfluencers() { + return influencers; + } + + public void setInfluencers(List influences) { + this.influencers = influences; + } + + public List getBucketInfluencers() { + return bucketInfluencers; + } + + public void setBucketInfluencers(List bucketInfluencers) { + this.bucketInfluencers = bucketInfluencers; + } + + public void addBucketInfluencer(BucketInfluencer bucketInfluencer) { + if (bucketInfluencers == null) { + bucketInfluencers = new ArrayList<>(); + } + bucketInfluencers.add(bucketInfluencer); + } + + public List getPartitionScores() { + return partitionScores; + } + + public void setPartitionScores(List scores) { + partitionScores = scores; + } + + /** + * Box class for the stream collector function below + */ + private final class DoubleMaxBox { + private double value = 0.0; + + public DoubleMaxBox() { + } + + public void accept(double d) { + if (d > value) { + value = d; + } + } + + public DoubleMaxBox combine(DoubleMaxBox other) { + return (this.value > other.value) ? this : other; + } + + public Double value() { + return this.value; + } + } + + public Map calcMaxNormalizedProbabilityPerPartition() { + perPartitionMaxProbability = records.stream().collect(Collectors.groupingBy(AnomalyRecord::getPartitionFieldValue, Collector + .of(DoubleMaxBox::new, (m, ar) -> m.accept(ar.getNormalizedProbability()), DoubleMaxBox::combine, DoubleMaxBox::value))); + + return perPartitionMaxProbability; + } + + public Map getPerPartitionMaxProbability() { + return perPartitionMaxProbability; + } + + public void setPerPartitionMaxProbability(Map perPartitionMaxProbability) { + this.perPartitionMaxProbability = perPartitionMaxProbability; + } + + public double partitionAnomalyScore(String partitionValue) { + Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) + .findFirst(); + + return first.isPresent() ? first.get().getAnomalyScore() : 0.0; + } + + @Override + public int hashCode() { + // hadBigNormalisedUpdate is deliberately excluded from the hash + // as is id, which is generated by the datastore + return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, maxNormalizedProbability, recordCount, records, + isInterim, bucketSpan, bucketInfluencers, influencers); + } + + /** + * Compare all the fields and embedded anomaly records (if any) + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Bucket == false) { + return false; + } + + Bucket that = (Bucket) other; + + // hadBigNormalisedUpdate is deliberately excluded from the test + // as is id, which is generated by the datastore + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) + && (this.eventCount == that.eventCount) && (this.bucketSpan == that.bucketSpan) + && (this.anomalyScore == that.anomalyScore) && (this.initialAnomalyScore == that.initialAnomalyScore) + && (this.maxNormalizedProbability == that.maxNormalizedProbability) && (this.recordCount == that.recordCount) + && Objects.equals(this.records, that.records) && Objects.equals(this.isInterim, that.isInterim) + && Objects.equals(this.bucketInfluencers, that.bucketInfluencers) && Objects.equals(this.influencers, that.influencers); + } + + public boolean hadBigNormalisedUpdate() { + return hadBigNormalisedUpdate; + } + + public void resetBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = false; + } + + public void raiseBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = true; + } + + /** + * This method encapsulated the logic for whether a bucket should be + * normalised. The decision depends on two factors. + * + * The first is whether the bucket has bucket influencers. Since bucket + * influencers were introduced, every bucket must have at least one bucket + * influencer. If it does not, it means it is a bucket persisted with an + * older version and should not be normalised. + * + * The second factor has to do with minimising the number of buckets that + * are sent for normalisation. Buckets that have no records and a score of + * zero should not be normalised as their score will not change and they + * will just add overhead. + * + * @return true if the bucket should be normalised or false otherwise + */ + public boolean isNormalisable() { + if (bucketInfluencers == null || bucketInfluencers.isEmpty()) { + return false; + } + return anomalyScore > 0.0 || recordCount > 0; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/BucketInfluencer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/BucketInfluencer.java new file mode 100644 index 00000000000..5f3c21d38f1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/BucketInfluencer.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class BucketInfluencer extends ToXContentToBytes implements Writeable { + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("bucketInfluencer"); + + /** + * This is the field name of the time bucket influencer. + */ + public static final ParseField BUCKET_TIME = new ParseField("bucketTime"); + + /* + * Field names + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencerFieldName"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initialAnomalyScore"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + public static final ParseField RAW_ANOMALY_SCORE = new ParseField("rawAnomalyScore"); + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField IS_INTERIM = new ParseField("isInterim"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new BucketInfluencer((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareString(BucketInfluencer::setInfluencerFieldName, INFLUENCER_FIELD_NAME); + PARSER.declareDouble(BucketInfluencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setRawAnomalyScore, RAW_ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setProbability, PROBABILITY); + PARSER.declareBoolean(BucketInfluencer::setIsInterim, IS_INTERIM); + PARSER.declareField(BucketInfluencer::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private final String jobId; + private String influenceField; + private double initialAnomalyScore; + private double anomalyScore; + private double rawAnomalyScore; + private double probability; + private boolean isInterim; + private Date timestamp; + + public BucketInfluencer(String jobId) { + this.jobId = jobId; + } + + public BucketInfluencer(BucketInfluencer prototype) { + jobId = prototype.jobId; + influenceField = prototype.influenceField; + initialAnomalyScore = prototype.initialAnomalyScore; + anomalyScore = prototype.anomalyScore; + rawAnomalyScore = prototype.rawAnomalyScore; + probability = prototype.probability; + isInterim = prototype.isInterim; + timestamp = prototype.timestamp; + } + + public BucketInfluencer(StreamInput in) throws IOException { + jobId = in.readString(); + influenceField = in.readOptionalString(); + initialAnomalyScore = in.readDouble(); + anomalyScore = in.readDouble(); + rawAnomalyScore = in.readDouble(); + probability = in.readDouble(); + isInterim = in.readBoolean(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(influenceField); + out.writeDouble(initialAnomalyScore); + out.writeDouble(anomalyScore); + out.writeDouble(rawAnomalyScore); + out.writeDouble(probability); + out.writeBoolean(isInterim); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (influenceField != null) { + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); + } + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(RAW_ANOMALY_SCORE.getPreferredName(), rawAnomalyScore); + builder.field(PROBABILITY.getPreferredName(), probability); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.field(IS_INTERIM.getPreferredName(), isInterim); + builder.endObject(); + return builder; + } + + + public String getJobId() { + return jobId; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + public String getInfluencerFieldName() { + return influenceField; + } + + public void setInfluencerFieldName(String fieldName) { + this.influenceField = fieldName; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double influenceScore) { + this.initialAnomalyScore = influenceScore; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double score) { + anomalyScore = score; + } + + public double getRawAnomalyScore() { + return rawAnomalyScore; + } + + public void setRawAnomalyScore(double score) { + rawAnomalyScore = score; + } + + public void setIsInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public boolean isInterim() { + return isInterim; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public Date getTimestamp() { + return timestamp; + } + + @Override + public int hashCode() { + return Objects.hash(influenceField, initialAnomalyScore, anomalyScore, rawAnomalyScore, probability, isInterim, timestamp, jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + BucketInfluencer other = (BucketInfluencer) obj; + + return Objects.equals(influenceField, other.influenceField) && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0 + && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(rawAnomalyScore, other.rawAnomalyScore) == 0 + && Double.compare(probability, other.probability) == 0 && Objects.equals(isInterim, other.isInterim) + && Objects.equals(timestamp, other.timestamp) && Objects.equals(jobId, other.jobId); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/CategoryDefinition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/CategoryDefinition.java new file mode 100644 index 00000000000..d093f1d5bbe --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/CategoryDefinition.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +public class CategoryDefinition extends ToXContentToBytes implements Writeable { + + public static final ParseField TYPE = new ParseField("categoryDefinition"); + public static final ParseField CATEGORY_ID = new ParseField("categoryId"); + public static final ParseField TERMS = new ParseField("terms"); + public static final ParseField REGEX = new ParseField("regex"); + public static final ParseField MAX_MATCHING_LENGTH = new ParseField("maxMatchingLength"); + public static final ParseField EXAMPLES = new ParseField("examples"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new CategoryDefinition((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID); + PARSER.declareString(CategoryDefinition::setTerms, TERMS); + PARSER.declareString(CategoryDefinition::setRegex, REGEX); + PARSER.declareLong(CategoryDefinition::setMaxMatchingLength, MAX_MATCHING_LENGTH); + PARSER.declareStringArray(CategoryDefinition::setExamples, EXAMPLES); + } + + private final String jobId; + private long id = 0L; + private String terms = ""; + private String regex = ""; + private long maxMatchingLength = 0L; + private final Set examples; + + public CategoryDefinition(String jobId) { + this.jobId = jobId; + examples = new TreeSet<>(); + } + + public CategoryDefinition(StreamInput in) throws IOException { + jobId = in.readString(); + id = in.readLong(); + terms = in.readString(); + regex = in.readString(); + maxMatchingLength = in.readLong(); + examples = new TreeSet<>(in.readList(StreamInput::readString)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(id); + out.writeString(terms); + out.writeString(regex); + out.writeLong(maxMatchingLength); + out.writeStringList(new ArrayList<>(examples)); + } + + public String getJobId() { + return jobId; + } + + public long getCategoryId() { + return id; + } + + public void setCategoryId(long categoryId) { + id = categoryId; + } + + public String getTerms() { + return terms; + } + + public void setTerms(String terms) { + this.terms = terms; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public long getMaxMatchingLength() { + return maxMatchingLength; + } + + public void setMaxMatchingLength(long maxMatchingLength) { + this.maxMatchingLength = maxMatchingLength; + } + + public List getExamples() { + return new ArrayList<>(examples); + } + + public void setExamples(Collection examples) { + this.examples.clear(); + this.examples.addAll(examples); + } + + public void addExample(String example) { + examples.add(example); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(CATEGORY_ID.getPreferredName(), id); + builder.field(TERMS.getPreferredName(), terms); + builder.field(REGEX.getPreferredName(), regex); + builder.field(MAX_MATCHING_LENGTH.getPreferredName(), maxMatchingLength); + builder.field(EXAMPLES.getPreferredName(), examples); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof CategoryDefinition == false) { + return false; + } + CategoryDefinition that = (CategoryDefinition) other; + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.id, that.id) + && Objects.equals(this.terms, that.terms) + && Objects.equals(this.regex, that.regex) + && Objects.equals(this.maxMatchingLength, that.maxMatchingLength) + && Objects.equals(this.examples, that.examples); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, id, terms, regex, maxMatchingLength, examples); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influence.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influence.java new file mode 100644 index 00000000000..c8daaeda6e5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influence.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Influence field name and list of influence field values/score pairs + */ +public class Influence extends ToXContentToBytes implements Writeable +{ + /** + * Note all publicly exposed field names are "influencer" not "influence" + */ + public static final ParseField INFLUENCER = new ParseField("influencer"); + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencerFieldName"); + public static final ParseField INFLUENCER_FIELD_VALUES = new ParseField("influencerFieldValues"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + INFLUENCER.getPreferredName(), a -> new Influence((String) a[0], (List) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUES); + } + + private String field; + private List fieldValues; + + public Influence(String field, List fieldValues) + { + this.field = field; + this.fieldValues = fieldValues; + } + + public Influence(StreamInput in) throws IOException { + this.field = in.readString(); + this.fieldValues = Arrays.asList(in.readStringArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + out.writeStringArray(fieldValues.toArray(new String[fieldValues.size()])); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), field); + builder.field(INFLUENCER_FIELD_VALUES.getPreferredName(), fieldValues); + builder.endObject(); + return builder; + } + + public String getInfluencerFieldName() + { + return field; + } + + public List getInfluencerFieldValues() + { + return fieldValues; + } + + @Override + public int hashCode() + { + return Objects.hash(field, fieldValues); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) + { + return true; + } + + if (obj == null) + { + return false; + } + + if (getClass() != obj.getClass()) + { + return false; + } + + Influence other = (Influence) obj; + + return Objects.equals(field, other.field) && + Objects.equals(fieldValues, other.fieldValues); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influencer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influencer.java new file mode 100644 index 00000000000..68a3a9132da --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/Influencer.java @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class Influencer extends ToXContentToBytes implements Writeable { + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("influencer"); + + /* + * Field names + */ + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencerFieldName"); + public static final ParseField INFLUENCER_FIELD_VALUE = new ParseField("influencerFieldValue"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initialAnomalyScore"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomalyScore"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new Influencer((String) a[0], (String) a[1], (String) a[2])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE); + PARSER.declareDouble(Influencer::setProbability, PROBABILITY); + PARSER.declareDouble(Influencer::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Influencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareBoolean(Influencer::setInterim, Bucket.IS_INTERIM); + PARSER.declareField(Influencer::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private String jobId; + private String id; + private Date timestamp; + private String influenceField; + private String influenceValue; + private double probability; + private double initialAnomalyScore; + private double anomalyScore; + private boolean hadBigNormalisedUpdate; + private boolean isInterim; + + public Influencer(String jobId, String fieldName, String fieldValue) { + this.jobId = jobId; + influenceField = fieldName; + influenceValue = fieldValue; + } + + public Influencer(StreamInput in) throws IOException { + jobId = in.readString(); + id = in.readOptionalString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + influenceField = in.readString(); + influenceValue = in.readString(); + probability = in.readDouble(); + initialAnomalyScore = in.readDouble(); + anomalyScore = in.readDouble(); + hadBigNormalisedUpdate = in.readBoolean(); + isInterim = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(id); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeString(influenceField); + out.writeString(influenceValue); + out.writeDouble(probability); + out.writeDouble(initialAnomalyScore); + out.writeDouble(anomalyScore); + out.writeBoolean(hadBigNormalisedUpdate); + out.writeBoolean(isInterim); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); + builder.field(INFLUENCER_FIELD_VALUE.getPreferredName(), influenceValue); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(PROBABILITY.getPreferredName(), probability); + builder.field(Bucket.IS_INTERIM.getPreferredName(), isInterim); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + /** + * Data store ID of this record. May be null for records that have not been + * read from the data store. + */ + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date date) { + timestamp = date; + } + + public String getInfluencerFieldName() { + return influenceField; + } + + public String getInfluencerFieldValue() { + return influenceValue; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double influenceScore) { + initialAnomalyScore = influenceScore; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double score) { + anomalyScore = score; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean value) { + isInterim = value; + } + + public boolean hadBigNormalisedUpdate() { + return this.hadBigNormalisedUpdate; + } + + public void resetBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = false; + } + + public void raiseBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = true; + } + + @Override + public int hashCode() { + // ID is NOT included in the hash, so that a record from the data store + // will hash the same as a record representing the same anomaly that did + // not come from the data store + + // hadBigNormalisedUpdate is also deliberately excluded from the hash + + return Objects.hash(jobId, timestamp, influenceField, influenceValue, initialAnomalyScore, anomalyScore, probability, isInterim); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + Influencer other = (Influencer) obj; + + // ID is NOT compared, so that a record from the data store will compare + // equal to a record representing the same anomaly that did not come + // from the data store + + // hadBigNormalisedUpdate is also deliberately excluded from the test + return Objects.equals(jobId, other.jobId) && Objects.equals(timestamp, other.timestamp) + && Objects.equals(influenceField, other.influenceField) + && Objects.equals(influenceValue, other.influenceValue) + && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0 + && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(probability, other.probability) == 0 + && (isInterim == other.isInterim); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ModelDebugOutput.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ModelDebugOutput.java new file mode 100644 index 00000000000..58b2365b284 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ModelDebugOutput.java @@ -0,0 +1,338 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Model Debug POJO. + * Some of the fields being with the word "debug". This avoids creation of + * reserved words that are likely to clash with fields in the input data (due to + * the restrictions on Elasticsearch mappings). + */ +public class ModelDebugOutput extends ToXContentToBytes implements Writeable +{ + public static final ParseField TYPE = new ParseField("modelDebugOutput"); + public static final ParseField JOB_ID = new ParseField("jobId"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partitionFieldName"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partitionFieldValue"); + public static final ParseField OVER_FIELD_NAME = new ParseField("overFieldName"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("overFieldValue"); + public static final ParseField BY_FIELD_NAME = new ParseField("byFieldName"); + public static final ParseField BY_FIELD_VALUE = new ParseField("byFieldValue"); + public static final ParseField DEBUG_FEATURE = new ParseField("debugFeature"); + public static final ParseField DEBUG_LOWER = new ParseField("debugLower"); + public static final ParseField DEBUG_UPPER = new ParseField("debugUpper"); + public static final ParseField DEBUG_MEDIAN = new ParseField("debugMedian"); + public static final ParseField ACTUAL = new ParseField("actual"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new ModelDebugOutput((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareField(ModelDebugOutput::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareString(ModelDebugOutput::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setDebugFeature, DEBUG_FEATURE); + PARSER.declareDouble(ModelDebugOutput::setDebugLower, DEBUG_LOWER); + PARSER.declareDouble(ModelDebugOutput::setDebugUpper, DEBUG_UPPER); + PARSER.declareDouble(ModelDebugOutput::setDebugMedian, DEBUG_MEDIAN); + PARSER.declareDouble(ModelDebugOutput::setActual, ACTUAL); + } + + private final String jobId; + private Date timestamp; + private String id; + private String partitionFieldName; + private String partitionFieldValue; + private String overFieldName; + private String overFieldValue; + private String byFieldName; + private String byFieldValue; + private String debugFeature; + private double debugLower; + private double debugUpper; + private double debugMedian; + private double actual; + + public ModelDebugOutput(String jobId) { + this.jobId = jobId; + } + + public ModelDebugOutput(StreamInput in) throws IOException { + jobId = in.readString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + id = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + debugFeature = in.readOptionalString(); + debugLower = in.readDouble(); + debugUpper = in.readDouble(); + debugMedian = in.readDouble(); + actual = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeOptionalString(id); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(debugFeature); + out.writeDouble(debugLower); + out.writeDouble(debugUpper); + out.writeDouble(debugMedian); + out.writeDouble(actual); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (debugFeature != null) { + builder.field(DEBUG_FEATURE.getPreferredName(), debugFeature); + } + builder.field(DEBUG_LOWER.getPreferredName(), debugLower); + builder.field(DEBUG_UPPER.getPreferredName(), debugUpper); + builder.field(DEBUG_MEDIAN.getPreferredName(), debugMedian); + builder.field(ACTUAL.getPreferredName(), actual); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() + { + return id; + } + + public void setId(String id) + { + this.id = id; + } + + public Date getTimestamp() + { + return timestamp; + } + + public void setTimestamp(Date timestamp) + { + this.timestamp = timestamp; + } + + public String getPartitionFieldName() + { + return partitionFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) + { + this.partitionFieldName = partitionFieldName; + } + + public String getPartitionFieldValue() + { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String partitionFieldValue) + { + this.partitionFieldValue = partitionFieldValue; + } + + public String getOverFieldName() + { + return overFieldName; + } + + public void setOverFieldName(String overFieldName) + { + this.overFieldName = overFieldName; + } + + public String getOverFieldValue() + { + return overFieldValue; + } + + public void setOverFieldValue(String overFieldValue) + { + this.overFieldValue = overFieldValue; + } + + public String getByFieldName() + { + return byFieldName; + } + + public void setByFieldName(String byFieldName) + { + this.byFieldName = byFieldName; + } + + public String getByFieldValue() + { + return byFieldValue; + } + + public void setByFieldValue(String byFieldValue) + { + this.byFieldValue = byFieldValue; + } + + public String getDebugFeature() + { + return debugFeature; + } + + public void setDebugFeature(String debugFeature) + { + this.debugFeature = debugFeature; + } + + public double getDebugLower() + { + return debugLower; + } + + public void setDebugLower(double debugLower) + { + this.debugLower = debugLower; + } + + public double getDebugUpper() + { + return debugUpper; + } + + public void setDebugUpper(double debugUpper) + { + this.debugUpper = debugUpper; + } + + public double getDebugMedian() + { + return debugMedian; + } + + public void setDebugMedian(double debugMedian) + { + this.debugMedian = debugMedian; + } + + public double getActual() + { + return actual; + } + + public void setActual(double actual) + { + this.actual = actual; + } + + @Override + public boolean equals(Object other) + { + if (this == other) + { + return true; + } + if (other instanceof ModelDebugOutput == false) + { + return false; + } + // id excluded here as it is generated by the datastore + ModelDebugOutput that = (ModelDebugOutput) other; + return Objects.equals(this.jobId, that.jobId) && + Objects.equals(this.timestamp, that.timestamp) && + Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.overFieldValue, that.overFieldValue) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.byFieldValue, that.byFieldValue) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.debugFeature, that.debugFeature) && + this.debugLower == that.debugLower && + this.debugUpper == that.debugUpper && + this.debugMedian == that.debugMedian && + this.actual == that.actual; + } + + @Override + public int hashCode() + { + // id excluded here as it is generated by the datastore + return Objects.hash(jobId, timestamp, partitionFieldName, partitionFieldValue, + overFieldName, overFieldValue, byFieldName, byFieldValue, + debugFeature, debugLower, debugUpper, debugMedian, actual); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PageParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PageParams.java new file mode 100644 index 00000000000..5524777e1f0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PageParams.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class PageParams extends ToXContentToBytes implements Writeable { + + public static final ParseField PAGE = new ParseField("page"); + public static final ParseField FROM = new ParseField("from"); + public static final ParseField SIZE = new ParseField("size"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + PAGE.getPreferredName(), a -> new PageParams((int) a[0], (int) a[1])); + + public static final int MAX_FROM_SIZE_SUM = 10000; + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), FROM); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE); + } + + private final int from; + private final int size; + + public PageParams(StreamInput in) throws IOException { + this(in.readVInt(), in.readVInt()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(from); + out.writeVInt(size); + } + + public PageParams(int from, int SIZE) { + if (from < 0) { + throw new IllegalArgumentException("Parameter [" + FROM.getPreferredName() + "] cannot be < 0"); + } + if (SIZE < 0) { + throw new IllegalArgumentException("Parameter [" + PageParams.SIZE.getPreferredName() + "] cannot be < 0"); + } + if (from + SIZE > MAX_FROM_SIZE_SUM) { + throw new IllegalArgumentException("The sum of parameters [" + FROM.getPreferredName() + "] and [" + + PageParams.SIZE.getPreferredName() + "] cannot be higher than " + MAX_FROM_SIZE_SUM + "."); + } + this.from = from; + this.size = SIZE; + } + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FROM.getPreferredName(), from); + builder.field(SIZE.getPreferredName(), size); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(from, size); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + PageParams other = (PageParams) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PartitionScore.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PartitionScore.java new file mode 100644 index 00000000000..480c5085323 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/PartitionScore.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.Objects; + +public class PartitionScore extends ToXContentToBytes implements Writeable { + public static final ParseField PARTITION_SCORE = new ParseField("partitionScore"); + + private String partitionFieldValue; + private String partitionFieldName; + private double anomalyScore; + private double probability; + private boolean hadBigNormalisedUpdate; + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + PARTITION_SCORE.getPreferredName(), a -> new PartitionScore((String) a[0], (String) a[1], (Double) a[2], (Double) a[3])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); + PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.ANOMALY_SCORE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); + } + + public PartitionScore(String fieldName, String fieldValue, double anomalyScore, double probability) { + hadBigNormalisedUpdate = false; + partitionFieldName = fieldName; + partitionFieldValue = fieldValue; + this.anomalyScore = anomalyScore; + this.probability = probability; + } + + public PartitionScore(StreamInput in) throws IOException { + partitionFieldName = in.readString(); + partitionFieldValue = in.readString(); + anomalyScore = in.readDouble(); + probability = in.readDouble(); + hadBigNormalisedUpdate = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(partitionFieldName); + out.writeString(partitionFieldValue); + out.writeDouble(anomalyScore); + out.writeDouble(probability); + out.writeBoolean(hadBigNormalisedUpdate); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + builder.field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + builder.field(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(AnomalyRecord.PROBABILITY.getPreferredName(), probability); + builder.endObject(); + return builder; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String partitionFieldValue) { + this.partitionFieldValue = partitionFieldValue; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + @Override + public int hashCode() { + return Objects.hash(partitionFieldName, partitionFieldValue, probability, anomalyScore); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof PartitionScore == false) { + return false; + } + + PartitionScore that = (PartitionScore) other; + + // hadBigNormalisedUpdate is deliberately excluded from the test + // as is id, which is generated by the datastore + return Objects.equals(this.partitionFieldValue, that.partitionFieldValue) + && Objects.equals(this.partitionFieldName, that.partitionFieldName) && (this.probability == that.probability) + && (this.anomalyScore == that.anomalyScore); + } + + public boolean hadBigNormalisedUpdate() { + return hadBigNormalisedUpdate; + } + + public void resetBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = false; + } + + public void raiseBigNormalisedUpdateFlag() { + hadBigNormalisedUpdate = true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ReservedFieldNames.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ReservedFieldNames.java new file mode 100644 index 00000000000..e2f001d459c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/results/ReservedFieldNames.java @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.results; + +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.usage.Usage; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + + +/** + * Defines the field names that we use for our results. + * Fields from the raw data with these names are not added to any result. Even + * different types of results will not have raw data fields with reserved names + * added to them, as it could create confusion if in some results a given field + * contains raw data and in others it contains some aspect of our output. + */ +public final class ReservedFieldNames { + /** + * jobId isn't in this package, so redefine. + */ + private static final String JOB_ID_NAME = "jobId"; + + /** + * @timestamp isn't in this package, so redefine. + */ + private static final String ES_TIMESTAMP = "timestamp"; + + public static final String BUCKET_PROCESSING_TIME_TYPE = "bucketProcessingTime"; + public static final String AVERAGE_PROCESSING_TIME_MS = "averageProcessingTimeMs"; + + public static final String PARTITION_NORMALIZED_PROB_TYPE = "partitionNormalizedProb"; + public static final String PARTITION_NORMALIZED_PROBS = "partitionNormalizedProbs"; + + /** + * This array should be updated to contain all the field names that appear + * in any documents we store in our results index. (The reason it's any + * documents we store and not just results documents is that Elasticsearch + * 2.x requires mappings for given fields be consistent across all types + * in a given index.) + */ + private static final String[] RESERVED_FIELD_NAME_ARRAY = { + AnomalyCause.PROBABILITY.getPreferredName(), + AnomalyCause.OVER_FIELD_NAME.getPreferredName(), + AnomalyCause.OVER_FIELD_VALUE.getPreferredName(), + AnomalyCause.BY_FIELD_NAME.getPreferredName(), + AnomalyCause.BY_FIELD_VALUE.getPreferredName(), + AnomalyCause.CORRELATED_BY_FIELD_VALUE.getPreferredName(), + AnomalyCause.PARTITION_FIELD_NAME.getPreferredName(), + AnomalyCause.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyCause.FUNCTION.getPreferredName(), + AnomalyCause.FUNCTION_DESCRIPTION.getPreferredName(), + AnomalyCause.TYPICAL.getPreferredName(), + AnomalyCause.ACTUAL.getPreferredName(), + AnomalyCause.INFLUENCERS.getPreferredName(), + AnomalyCause.FIELD_NAME.getPreferredName(), + + AnomalyRecord.DETECTOR_INDEX.getPreferredName(), + AnomalyRecord.PROBABILITY.getPreferredName(), + AnomalyRecord.BY_FIELD_NAME.getPreferredName(), + AnomalyRecord.BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.CORRELATED_BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyRecord.FUNCTION.getPreferredName(), + AnomalyRecord.FUNCTION_DESCRIPTION.getPreferredName(), + AnomalyRecord.TYPICAL.getPreferredName(), + AnomalyRecord.ACTUAL.getPreferredName(), + AnomalyRecord.IS_INTERIM.getPreferredName(), + AnomalyRecord.INFLUENCERS.getPreferredName(), + AnomalyRecord.FIELD_NAME.getPreferredName(), + AnomalyRecord.OVER_FIELD_NAME.getPreferredName(), + AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), + AnomalyRecord.CAUSES.getPreferredName(), + AnomalyRecord.ANOMALY_SCORE.getPreferredName(), + AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), + AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), + AnomalyRecord.BUCKET_SPAN.getPreferredName(), + + Bucket.ANOMALY_SCORE.getPreferredName(), + Bucket.BUCKET_SPAN.getPreferredName(), + Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), + Bucket.IS_INTERIM.getPreferredName(), + Bucket.RECORD_COUNT.getPreferredName(), + Bucket.EVENT_COUNT.getPreferredName(), + Bucket.RECORDS.getPreferredName(), + Bucket.BUCKET_INFLUENCERS.getPreferredName(), + Bucket.INFLUENCERS.getPreferredName(), + Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), + Bucket.PROCESSING_TIME_MS.getPreferredName(), + Bucket.PARTITION_SCORES.getPreferredName(), + + BucketInfluencer.BUCKET_TIME.getPreferredName(), BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName(), + BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName(), BucketInfluencer.ANOMALY_SCORE.getPreferredName(), + BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName(), BucketInfluencer.PROBABILITY.getPreferredName(), + + AVERAGE_PROCESSING_TIME_MS, + + PARTITION_NORMALIZED_PROBS, + PARTITION_NORMALIZED_PROB_TYPE, + + CategoryDefinition.CATEGORY_ID.getPreferredName(), + CategoryDefinition.TERMS.getPreferredName(), + CategoryDefinition.REGEX.getPreferredName(), + CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName(), + CategoryDefinition.EXAMPLES.getPreferredName(), + + DataCounts.PROCESSED_RECORD_COUNT.getPreferredName(), + DataCounts.PROCESSED_FIELD_COUNT.getPreferredName(), + DataCounts.INPUT_BYTES.getPreferredName(), + DataCounts.INPUT_RECORD_COUNT.getPreferredName(), + DataCounts.INPUT_FIELD_COUNT.getPreferredName(), + DataCounts.INVALID_DATE_COUNT.getPreferredName(), + DataCounts.MISSING_FIELD_COUNT.getPreferredName(), + DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName(), + DataCounts.LATEST_RECORD_TIME.getPreferredName(), + + Influence.INFLUENCER_FIELD_NAME.getPreferredName(), + Influence.INFLUENCER_FIELD_VALUES.getPreferredName(), + + Influencer.PROBABILITY.getPreferredName(), + Influencer.INFLUENCER_FIELD_NAME.getPreferredName(), + Influencer.INFLUENCER_FIELD_VALUE.getPreferredName(), + Influencer.INITIAL_ANOMALY_SCORE.getPreferredName(), + Influencer.ANOMALY_SCORE.getPreferredName(), + + ModelDebugOutput.PARTITION_FIELD_NAME.getPreferredName(), ModelDebugOutput.PARTITION_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.OVER_FIELD_NAME.getPreferredName(), ModelDebugOutput.OVER_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.BY_FIELD_NAME.getPreferredName(), ModelDebugOutput.BY_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.DEBUG_FEATURE.getPreferredName(), ModelDebugOutput.DEBUG_LOWER.getPreferredName(), + ModelDebugOutput.DEBUG_UPPER.getPreferredName(), ModelDebugOutput.DEBUG_MEDIAN.getPreferredName(), + ModelDebugOutput.ACTUAL.getPreferredName(), + + ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), + ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName(), + ModelSizeStats.LOG_TIME_FIELD.getPreferredName(), + + // ModelSnapshot.DESCRIPTION is not reserved because it is an analyzed string + ModelSnapshot.RESTORE_PRIORITY.getPreferredName(), + ModelSnapshot.SNAPSHOT_ID.getPreferredName(), + ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName(), + ModelSizeStats.TYPE.getPreferredName(), + ModelSnapshot.LATEST_RECORD_TIME.getPreferredName(), + ModelSnapshot.LATEST_RESULT_TIME.getPreferredName(), + + Quantiles.QUANTILE_STATE.getPreferredName(), + + Usage.INPUT_BYTES, + Usage.INPUT_FIELD_COUNT, + Usage.INPUT_RECORD_COUNT, + Usage.TIMESTAMP, + Usage.TYPE, + + + JOB_ID_NAME, + ES_TIMESTAMP + }; + + /** + * A set of all reserved field names in our results. Fields from the raw + * data with these names are not added to any result. + */ + public static final Set RESERVED_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_FIELD_NAME_ARRAY)); + + private ReservedFieldNames() { + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ProblemTracker.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ProblemTracker.java new file mode 100644 index 00000000000..c7bc1afe815 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ProblemTracker.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler; + +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + *

+ * Keeps track of problems the scheduler encounters and audits + * messages appropriately. + *

+ *

+ * The {@code ProblemTracker} is expected to interact with multiple + * threads (lookback executor, real-time executor). However, each + * thread will be accessing in a sequential manner therefore we + * only need to ensure correct visibility. + *

+ */ +class ProblemTracker { + + private static final int EMPTY_DATA_WARN_COUNT = 10; + + private final Supplier auditor; + + private volatile boolean hasProblems; + private volatile boolean hadProblems; + private volatile String previousProblem; + + private volatile int emptyDataCount; + + public ProblemTracker(Supplier auditor) { + this.auditor = Objects.requireNonNull(auditor); + } + + /** + * Reports as analysis problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + public void reportAnalysisProblem(String problemMessage) { + reportProblem(Messages.JOB_AUDIT_SCHEDULER_DATA_ANALYSIS_ERROR, problemMessage); + } + + /** + * Reports as extraction problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + public void reportExtractionProblem(String problemMessage) { + reportProblem(Messages.JOB_AUDIT_SCHEDULER_DATA_EXTRACTION_ERROR, problemMessage); + } + + /** + * Reports the problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + private void reportProblem(String template, String problemMessage) { + hasProblems = true; + if (!Objects.equals(previousProblem, problemMessage)) { + previousProblem = problemMessage; + auditor.get().error(Messages.getMessage(template, problemMessage)); + } + } + + /** + * Updates the tracking of empty data cycles. If the number of consecutive empty data + * cycles reaches {@code EMPTY_DATA_WARN_COUNT}, a warning is reported. If non-empty + * is reported and a warning was issued previously, a recovery info is reported. + * + * @param empty Whether data was seen since last report + * @return {@code true} if an empty data warning was issued, {@code false} otherwise + */ + public boolean updateEmptyDataCount(boolean empty) { + if (empty && emptyDataCount < EMPTY_DATA_WARN_COUNT) { + emptyDataCount++; + if (emptyDataCount == EMPTY_DATA_WARN_COUNT) { + auditor.get().warning(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_NO_DATA)); + return true; + } + } else if (!empty) { + if (emptyDataCount >= EMPTY_DATA_WARN_COUNT) { + auditor.get().info(Messages.getMessage(Messages.JOB_AUDIR_SCHEDULER_DATA_SEEN_AGAIN)); + } + emptyDataCount = 0; + } + return false; + } + + public boolean hasProblems() { + return hasProblems; + } + + /** + * Issues a recovery message if appropriate and prepares for next report + */ + public void finishReport() { + if (!hasProblems && hadProblems) { + auditor.get().info(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_RECOVERED)); + } + + hadProblems = hasProblems; + hasProblems = false; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJob.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJob.java new file mode 100644 index 00000000000..9a1808d3271 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJob.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractor; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.TimeRange; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; +import java.util.function.Supplier; + +class ScheduledJob { + + private static final DataLoadParams DATA_LOAD_PARAMS = new DataLoadParams(TimeRange.builder().build()); + private static final int NEXT_TASK_DELAY_MS = 100; + + private final Logger logger; + private final Auditor auditor; + private final String jobId; + private final long frequencyMs; + private final long queryDelayMs; + private final DataExtractor dataExtractor; + private final DataProcessor dataProcessor; + private final Supplier currentTimeSupplier; + + private volatile long lookbackStartTimeMs; + private volatile Long lastEndTimeMs; + private volatile boolean running = true; + + ScheduledJob(String jobId, long frequencyMs, long queryDelayMs, DataExtractor dataExtractor, + DataProcessor dataProcessor, Auditor auditor, Supplier currentTimeSupplier, + long latestFinalBucketEndTimeMs, long latestRecordTimeMs) { + this.logger = Loggers.getLogger(jobId); + this.jobId = jobId; + this.frequencyMs = frequencyMs; + this.queryDelayMs = queryDelayMs; + this.dataExtractor = dataExtractor; + this.dataProcessor = dataProcessor; + this.auditor = auditor; + this.currentTimeSupplier = currentTimeSupplier; + + long lastEndTime = Math.max(latestFinalBucketEndTimeMs, latestRecordTimeMs); + if (lastEndTime > 0) { + lastEndTimeMs = lastEndTime; + } + } + + Long runLookBack(SchedulerState schedulerState) throws Exception { + long startMs = schedulerState.getStartTimeMillis(); + lookbackStartTimeMs = (lastEndTimeMs != null && lastEndTimeMs + 1 > startMs) ? lastEndTimeMs + 1 : startMs; + + Optional endMs = Optional.ofNullable(schedulerState.getEndTimeMillis()); + long lookbackEnd = endMs.orElse(currentTimeSupplier.get() - queryDelayMs); + boolean isLookbackOnly = endMs.isPresent(); + if (lookbackEnd <= lookbackStartTimeMs) { + if (isLookbackOnly) { + return null; + } else { + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_STARTED_REALTIME)); + return nextRealtimeTimestamp(); + } + } + + String msg = Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_STARTED_FROM_TO, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackStartTimeMs), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackEnd)); + auditor.info(msg); + + run(lookbackStartTimeMs, lookbackEnd, InterimResultsParams.builder().calcInterim(true).build()); + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_LOOKBACK_COMPLETED)); + logger.info("Lookback has finished"); + if (isLookbackOnly) { + return null; + } else { + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_CONTINUED_REALTIME)); + return nextRealtimeTimestamp(); + } + } + + long runRealtime() throws Exception { + long start = lastEndTimeMs == null ? lookbackStartTimeMs : lastEndTimeMs + 1; + long nowMinusQueryDelay = currentTimeSupplier.get() - queryDelayMs; + long end = toIntervalStartEpochMs(nowMinusQueryDelay); + InterimResultsParams.Builder flushParams = InterimResultsParams.builder() + .calcInterim(true) + .advanceTime(String.valueOf(lastEndTimeMs)); + run(start, end, flushParams.build()); + return nextRealtimeTimestamp(); + } + + public void stop() { + running = false; + dataExtractor.cancel(); + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_SCHEDULER_STOPPED)); + } + + public boolean isRunning() { + return running; + } + + private void run(long start, long end, InterimResultsParams flushParams) throws IOException { + if (end <= start) { + return; + } + + logger.trace("Searching data in: [" + start + ", " + end + ")"); + + RuntimeException error = null; + long recordCount = 0; + dataExtractor.newSearch(start, end, logger); + while (running && dataExtractor.hasNext()) { + Optional extractedData; + try { + extractedData = dataExtractor.next(); + } catch (Exception e) { + error = new ExtractionProblemException(e); + break; + } + if (extractedData.isPresent()) { + DataCounts counts; + try { + counts = dataProcessor.processData(jobId, extractedData.get(), DATA_LOAD_PARAMS); + } catch (Exception e) { + error = new AnalysisProblemException(e); + break; + } + recordCount += counts.getProcessedRecordCount(); + if (counts.getLatestRecordTimeStamp() != null) { + lastEndTimeMs = counts.getLatestRecordTimeStamp().getTime(); + } + } + } + + lastEndTimeMs = Math.max(lastEndTimeMs == null ? 0 : lastEndTimeMs, end - 1); + + // Ensure time is always advanced in order to avoid importing duplicate data. + // This is the reason we store the error rather than throw inline. + if (error != null) { + throw error; + } + + if (recordCount == 0) { + throw new EmptyDataCountException(); + } + + dataProcessor.flushJob(jobId, flushParams); + } + + private long nextRealtimeTimestamp() { + long epochMs = currentTimeSupplier.get() + frequencyMs; + return toIntervalStartEpochMs(epochMs) + NEXT_TASK_DELAY_MS; + } + + private long toIntervalStartEpochMs(long epochMs) { + return (epochMs / frequencyMs) * frequencyMs; + } + + class AnalysisProblemException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + AnalysisProblemException(Throwable cause) { + super(cause); + } + + } + + class ExtractionProblemException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + ExtractionProblemException(Throwable cause) { + super(cause); + } + } + + class EmptyDataCountException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + EmptyDataCountException() {} + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobService.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobService.java new file mode 100644 index 00000000000..6d17f4e5d8c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/ScheduledJobService.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.UpdateJobSchedulerStatusAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.config.DefaultFrequency; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractor; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractorFactory; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.persistence.BucketsQueryBuilder; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.Bucket; + +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; + +public class ScheduledJobService extends AbstractComponent { + + private final Client client; + private final JobProvider jobProvider; + private final DataProcessor dataProcessor; + private final DataExtractorFactory dataExtractorFactory; + private final ThreadPool threadPool; + private final Supplier currentTimeSupplier; + final ConcurrentMap registry; + + public ScheduledJobService(ThreadPool threadPool, Client client, JobProvider jobProvider, DataProcessor dataProcessor, + DataExtractorFactory dataExtractorFactory, Supplier currentTimeSupplier) { + super(Settings.EMPTY); + this.threadPool = threadPool; + this.client = Objects.requireNonNull(client); + this.jobProvider = Objects.requireNonNull(jobProvider); + this.dataProcessor = Objects.requireNonNull(dataProcessor); + this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); + this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier); + this.registry = ConcurrentCollections.newConcurrentMap(); + } + + public void start(Job job, Allocation allocation) { + SchedulerState schedulerState = allocation.getSchedulerState(); + if (schedulerState == null) { + throw new IllegalStateException("Job [" + job.getId() + "] is not a scheduled job"); + } + + if (schedulerState.getStatus() != JobSchedulerStatus.STARTING) { + throw new IllegalStateException("expected job scheduler status [" + JobSchedulerStatus.STARTING + "], but got [" + + schedulerState.getStatus() + "] instead"); + } + + if (registry.containsKey(allocation.getJobId())) { + throw new IllegalStateException("job [" + allocation.getJobId() + "] has already been started"); + } + + logger.info("Starting scheduler [{}]", allocation); + Holder holder = createJobScheduler(job); + registry.put(job.getId(), holder); + + threadPool.executor(PrelertPlugin.THREAD_POOL_NAME).execute(() -> { + try { + Long next = holder.scheduledJob.runLookBack(allocation.getSchedulerState()); + if (next != null) { + doScheduleRealtime(next, job.getId(), holder); + } else { + holder.scheduledJob.stop(); + requestStopping(job.getId()); + } + } catch (ScheduledJob.ExtractionProblemException e) { + holder.problemTracker.reportExtractionProblem(e.getCause().getMessage()); + } catch (ScheduledJob.AnalysisProblemException e) { + holder.problemTracker.reportAnalysisProblem(e.getCause().getMessage()); + } catch (ScheduledJob.EmptyDataCountException e) { + if (holder.problemTracker.updateEmptyDataCount(true)) { + requestStopping(job.getJobId()); + } + } catch (Exception e) { + logger.error("Failed lookback import for job[" + job.getId() + "]", e); + requestStopping(job.getId()); + } + holder.problemTracker.finishReport(); + }); + setJobSchedulerStatus(job.getId(), JobSchedulerStatus.STARTED); + } + + public void stop(Allocation allocation) { + SchedulerState schedulerState = allocation.getSchedulerState(); + if (schedulerState == null) { + throw new IllegalStateException("Job [" + allocation.getJobId() + "] is not a scheduled job"); + } + if (schedulerState.getStatus() != JobSchedulerStatus.STOPPING) { + throw new IllegalStateException("expected job scheduler status [" + JobSchedulerStatus.STOPPING + "], but got [" + + schedulerState.getStatus() + "] instead"); + } + + if (registry.containsKey(allocation.getJobId()) == false) { + throw new IllegalStateException("job [" + allocation.getJobId() + "] has not been started"); + } + + logger.info("Stopping scheduler for job [{}]", allocation.getJobId()); + Holder holder = registry.remove(allocation.getJobId()); + holder.scheduledJob.stop(); + dataProcessor.closeJob(allocation.getJobId()); + setJobSchedulerStatus(allocation.getJobId(), JobSchedulerStatus.STOPPED); + } + + public void stopAllJobs() { + for (Map.Entry entry : registry.entrySet()) { + entry.getValue().scheduledJob.stop(); + dataProcessor.closeJob(entry.getKey()); + } + registry.clear(); + } + + private void doScheduleRealtime(long delayInMsSinceEpoch, String jobId, Holder holder) { + if (holder.scheduledJob.isRunning()) { + TimeValue delay = computeNextDelay(delayInMsSinceEpoch); + logger.debug("Waiting [{}] before executing next realtime import for job [{}]", delay, jobId); + threadPool.schedule(delay, PrelertPlugin.THREAD_POOL_NAME, () -> { + long nextDelayInMsSinceEpoch; + try { + nextDelayInMsSinceEpoch = holder.scheduledJob.runRealtime(); + } catch (ScheduledJob.ExtractionProblemException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + holder.problemTracker.reportExtractionProblem(e.getCause().getMessage()); + } catch (ScheduledJob.AnalysisProblemException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + holder.problemTracker.reportAnalysisProblem(e.getCause().getMessage()); + } catch (ScheduledJob.EmptyDataCountException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + if (holder.problemTracker.updateEmptyDataCount(true)) { + holder.problemTracker.finishReport(); + requestStopping(jobId); + return; + } + } catch (Exception e) { + logger.error("Unexpected scheduler failure for job [" + jobId + "] stopping...", e); + requestStopping(jobId); + return; + } + holder.problemTracker.finishReport(); + doScheduleRealtime(nextDelayInMsSinceEpoch, jobId, holder); + }); + } else { + requestStopping(jobId); + } + } + + private void requestStopping(String jobId) { + setJobSchedulerStatus(jobId, JobSchedulerStatus.STOPPING); + } + + Holder createJobScheduler(Job job) { + Auditor auditor = jobProvider.audit(job.getJobId()); + Duration frequency = getFrequencyOrDefault(job); + Duration queryDelay = Duration.ofSeconds(job.getSchedulerConfig().getQueryDelay()); + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(job); + ScheduledJob scheduledJob = new ScheduledJob(job.getJobId(), frequency.toMillis(), queryDelay.toMillis(), + dataExtractor, dataProcessor, auditor, currentTimeSupplier, getLatestFinalBucketEndTimeMs(job), + getLatestRecordTimestamp(job)); + return new Holder(scheduledJob, new ProblemTracker(() -> auditor)); + } + + private long getLatestFinalBucketEndTimeMs(Job job) { + Duration bucketSpan = Duration.ofSeconds(job.getAnalysisConfig().getBucketSpan()); + long latestFinalBucketEndMs = -1L; + BucketsQueryBuilder.BucketsQuery latestBucketQuery = new BucketsQueryBuilder() + .sortField(Bucket.TIMESTAMP.getPreferredName()) + .sortDescending(true).size(1) + .includeInterim(false) + .build(); + QueryPage buckets; + try { + buckets = jobProvider.buckets(job.getId(), latestBucketQuery); + if (buckets.hits().size() == 1) { + latestFinalBucketEndMs = buckets.hits().get(0).getTimestamp().getTime() + bucketSpan.toMillis() - 1; + } + } catch (ResourceNotFoundException e) { + logger.error("Could not retrieve latest bucket timestamp", e); + } + return latestFinalBucketEndMs; + } + + private long getLatestRecordTimestamp(Job job) { + long latestRecordTimeMs = -1L; + if (job.getCounts() != null && job.getCounts().getLatestRecordTimeStamp() != null) { + latestRecordTimeMs = job.getCounts().getLatestRecordTimeStamp().getTime(); + } + return latestRecordTimeMs; + } + + private static Duration getFrequencyOrDefault(Job job) { + Long frequency = job.getSchedulerConfig().getFrequency(); + Long bucketSpan = job.getAnalysisConfig().getBucketSpan(); + return frequency == null ? DefaultFrequency.ofBucketSpan(bucketSpan) : Duration.ofSeconds(frequency); + } + + private TimeValue computeNextDelay(long next) { + return new TimeValue(Math.max(1, next - currentTimeSupplier.get())); + } + + private void setJobSchedulerStatus(String jobId, JobSchedulerStatus status) { + UpdateJobSchedulerStatusAction.Request request = new UpdateJobSchedulerStatusAction.Request(jobId, status); + client.execute(UpdateJobSchedulerStatusAction.INSTANCE, request, new ActionListener() { + @Override + public void onResponse(UpdateJobSchedulerStatusAction.Response response) { + if (response.isAcknowledged()) { + logger.debug("successfully set job scheduler status to [{}] for job [{}]", status, jobId); + } else { + logger.info("set job scheduler status to [{}] for job [{}], but was not acknowledged", status, jobId); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("could not set job scheduler status to [" + status + "] for job [" + jobId +"]", e); + } + }); + } + + private static class Holder { + + private final ScheduledJob scheduledJob; + private final ProblemTracker problemTracker; + + private Holder(ScheduledJob scheduledJob, ProblemTracker problemTracker) { + this.scheduledJob = scheduledJob; + this.problemTracker = problemTracker; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchDataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchDataExtractor.java new file mode 100644 index 00000000000..7348f757b10 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchDataExtractor.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractor; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Locale; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ElasticsearchDataExtractor implements DataExtractor { + + private static final String CLEAR_SCROLL_TEMPLATE = "{\"scroll_id\":[\"%s\"]}"; + private static final Pattern TOTAL_HITS_PATTERN = Pattern.compile("\"hits\":\\{.*?\"total\":(.*?),"); + private static final Pattern EARLIEST_TIME_PATTERN = Pattern.compile("\"earliestTime\":\\{.*?\"value\":(.*?),"); + private static final Pattern LATEST_TIME_PATTERN = Pattern.compile("\"latestTime\":\\{.*?\"value\":(.*?),"); + private static final Pattern INDEX_PATTERN = Pattern.compile("\"_index\":\"(.*?)\""); + private static final Pattern NUMBER_OF_SHARDS_PATTERN = Pattern.compile("\"number_of_shards\":\"(.*?)\""); + private static final long CHUNK_THRESHOLD_MS = 3600000; + private static final long MIN_CHUNK_SIZE_MS = 10000L; + + private final HttpRequester httpRequester; + private final ElasticsearchUrlBuilder urlBuilder; + private final ElasticsearchQueryBuilder queryBuilder; + private final int scrollSize; + private final ScrollState scrollState; + private volatile long currentStartTime; + private volatile long currentEndTime; + private volatile long endTime; + private volatile boolean isFirstSearch; + private volatile boolean isCancelled; + + /** + * The interval of each scroll search. Will be null when search is not chunked. + */ + private volatile Long m_Chunk; + + private volatile Logger m_Logger; + + public ElasticsearchDataExtractor(HttpRequester httpRequester, ElasticsearchUrlBuilder urlBuilder, + ElasticsearchQueryBuilder queryBuilder, int scrollSize) { + this.httpRequester = Objects.requireNonNull(httpRequester); + this.urlBuilder = Objects.requireNonNull(urlBuilder); + this.scrollSize = scrollSize; + this.queryBuilder = Objects.requireNonNull(queryBuilder); + scrollState = queryBuilder.isAggregated() ? ScrollState.createAggregated() : ScrollState.createDefault(); + isFirstSearch = true; + } + + @Override + public void newSearch(long startEpochMs, long endEpochMs, Logger logger) throws IOException { + m_Logger = logger; + m_Logger.info("Requesting data from '" + urlBuilder.getBaseUrl() + "' within [" + startEpochMs + ", " + endEpochMs + ")"); + scrollState.reset(); + currentStartTime = startEpochMs; + currentEndTime = endEpochMs; + endTime = endEpochMs; + isCancelled = false; + if (endEpochMs - startEpochMs > CHUNK_THRESHOLD_MS) { + setUpChunkedSearch(); + } + if (isFirstSearch) { + queryBuilder.logQueryInfo(m_Logger); + isFirstSearch = false; + } + } + + private void setUpChunkedSearch() throws IOException { + m_Chunk = null; + String url = urlBuilder.buildSearchSizeOneUrl(); + String response = requestAndGetStringResponse(url, queryBuilder.createDataSummaryQuery(currentStartTime, endTime)); + long totalHits = matchLong(response, TOTAL_HITS_PATTERN); + if (totalHits > 0) { + // Aggregation value may be a double + currentStartTime = (long) matchDouble(response, EARLIEST_TIME_PATTERN); + long latestTime = (long) matchDouble(response, LATEST_TIME_PATTERN); + long dataTimeSpread = latestTime - currentStartTime; + String index = matchString(response, INDEX_PATTERN); + long shards = readNumberOfShards(index); + m_Chunk = Math.max(MIN_CHUNK_SIZE_MS, (shards * scrollSize * dataTimeSpread) / totalHits); + currentEndTime = currentStartTime + m_Chunk; + m_Logger.debug("Chunked search configured: totalHits = " + totalHits + + ", dataTimeSpread = " + dataTimeSpread + " ms, chunk size = " + m_Chunk + + " ms"); + } else { + currentStartTime = endTime; + } + } + + private String requestAndGetStringResponse(String url, String body) throws IOException { + m_Logger.trace("url ={}, body={}", url, body); + HttpResponse response = httpRequester.get(url, body); + if (response.getResponseCode() != HttpResponse.OK_STATUS) { + throw new IOException("Request '" + url + "' failed with status code: " + + response.getResponseCode() + ". Response was:\n" + response.getResponseAsString()); + } + return response.getResponseAsString(); + } + + private static long matchLong(String response, Pattern pattern) throws IOException { + String match = matchString(response, pattern); + try { + return Long.parseLong(match); + } catch (NumberFormatException e) { + throw new IOException("Failed to parse long from pattern '" + pattern + "'. Response was:\n" + response, e); + } + } + + private static double matchDouble(String response, Pattern pattern) throws IOException { + String match = matchString(response, pattern); + try { + return Double.parseDouble(match); + } catch (NumberFormatException e) { + throw new IOException("Failed to parse double from pattern '" + pattern + "'. Response was:\n" + response, e); + } + } + + private static String matchString(String response, Pattern pattern) throws IOException { + Matcher matcher = pattern.matcher(response); + if (!matcher.find()) { + throw new IOException("Failed to parse string from pattern '" + pattern + "'. Response was:\n" + response); + } + return matcher.group(1); + } + + private long readNumberOfShards(String index) throws IOException { + String url = urlBuilder.buildIndexSettingsUrl(index); + String response = requestAndGetStringResponse(url, null); + return matchLong(response, NUMBER_OF_SHARDS_PATTERN); + } + + @Override + public void clear() { + scrollState.reset(); + } + + @Override + public Optional next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + try { + return getNextStream(); + } catch (IOException e) { + m_Logger.error("An error occurred during requesting data from: " + urlBuilder.getBaseUrl(), e); + scrollState.forceComplete(); + throw e; + } + } + + private Optional getNextStream() throws IOException { + while (hasNext()) { + boolean isNewScroll = scrollState.getScrollId() == null || scrollState.isComplete(); + InputStream stream = isNewScroll ? initScroll() : continueScroll(); + stream = scrollState.updateFromStream(stream); + if (scrollState.isComplete()) { + clearScroll(); + advanceTime(); + + // If it was a new scroll it means it returned 0 hits. If we are doing + // a chunked search, we reconfigure the search in order to jump to the next + // time interval where there are data. + if (isNewScroll && hasNext() && !isCancelled && m_Chunk != null) { + setUpChunkedSearch(); + } + } else { + return Optional.of(stream); + } + } + return Optional.empty(); + } + + private void clearScroll() { + if (scrollState.getScrollId() == null) { + return; + } + + String url = urlBuilder.buildClearScrollUrl(); + try { + HttpResponse response = httpRequester.delete(url, String.format(Locale.ROOT, CLEAR_SCROLL_TEMPLATE, scrollState.getScrollId())); + + // This is necessary to ensure the response stream has been consumed entirely. + // Failing to do this can cause a lot of issues with Elasticsearch when + // scheduled jobs are running concurrently. + response.getResponseAsString(); + } catch (IOException e) { + m_Logger.error("An error ocurred during clearing scroll context", e); + } + scrollState.clearScrollId(); + } + + @Override + public boolean hasNext() { + return !scrollState.isComplete() || (!isCancelled && currentStartTime < endTime); + } + + private InputStream initScroll() throws IOException { + String url = buildInitScrollUrl(); + String searchBody = queryBuilder.createSearchBody(currentStartTime, currentEndTime); + m_Logger.trace("About to submit body " + searchBody + " to URL " + url); + HttpResponse response = httpRequester.get(url, searchBody); + if (response.getResponseCode() != HttpResponse.OK_STATUS) { + throw new IOException("Request '" + url + "' failed with status code: " + + response.getResponseCode() + ". Response was:\n" + response.getResponseAsString()); + } + return response.getStream(); + } + + private void advanceTime() { + currentStartTime = currentEndTime; + currentEndTime = m_Chunk == null ? endTime : Math.min(currentStartTime + m_Chunk, endTime); + } + + private String buildInitScrollUrl() throws IOException { + // With aggregations we don't want any hits returned for the raw data, + // just the aggregations + int size = queryBuilder.isAggregated() ? 0 : scrollSize; + return urlBuilder.buildInitScrollUrl(size); + } + + private InputStream continueScroll() throws IOException { + // Aggregations never need a continuation + if (!queryBuilder.isAggregated()) { + String url = urlBuilder.buildContinueScrollUrl(); + HttpResponse response = httpRequester.get(url, scrollState.getScrollId()); + if (response.getResponseCode() == HttpResponse.OK_STATUS) { + return response.getStream(); + } + throw new IOException("Request '" + url + "' with scroll id '" + + scrollState.getScrollId() + "' failed with status code: " + + response.getResponseCode() + ". Response was:\n" + + response.getResponseAsString()); + } + return null; + } + + @Override + public void cancel() { + isCancelled = true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchQueryBuilder.java new file mode 100644 index 00000000000..0366fbeb7bd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchQueryBuilder.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import org.apache.logging.log4j.Logger; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.Objects; + +public class ElasticsearchQueryBuilder { + + /** + * The search body for Elasticsearch version 2.x contains sorting + * based on the time field and a query. The query is composed by + * a bool query with two must clauses, the recommended way to perform an AND query. + * There are 6 placeholders: + *
    + *
  1. sort field + *
  2. user defined query + *
  3. time field + *
  4. start time (String in date_time format) + *
  5. end time (String in date_time format) + *
  6. extra (may be empty or contain aggregations, fields, etc.) + *
+ */ + private static final String SEARCH_BODY_TEMPLATE_2_X = "{" + + "\"sort\": [" + + "{\"%s\": {\"order\": \"asc\"}}" + + "]," + + "\"query\": {" + + "\"bool\": {" + + "\"filter\": [" + + "{%s}," + + "{" + + "\"range\": {" + + "\"%s\": {" + + "\"gte\": \"%s\"," + + "\"lt\": \"%s\"," + + "\"format\": \"date_time\"" + + "}" + + "}" + + "}" + + "]" + + "}" + + "}%s" + + "}"; + + private static final String DATA_SUMMARY_SORT_FIELD = "_doc"; + + /** + * Aggregations in order to retrieve the earliest and latest record times. + * The single placeholder expects the time field. + */ + private static final String DATA_SUMMARY_AGGS_TEMPLATE = "" + + "{" + + "\"earliestTime\":{" + + "\"min\":{\"field\":\"%1$s\"}" + + "}," + + "\"latestTime\":{" + + "\"max\":{\"field\":\"%1$s\"}" + + "}" + + "}"; + + private static final String AGGREGATION_TEMPLATE = ", \"aggs\": %s"; + private static final String SCRIPT_FIELDS_TEMPLATE = ", \"script_fields\": %s"; + private static final String FIELDS_TEMPLATE = "%s, \"_source\": %s"; + + private final String search; + private final String aggregations; + private final String scriptFields; + private final String fields; + private final String timeField; + + public ElasticsearchQueryBuilder(String search, String aggs, String scriptFields, String fields, String timeField) { + this.search = Objects.requireNonNull(search); + aggregations = aggs; + this.scriptFields = scriptFields; + this.fields = fields; + this.timeField = Objects.requireNonNull(timeField); + } + + public String createSearchBody(long start, long end) { + return createSearchBody(start, end, timeField, aggregations); + } + + private String createSearchBody(long start, long end, String sortField, String aggs) { + return String.format(Locale.ROOT, SEARCH_BODY_TEMPLATE_2_X, sortField, search, timeField, formatAsDateTime(start), + formatAsDateTime(end), createResultsFormatSpec(aggs)); + } + + private static String formatAsDateTime(long epochMs) { + Instant instant = Instant.ofEpochMilli(epochMs); + ZonedDateTime dateTime = ZonedDateTime.ofInstant(instant, ZoneOffset.UTC); + return dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT)); + } + + private String createResultsFormatSpec(String aggs) { + return (aggs != null) ? createAggregations(aggs) : ((fields != null) ? createFieldDataFields() : ""); + } + + private String createAggregations(String aggs) { + return String.format(Locale.ROOT, AGGREGATION_TEMPLATE, aggs); + } + + private String createFieldDataFields() { + return String.format(Locale.ROOT, FIELDS_TEMPLATE, createScriptFields(), fields); + } + + private String createScriptFields() { + return (scriptFields != null) ? String.format(Locale.ROOT, SCRIPT_FIELDS_TEMPLATE, scriptFields) : ""; + } + + public String createDataSummaryQuery(long start, long end) { + String aggs = String.format(Locale.ROOT, DATA_SUMMARY_AGGS_TEMPLATE, timeField); + return createSearchBody(start, end, DATA_SUMMARY_SORT_FIELD, aggs); + } + + public void logQueryInfo(Logger logger) { + if (aggregations != null) { + logger.debug("Will use the following Elasticsearch aggregations: " + aggregations); + } else { + if (fields != null) { + logger.debug("Will request only the following field(s) from Elasticsearch: " + String.join(" ", fields)); + } else { + logger.debug("Will retrieve whole _source document from Elasticsearch"); + } + } + } + + public boolean isAggregated() { + return aggregations != null; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchUrlBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchUrlBuilder.java new file mode 100644 index 00000000000..1f7fc794d8c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ElasticsearchUrlBuilder.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ElasticsearchUrlBuilder { + + private static final String SLASH = "/"; + private static final String COMMA = ","; + private static final int SCROLL_CONTEXT_MINUTES = 60; + private static final String INDEX_SETTINGS_END_POINT = "%s/_settings"; + private static final String SEARCH_SIZE_ONE_END_POINT = "_search?size=1"; + private static final String SEARCH_SCROLL_END_POINT = "_search?scroll=" + SCROLL_CONTEXT_MINUTES + "m&size=%d"; + private static final String CONTINUE_SCROLL_END_POINT = "_search/scroll?scroll=" + SCROLL_CONTEXT_MINUTES + "m"; + private static final String CLEAR_SCROLL_END_POINT = "_search/scroll"; + + private final String baseUrl; + private final String indexes; + private final String types; + + private ElasticsearchUrlBuilder(String baseUrl, String indexes, String types) { + this.baseUrl = Objects.requireNonNull(baseUrl); + this.indexes = Objects.requireNonNull(indexes); + this.types = Objects.requireNonNull(types); + } + + public static ElasticsearchUrlBuilder create(String baseUrl, List indexes, List types) { + String sanitisedBaseUrl = baseUrl.endsWith(SLASH) ? baseUrl : baseUrl + SLASH; + String indexesAsString = indexes.stream().collect(Collectors.joining(COMMA)); + String typesAsString = types.stream().collect(Collectors.joining(COMMA)); + return new ElasticsearchUrlBuilder(sanitisedBaseUrl, indexesAsString, typesAsString); + } + + public String buildIndexSettingsUrl(String index) { + return newUrlBuilder().append(String.format(Locale.ROOT, INDEX_SETTINGS_END_POINT, index)).toString(); + } + + public String buildSearchSizeOneUrl() { + return buildUrlWithIndicesAndTypes().append(SEARCH_SIZE_ONE_END_POINT).toString(); + } + + public String buildInitScrollUrl(int scrollSize) { + return buildUrlWithIndicesAndTypes() + .append(String.format(Locale.ROOT, SEARCH_SCROLL_END_POINT, scrollSize)) + .toString(); + } + + public String buildContinueScrollUrl() { + return newUrlBuilder().append(CONTINUE_SCROLL_END_POINT).toString(); + } + + public String buildClearScrollUrl() { + return newUrlBuilder().append(CLEAR_SCROLL_END_POINT).toString(); + } + + private StringBuilder newUrlBuilder() { + return new StringBuilder(baseUrl); + } + + private StringBuilder buildUrlWithIndicesAndTypes() { + StringBuilder urlBuilder = buildUrlWithIndices(); + if (!types.isEmpty()) { + urlBuilder.append(types).append(SLASH); + } + return urlBuilder; + } + + private StringBuilder buildUrlWithIndices() { + return newUrlBuilder().append(indexes).append(SLASH); + } + + public String getBaseUrl() { + return baseUrl; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpDataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpDataExtractorFactory.java new file mode 100644 index 00000000000..3e7008149cd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpDataExtractorFactory.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractor; +import org.elasticsearch.xpack.prelert.job.extraction.DataExtractorFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class HttpDataExtractorFactory implements DataExtractorFactory { + + public HttpDataExtractorFactory() {} + + @Override + public DataExtractor newExtractor(Job job) { + SchedulerConfig schedulerConfig = job.getSchedulerConfig(); + if (schedulerConfig.getDataSource() == SchedulerConfig.DataSource.ELASTICSEARCH) { + return createElasticsearchDataExtractor(job); + } + throw new IllegalArgumentException(); + } + + private DataExtractor createElasticsearchDataExtractor(Job job) { + String timeField = job.getDataDescription().getTimeField(); + SchedulerConfig schedulerConfig = job.getSchedulerConfig(); + ElasticsearchQueryBuilder queryBuilder = new ElasticsearchQueryBuilder( + stringifyElasticsearchQuery(schedulerConfig.getQuery()), + stringifyElasticsearchAggregations(schedulerConfig.getAggregations(), schedulerConfig.getAggs()), + stringifyElasticsearchScriptFields(schedulerConfig.getScriptFields()), + Boolean.TRUE.equals(schedulerConfig.getRetrieveWholeSource()) ? null : writeListAsJson(job.allFields()), + timeField); + HttpRequester httpRequester = new HttpRequester(); + ElasticsearchUrlBuilder urlBuilder = ElasticsearchUrlBuilder + .create(schedulerConfig.getBaseUrl(), schedulerConfig.getIndexes(), schedulerConfig.getTypes()); + return new ElasticsearchDataExtractor(httpRequester, urlBuilder, queryBuilder, schedulerConfig.getScrollSize()); + } + + String stringifyElasticsearchQuery(Map queryMap) { + String queryStr = writeMapAsJson(queryMap); + if (queryStr.startsWith("{") && queryStr.endsWith("}")) { + return queryStr.substring(1, queryStr.length() - 1); + } + return queryStr; + } + + String stringifyElasticsearchAggregations(Map aggregationsMap, Map aggsMap) { + if (aggregationsMap != null) { + return writeMapAsJson(aggregationsMap); + } + if (aggsMap != null) { + return writeMapAsJson(aggsMap); + } + return null; + } + + String stringifyElasticsearchScriptFields(Map scriptFieldsMap) { + if (scriptFieldsMap != null) { + return writeMapAsJson(scriptFieldsMap); + } + return null; + } + + private static String writeMapAsJson(Map map) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.map(map); + return builder.string(); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to convert map to JSON string", e); + } + } + + private static String writeListAsJson(List list) { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startArray("a"); + for (String e : list) { + builder.value(e); + } + builder.endArray(); + builder.endObject(); + return builder.string().replace("{\"a\":", "").replace("}", ""); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to convert map to JSON string", e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpRequester.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpRequester.java new file mode 100644 index 00000000000..4dc7c506694 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpRequester.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + + +/** + * Executes requests from an HTTP or HTTPS URL by sending a request body. + * HTTP or HTTPS is deduced from the supplied URL. + * Invalid certificates are tolerated for HTTPS access, similar to "curl -k". + */ +public class HttpRequester { + + private static final Logger LOGGER = Loggers.getLogger(HttpRequester.class); + + private static final String TLS = "TLS"; + private static final String GET = "GET"; + private static final String DELETE = "DELETE"; + private static final String AUTH_HEADER = "Authorization"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + + private static final SSLSocketFactory TRUSTING_SOCKET_FACTORY; + private static final HostnameVerifier TRUSTING_HOSTNAME_VERIFIER; + + private static final int CONNECT_TIMEOUT_MILLIS = 30000; + private static final int READ_TIMEOUT_MILLIS = 600000; + + static { + SSLSocketFactory trustingSocketFactory = null; + try { + SSLContext sslContext = SSLContext.getInstance(TLS); + sslContext.init(null, new TrustManager[]{ new NoOpTrustManager() }, null); + trustingSocketFactory = sslContext.getSocketFactory(); + } catch (KeyManagementException | NoSuchAlgorithmException e) { + LOGGER.warn("Unable to set up trusting socket factory", e); + } + + TRUSTING_SOCKET_FACTORY = trustingSocketFactory; + TRUSTING_HOSTNAME_VERIFIER = new NoOpHostnameVerifier(); + } + + private final String authHeader; + private final String contentTypeHeader; + + public HttpRequester() { + this(null); + } + + public HttpRequester(String authHeader) { + this(authHeader, null); + } + + public HttpRequester(String authHeader, String contentTypeHeader) { + this.authHeader = authHeader; + this.contentTypeHeader = contentTypeHeader; + } + + public HttpResponse get(String url, String requestBody) throws IOException { + return request(url, requestBody, GET); + } + + public HttpResponse delete(String url, String requestBody) throws IOException { + return request(url, requestBody, DELETE); + } + + private HttpResponse request(String url, String requestBody, String method) throws IOException { + URL urlObject = new URL(url); + HttpURLConnection connection = (HttpURLConnection) urlObject.openConnection(); + connection.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); + connection.setReadTimeout(READ_TIMEOUT_MILLIS); + + // TODO: we could add a config option to allow users who want to + // rigorously enforce valid certificates to do so + if (connection instanceof HttpsURLConnection) { + // This is the equivalent of "curl -k", i.e. tolerate connecting to + // an Elasticsearch with a self-signed certificate or a certificate + // that doesn't match its hostname. + HttpsURLConnection httpsConnection = (HttpsURLConnection)connection; + if (TRUSTING_SOCKET_FACTORY != null) { + httpsConnection.setSSLSocketFactory(TRUSTING_SOCKET_FACTORY); + } + httpsConnection.setHostnameVerifier(TRUSTING_HOSTNAME_VERIFIER); + } + connection.setRequestMethod(method); + if (authHeader != null) { + connection.setRequestProperty(AUTH_HEADER, authHeader); + } + if (contentTypeHeader != null) { + connection.setRequestProperty(CONTENT_TYPE_HEADER, contentTypeHeader); + } + if (requestBody != null) { + connection.setDoOutput(true); + writeRequestBody(requestBody, connection); + } + if (connection.getResponseCode() != HttpResponse.OK_STATUS) { + return new HttpResponse(connection.getErrorStream(), connection.getResponseCode()); + } + return new HttpResponse(connection.getInputStream(), connection.getResponseCode()); + } + + private static void writeRequestBody(String requestBody, HttpURLConnection connection) throws IOException { + DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream()); + dataOutputStream.writeBytes(requestBody); + dataOutputStream.flush(); + dataOutputStream.close(); + } + + /** + * Hostname verifier that ignores hostname discrepancies. + */ + private static final class NoOpHostnameVerifier implements HostnameVerifier { + @Override + public boolean verify(String hostname, SSLSession session) { + return true; + } + } + + /** + * Certificate trust manager that ignores certificate issues. + */ + private static final class NoOpTrustManager implements X509TrustManager { + private static final X509Certificate[] EMPTY_CERTIFICATE_ARRAY = new X509Certificate[0]; + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + // Ignore certificate problems + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + // Ignore certificate problems + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return EMPTY_CERTIFICATE_ARRAY; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpResponse.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpResponse.java new file mode 100644 index 00000000000..15dd187683b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/HttpResponse.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + +/** + * Encapsulates the HTTP response stream and the status code. + * + *

Important note: The stream has to be consumed thoroughly. + * Java is keeping connections alive thus reusing them and any + * streams with dangling data can lead to problems. + */ +class HttpResponse { + + public static final int OK_STATUS = 200; + + private static final String NEW_LINE = "\n"; + + private final InputStream stream; + private final int responseCode; + + public HttpResponse(InputStream responseStream, int responseCode) { + stream = responseStream; + this.responseCode = responseCode; + } + + public int getResponseCode() { + return responseCode; + } + + public InputStream getStream() { + return stream; + } + + public String getResponseAsString() throws IOException { + return getStreamAsString(stream); + } + + public static String getStreamAsString(InputStream stream) throws IOException { + try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + return buffer.lines().collect(Collectors.joining(NEW_LINE)); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ScrollState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ScrollState.java new file mode 100644 index 00000000000..e69175e9f4a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/scheduler/http/ScrollState.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.scheduler.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PushbackInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Holds the state of an Elasticsearch scroll. + */ +class ScrollState { + + private static final Pattern SCROLL_ID_PATTERN = Pattern.compile("\"_scroll_id\":\"(.*?)\""); + private static final Pattern DEFAULT_PEEK_END_PATTERN = Pattern.compile("\"hits\":\\[(.)"); + private static final Pattern AGGREGATED_PEEK_END_PATTERN = Pattern.compile("\"aggregations\":.*?\"buckets\":\\[(.)"); + private static final String CLOSING_SQUARE_BRACKET = "]"; + + /** + * We want to read up until the "hits" or "buckets" array. Scroll IDs can be + * quite long in clusters that have many nodes/shards. The longest reported + * scroll ID is 20708 characters - see + * http://elasticsearch-users.115913.n3.nabble.com/Ridiculously-long-Scroll-id-td4038567.html + *
+ * We set a max byte limit for the stream peeking to 1 MB. + */ + private static final int MAX_PEEK_BYTES = 1024 * 1024; + + /** + * We try to search for the scroll ID and whether the scroll is complete every time + * we have read a chunk of size 32 KB. + */ + private static final int PEEK_CHUNK_SIZE = 32 * 1024; + + private final int peekMaxSize; + private final int peekChunkSize; + private final Pattern peekEndPattern; + private volatile String scrollId; + private volatile boolean isComplete; + + private ScrollState(int peekMaxSize, int peekChunkSize, Pattern scrollCompletePattern) { + this.peekMaxSize = peekMaxSize; + this.peekChunkSize = peekChunkSize; + peekEndPattern = Objects.requireNonNull(scrollCompletePattern); + } + + /** + * Creates a {@code ScrollState} for a search without aggregations + * @return the {@code ScrollState} + */ + public static ScrollState createDefault() { + return new ScrollState(MAX_PEEK_BYTES, PEEK_CHUNK_SIZE, DEFAULT_PEEK_END_PATTERN); + } + + /** + * Creates a {@code ScrollState} for a search with aggregations + * @return the {@code ScrollState} + */ + public static ScrollState createAggregated() { + return new ScrollState(MAX_PEEK_BYTES, PEEK_CHUNK_SIZE, AGGREGATED_PEEK_END_PATTERN); + } + + public final void reset() { + scrollId = null; + isComplete = false; + } + + public void clearScrollId() { + scrollId = null; + } + + public String getScrollId() { + return scrollId; + } + + public boolean isComplete() { + return isComplete; + } + + public void forceComplete() { + isComplete = true; + } + + /** + * Peeks into the stream and updates the scroll ID and whether the scroll is complete. + *

+ * After calling that method the given stream cannot be reused. + * Use the returned stream instead. + *

+ * + * @param stream the stream + * @return a new {@code InputStream} object which should be used instead of the given stream + * for further processing + * @throws IOException if an I/O error occurs while manipulating the stream or the stream + * contains no scroll ID + */ + public InputStream updateFromStream(InputStream stream) throws IOException { + if (stream == null) { + isComplete = true; + return null; + } + + PushbackInputStream pushbackStream = new PushbackInputStream(stream, peekMaxSize); + byte[] buffer = new byte[peekMaxSize]; + int totalBytesRead = 0; + int currentChunkSize = 0; + + int bytesRead = 0; + while (bytesRead >= 0 && totalBytesRead < peekMaxSize) { + bytesRead = stream.read(buffer, totalBytesRead, Math.min(peekChunkSize, peekMaxSize - totalBytesRead)); + if (bytesRead > 0) { + totalBytesRead += bytesRead; + currentChunkSize += bytesRead; + } + + if (bytesRead < 0 || currentChunkSize >= peekChunkSize) { + // We make the assumption here that invalid byte sequences will be read as invalid + // char rather than throwing an exception + String peekString = new String(buffer, 0, totalBytesRead, StandardCharsets.UTF_8); + + if (matchScrollState(peekString)) { + break; + } + currentChunkSize = 0; + } + } + + pushbackStream.unread(buffer, 0, totalBytesRead); + + if (scrollId == null) { + throw new IOException("Field '_scroll_id' was expected but not found in first " + + totalBytesRead + " bytes of response:\n" + + HttpResponse.getStreamAsString(pushbackStream)); + } + return pushbackStream; + } + + /** + * Searches the peek end pattern into the given {@code sequence}. If it is matched, + * it also searches for the scroll ID and updates the state accordingly. + * + * @param sequence the String to search into + * @return {@code true} if the peek end pattern was matched or {@code false} otherwise + */ + private boolean matchScrollState(String sequence) { + Matcher peekEndMatcher = peekEndPattern.matcher(sequence); + if (peekEndMatcher.find()) { + Matcher scrollIdMatcher = SCROLL_ID_PATTERN.matcher(sequence); + if (!scrollIdMatcher.find()) { + scrollId = null; + } else { + scrollId = scrollIdMatcher.group(1); + isComplete = CLOSING_SQUARE_BRACKET.equals(peekEndMatcher.group(1)); + return true; + } + } + return false; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/CountingInputStream.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/CountingInputStream.java new file mode 100644 index 00000000000..1095aaef4f0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/CountingInputStream.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.status; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Simple wrapper around an inputstream instance that counts + * all the bytes passing through it reporting that number to + * the {@link StatusReporter} + *

+ * Overrides the read methods counting the number of bytes read. + */ +public class CountingInputStream extends FilterInputStream { + private StatusReporter statusReporter; + + /** + * @param in + * input stream + * @param statusReporter + * Write number of records, bytes etc. + */ + public CountingInputStream(InputStream in, StatusReporter statusReporter) { + super(in); + this.statusReporter = statusReporter; + } + + /** + * We don't care if the count is one byte out + * because we don't check for the case where read + * returns -1. + *

+ * One of the buffered read(..) methods is more likely to + * be called anyway. + */ + @Override + public int read() throws IOException { + statusReporter.reportBytesRead(1); + + return in.read(); + } + + /** + * Don't bother checking for the special case where + * the stream is closed/finished and read returns -1. + * Our count will be 1 byte out. + */ + @Override + public int read(byte[] b) throws IOException { + int read = in.read(b); + + statusReporter.reportBytesRead(read); + + return read; + } + + /** + * Don't bother checking for the special case where + * the stream is closed/finished and read returns -1. + * Our count will be 1 byte out. + */ + @Override + public int read(byte[] b, int off, int len) throws IOException { + int read = in.read(b, off, len); + + statusReporter.reportBytesRead(read); + return read; + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/StatusReporter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/StatusReporter.java new file mode 100644 index 00000000000..b539cdbe92a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/status/StatusReporter.java @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.status; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.persistence.JobDataCountsPersister; +import org.elasticsearch.xpack.prelert.job.usage.UsageReporter; + +import java.util.Date; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Status reporter for tracking all the good/bad + * records written to the API. Call one of the reportXXX() methods + * to update the records counts if {@linkplain #isReportingBoundary(long)} + * returns true then the count will be logged and the counts persisted + * via the {@linkplain JobDataCountsPersister}. + */ +public class StatusReporter { + /** + * The max percentage of date parse errors allowed before + * an exception is thrown. + */ + public static final Setting ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING = Setting.intSetting("max.percent.date.errors", 25, + Property.NodeScope); + + /** + * The max percentage of out of order records allowed before + * an exception is thrown. + */ + public static final Setting ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING = Setting + .intSetting("max.percent.outoforder.errors", 25, Property.NodeScope); + + private final String jobId; + private final UsageReporter usageReporter; + private final JobDataCountsPersister dataCountsPersister; + private final Logger logger; + + private final DataCounts totalRecordStats; + private volatile DataCounts incrementalRecordStats; + + private long analyzedFieldsPerRecord = 1; + + private long recordCountDivisor = 100; + private long lastRecordCountQuotient = 0; + private long logEvery = 1; + private long logCount = 0; + + private final int acceptablePercentDateParseErrors; + private final int acceptablePercentOutOfOrderErrors; + + public StatusReporter(Environment env, Settings settings, String jobId, UsageReporter usageReporter, + JobDataCountsPersister dataCountsPersister, Logger logger, long bucketSpan) { + this(env, settings, jobId, usageReporter, dataCountsPersister, logger, new DataCounts(jobId), bucketSpan); + } + + public StatusReporter(Environment env, Settings settings, String jobId, DataCounts counts, UsageReporter usageReporter, + JobDataCountsPersister dataCountsPersister, Logger logger, long bucketSpan) { + this(env, settings, jobId, usageReporter, dataCountsPersister, logger, new DataCounts(counts), bucketSpan); + } + + private StatusReporter(Environment env, Settings settings, String jobId, UsageReporter usageReporter, + JobDataCountsPersister dataCountsPersister, Logger logger, DataCounts totalCounts, long bucketSpan) { + this.jobId = jobId; + this.usageReporter = usageReporter; + this.dataCountsPersister = dataCountsPersister; + this.logger = logger; + + totalRecordStats = totalCounts; + incrementalRecordStats = new DataCounts(jobId); + + acceptablePercentDateParseErrors = ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING.get(settings); + acceptablePercentOutOfOrderErrors = ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING.get(settings); + } + + /** + * Increment the number of records written by 1 and increment + * the total number of fields read. + * + * @param inputFieldCount Number of fields in the record. + * Note this is not the number of processed fields (by field etc) + * but the actual number of fields in the record + * @param recordTimeMs The time of the latest record written + * in milliseconds from the epoch. + */ + public void reportRecordWritten(long inputFieldCount, long recordTimeMs) { + usageReporter.addFieldsRecordsRead(inputFieldCount); + + Date recordDate = new Date(recordTimeMs); + + totalRecordStats.incrementInputFieldCount(inputFieldCount); + totalRecordStats.incrementProcessedRecordCount(1); + totalRecordStats.setLatestRecordTimeStamp(recordDate); + + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + incrementalRecordStats.incrementProcessedRecordCount(1); + incrementalRecordStats.setLatestRecordTimeStamp(recordDate); + + if (totalRecordStats.getEarliestRecordTimeStamp() == null) { + totalRecordStats.setEarliestRecordTimeStamp(recordDate); + incrementalRecordStats.setEarliestRecordTimeStamp(recordDate); + } + + // report at various boundaries + long totalRecords = getInputRecordCount(); + if (isReportingBoundary(totalRecords)) { + logStatus(totalRecords); + + dataCountsPersister.persistDataCounts(jobId, runningTotalStats()); + } + } + + /** + * Update only the incremental stats with the newest record time + * + * @param latestRecordTimeMs latest record time as epoch millis + */ + public void reportLatestTimeIncrementalStats(long latestRecordTimeMs) { + incrementalRecordStats.setLatestRecordTimeStamp(new Date(latestRecordTimeMs)); + } + + /** + * Increments the date parse error count + */ + public void reportDateParseError(long inputFieldCount) { + totalRecordStats.incrementInvalidDateCount(1); + totalRecordStats.incrementInputFieldCount(inputFieldCount); + + incrementalRecordStats.incrementInvalidDateCount(1); + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + + usageReporter.addFieldsRecordsRead(inputFieldCount); + } + + /** + * Increments the missing field count + * Records with missing fields are still processed + */ + public void reportMissingField() { + totalRecordStats.incrementMissingFieldCount(1); + incrementalRecordStats.incrementMissingFieldCount(1); + } + + public void reportMissingFields(long missingCount) { + totalRecordStats.incrementMissingFieldCount(missingCount); + incrementalRecordStats.incrementMissingFieldCount(missingCount); + } + + /** + * Add newBytes to the total volume processed + */ + public void reportBytesRead(long newBytes) { + totalRecordStats.incrementInputBytes(newBytes); + incrementalRecordStats.incrementInputBytes(newBytes); + usageReporter.addBytesRead(newBytes); + } + + /** + * Increments the out of order record count + */ + public void reportOutOfOrderRecord(long inputFieldCount) { + totalRecordStats.incrementOutOfOrderTimeStampCount(1); + totalRecordStats.incrementInputFieldCount(inputFieldCount); + + incrementalRecordStats.incrementOutOfOrderTimeStampCount(1); + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + + usageReporter.addFieldsRecordsRead(inputFieldCount); + } + + /** + * Total records seen = records written to the Engine (processed record + * count) + date parse error records count + out of order record count. + *

+ * Records with missing fields are counted as they are still written. + */ + public long getInputRecordCount() { + return totalRecordStats.getInputRecordCount(); + } + + public long getProcessedRecordCount() { + return totalRecordStats.getProcessedRecordCount(); + } + + public long getDateParseErrorsCount() { + return totalRecordStats.getInvalidDateCount(); + } + + public long getMissingFieldErrorCount() { + return totalRecordStats.getMissingFieldCount(); + } + + public long getOutOfOrderRecordCount() { + return totalRecordStats.getOutOfOrderTimeStampCount(); + } + + public long getBytesRead() { + return totalRecordStats.getInputBytes(); + } + + public Date getLatestRecordTime() { + return totalRecordStats.getLatestRecordTimeStamp(); + } + + public long getProcessedFieldCount() { + totalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + return totalRecordStats.getProcessedFieldCount(); + } + + public long getInputFieldCount() { + return totalRecordStats.getInputFieldCount(); + } + + public int getAcceptablePercentDateParseErrors() { + return acceptablePercentDateParseErrors; + } + + public int getAcceptablePercentOutOfOrderErrors() { + return acceptablePercentOutOfOrderErrors; + } + + public void setAnalysedFieldsPerRecord(long value) { + analyzedFieldsPerRecord = value; + } + + public long getAnalysedFieldsPerRecord() { + return analyzedFieldsPerRecord; + } + + + /** + * Report the the status now regardless of whether or + * not we are at a reporting boundary. + */ + public void finishReporting() { + usageReporter.reportUsage(); + dataCountsPersister.persistDataCounts(jobId, runningTotalStats()); + } + + /** + * Log the status. This is done progressively less frequently as the job + * processes more data. Logging every 10000 records when the data rate is + * 40000 per second quickly rolls the logs. + */ + private void logStatus(long totalRecords) { + if (++logCount % logEvery != 0) { + return; + } + + String status = String.format(Locale.ROOT, + "%d records written to autodetect; missingFieldCount=%d, invalidDateCount=%d, outOfOrderCount=%d", + getProcessedRecordCount(), getMissingFieldErrorCount(), getDateParseErrorsCount(), getOutOfOrderRecordCount()); + + logger.info(status); + + int log10TotalRecords = (int) Math.floor(Math.log10(totalRecords)); + // Start reducing the logging rate after 10 million records have been seen + if (log10TotalRecords > 6) { + logEvery = (int) Math.pow(10.0, log10TotalRecords - 6); + logCount = 0; + } + } + + /** + * Don't update status for every update instead update on these + * boundaries + *

    + *
  1. For the first 1000 records update every 100
    2. + *
  2. After 1000 records update every 1000
    3. + *
  3. After 20000 records update every 10000
    4. + *
+ */ + private boolean isReportingBoundary(long totalRecords) { + // after 20,000 records update every 10,000 + int divisor = 10000; + + if (totalRecords <= 1000) { + // for the first 1000 records update every 100 + divisor = 100; + } else if (totalRecords <= 20000) { + // before 20,000 records update every 1000 + divisor = 1000; + } + + if (divisor != recordCountDivisor) { + // have crossed one of the reporting bands + recordCountDivisor = divisor; + lastRecordCountQuotient = totalRecords / divisor; + + return false; + } + + long quotient = totalRecords / divisor; + if (quotient > lastRecordCountQuotient) { + lastRecordCountQuotient = quotient; + return true; + } + + return false; + } + + public void startNewIncrementalCount() { + incrementalRecordStats = new DataCounts(jobId); + } + + public DataCounts incrementalStats() { + incrementalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + return incrementalRecordStats; + } + + public synchronized DataCounts runningTotalStats() { + totalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + DataCounts tempResonse = new DataCounts(totalRecordStats); + return totalRecordStats; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/IntRange.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/IntRange.java new file mode 100644 index 00000000000..7517214c73e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/IntRange.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform; + +import java.util.Objects; + +public class IntRange { + public enum BoundType { + OPEN, CLOSED + } + + public static class Bound { + private final int value; + private final BoundType boundType; + + public Bound(int value, BoundType boundType) { + this.value = value; + this.boundType = Objects.requireNonNull(boundType); + } + } + + private static String PLUS_INFINITY = "+\u221E"; + private static String MINUS_INFINITY = "-\u221E"; + private static char LEFT_BRACKET = '('; + private static char RIGHT_BRACKET = ')'; + private static char LEFT_SQUARE_BRACKET = '['; + private static char RIGHT_SQUARE_BRACKET = ']'; + private static char BOUNDS_SEPARATOR = '\u2025'; + + private final Bound lower; + private final Bound upper; + + private IntRange(Bound lower, Bound upper) { + this.lower = Objects.requireNonNull(lower); + this.upper = Objects.requireNonNull(upper); + } + + public boolean contains(int value) { + int lowerIncludedValue = lower.boundType == BoundType.CLOSED ? lower.value : lower.value + 1; + int upperIncludedValue = upper.boundType == BoundType.CLOSED ? upper.value : upper.value - 1; + return value >= lowerIncludedValue && value <= upperIncludedValue; + } + + public boolean hasLowerBound() { + return lower.value != Integer.MIN_VALUE; + } + + public boolean hasUpperBound() { + return upper.value != Integer.MAX_VALUE; + } + + public int lower() { + return lower.value; + } + + public int upper() { + return upper.value; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(hasLowerBound() && lower.boundType == BoundType.CLOSED ? LEFT_SQUARE_BRACKET : LEFT_BRACKET); + builder.append(hasLowerBound() ? lower.value : MINUS_INFINITY); + builder.append(BOUNDS_SEPARATOR); + builder.append(hasUpperBound() ? upper.value : PLUS_INFINITY); + builder.append(hasUpperBound() && upper.boundType == BoundType.CLOSED ? RIGHT_SQUARE_BRACKET : RIGHT_BRACKET); + return builder.toString(); + } + + public static IntRange singleton(int value) { + return closed(value, value); + } + + public static IntRange closed(int lower, int upper) { + return new IntRange(closedBound(lower), closedBound(upper)); + } + + public static IntRange open(int lower, int upper) { + return new IntRange(openBound(lower), openBound(upper)); + } + + public static IntRange openClosed(int lower, int upper) { + return new IntRange(openBound(lower), closedBound(upper)); + } + + public static IntRange closedOpen(int lower, int upper) { + return new IntRange(closedBound(lower), openBound(upper)); + } + + public static IntRange atLeast(int lower) { + return closed(lower, Integer.MAX_VALUE); + } + + private static Bound openBound(int value) { + return new Bound(value, BoundType.OPEN); + } + + private static Bound closedBound(int value) { + return new Bound(value, BoundType.CLOSED); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfig.java new file mode 100644 index 00000000000..188e9f7eb0d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfig.java @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.prelert.job.condition.Condition; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Represents an API data transform + */ +// NORELEASE: to be replaced by ingest (https://github.com/elastic/prelert-legacy/issues/39) +public class TransformConfig extends ToXContentToBytes implements Writeable { + // Serialisation strings + public static final ParseField TYPE = new ParseField("transform"); + public static final ParseField TRANSFORM = new ParseField("transform"); + public static final ParseField CONDITION = new ParseField("condition"); + public static final ParseField ARGUMENTS = new ParseField("arguments"); + public static final ParseField INPUTS = new ParseField("inputs"); + public static final ParseField OUTPUTS = new ParseField("outputs"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), objects -> new TransformConfig((String) objects[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE); + PARSER.declareStringArray(TransformConfig::setInputs, INPUTS); + PARSER.declareStringArray(TransformConfig::setArguments, ARGUMENTS); + PARSER.declareStringArray(TransformConfig::setOutputs, OUTPUTS); + PARSER.declareObject(TransformConfig::setCondition, Condition.PARSER, CONDITION); + } + + private List inputs; + private String type; + private List arguments; + private List outputs; + private Condition condition; + + // lazily initialized: + private transient TransformType lazyType; + + public TransformConfig(String type) { + this.type = type; + lazyType = TransformType.fromString(type); + try { + outputs = lazyType.defaultOutputNames(); + } catch (IllegalArgumentException e) { + outputs = Collections.emptyList(); + } + arguments = Collections.emptyList(); + } + + @SuppressWarnings("unchecked") + public TransformConfig(StreamInput in) throws IOException { + this(in.readString()); + inputs = (List) in.readGenericValue(); + arguments = (List) in.readGenericValue(); + outputs = (List) in.readGenericValue(); + if (in.readBoolean()) { + condition = new Condition(in); + } + } + + public List getInputs() { + return inputs; + } + + public void setInputs(List fields) { + inputs = fields; + } + + /** + * Transform type see {@linkplain TransformType.Names} + */ + public String getTransform() { + return type; + } + + public List getArguments() { + return arguments; + } + + public void setArguments(List args) { + arguments = args; + } + + public List getOutputs() { + return outputs; + } + + public void setOutputs(List outputs) { + this.outputs = outputs; + } + + /** + * The condition object which may or may not be defined for this + * transform + * + * @return May be null + */ + public Condition getCondition() { + return condition; + } + + public void setCondition(Condition condition) { + this.condition = condition; + } + + /** + * This field shouldn't be serialised as its created dynamically + * Type may be null when the class is constructed. + */ + public TransformType type() { + return lazyType; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(type); + out.writeGenericValue(inputs); + out.writeGenericValue(arguments); + out.writeGenericValue(outputs); + if (condition != null) { + out.writeBoolean(true); + condition.writeTo(out); + } else { + out.writeBoolean(false); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TYPE.getPreferredName(), type); + if (inputs != null) { + builder.field(INPUTS.getPreferredName(), inputs); + } + if (arguments != null) { + builder.field(ARGUMENTS.getPreferredName(), arguments); + } + if (outputs != null) { + builder.field(OUTPUTS.getPreferredName(), outputs); + } + if (condition != null) { + builder.field(CONDITION.getPreferredName(), condition); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(inputs, type, outputs, arguments, condition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + TransformConfig other = (TransformConfig) obj; + + return Objects.equals(this.type, other.type) + && Objects.equals(this.inputs, other.inputs) + && Objects.equals(this.outputs, other.outputs) + && Objects.equals(this.arguments, other.arguments) + && Objects.equals(this.condition, other.condition); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigs.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigs.java new file mode 100644 index 00000000000..e5b4a230406 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformConfigs.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Utility class for methods involving arrays of transforms + */ +public class TransformConfigs extends ToXContentToBytes implements Writeable { + + public static final ParseField TRANSFORMS = new ParseField("transforms"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TRANSFORMS.getPreferredName(), a -> new TransformConfigs((List) a[0])); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), TransformConfig.PARSER, TRANSFORMS); + } + + private List transforms; + + public TransformConfigs(List transforms) { + this.transforms = Objects.requireNonNull(transforms); + } + + public TransformConfigs(StreamInput in) throws IOException { + transforms = in.readList(TransformConfig::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeList(transforms); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TRANSFORMS.getPreferredName(), transforms); + builder.endObject(); + return builder; + } + + public List getTransforms() { + return transforms; + } + + /** + * Set of all the field names that are required as inputs to transforms + */ + public Set inputFieldNames() { + Set fields = new HashSet<>(); + for (TransformConfig t : transforms) { + fields.addAll(t.getInputs()); + } + + return fields; + } + + /** + * Set of all the field names that are outputted (i.e. created) by + * transforms + */ + public Set outputFieldNames() { + Set fields = new HashSet<>(); + for (TransformConfig t : transforms) { + fields.addAll(t.getOutputs()); + } + + return fields; + } + + @Override + public int hashCode() { + return Objects.hash(transforms); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + TransformConfigs other = (TransformConfigs) obj; + return Objects.equals(transforms, other.transforms); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformType.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformType.java new file mode 100644 index 00000000000..a0d458d96c4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/TransformType.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Enum type representing the different transform functions + * with functions for converting between the enum and its + * pretty name i.e. human readable string. + */ +public enum TransformType implements ToXContent, Writeable { + // Name, arity, arguments, outputs, default output names, has condition + DOMAIN_SPLIT(Names.DOMAIN_SPLIT_NAME, IntRange.singleton(1), IntRange.singleton(0), + IntRange.closed(1, 2), Arrays.asList("subDomain", "hrd")), + CONCAT(Names.CONCAT_NAME, IntRange.atLeast(2), IntRange.closed(0, 1), IntRange.singleton(1), + Arrays.asList("concat")), + REGEX_EXTRACT(Names.EXTRACT_NAME, IntRange.singleton(1), IntRange.singleton(1), IntRange.atLeast(1), + Arrays.asList("extract"), false), + REGEX_SPLIT(Names.SPLIT_NAME, IntRange.singleton(1), IntRange.singleton(1), IntRange.atLeast(1), + Arrays.asList("split"), false), + EXCLUDE(Names.EXCLUDE_NAME, IntRange.atLeast(1), IntRange.singleton(0), IntRange.singleton(0), + Arrays.asList(), true), + LOWERCASE(Names.LOWERCASE_NAME, IntRange.singleton(1), IntRange.singleton(0), IntRange.singleton(1), + Arrays.asList("lowercase")), + UPPERCASE(Names.UPPERCASE_NAME, IntRange.singleton(1), IntRange.singleton(0), IntRange.singleton(1), + Arrays.asList("uppercase")), + TRIM(Names.TRIM_NAME, IntRange.singleton(1), IntRange.singleton(0), IntRange.singleton(1), + Arrays.asList("trim")); + + /** + * Transform names. + * + * Enums cannot use static fields in their constructors as the + * enum values are initialised before the statics. + * Having the static fields in nested class means they are created + * when required. + */ + public class Names { + public static final String DOMAIN_SPLIT_NAME = "domain_split"; + public static final String CONCAT_NAME = "concat"; + public static final String EXTRACT_NAME = "extract"; + public static final String SPLIT_NAME = "split"; + public static final String EXCLUDE_NAME = "exclude"; + public static final String LOWERCASE_NAME = "lowercase"; + public static final String UPPERCASE_NAME = "uppercase"; + public static final String TRIM_NAME = "trim"; + + private Names() { + } + } + + private final IntRange arityRange; + private final IntRange argumentsRange; + private final IntRange outputsRange; + private final String prettyName; + private final List defaultOutputNames; + private final boolean hasCondition; + + TransformType(String prettyName, IntRange arityIntRange, + IntRange argumentsIntRange, IntRange outputsIntRange, + List defaultOutputNames) { + this(prettyName, arityIntRange, argumentsIntRange, outputsIntRange, defaultOutputNames, false); + } + + TransformType(String prettyName, IntRange arityIntRange, + IntRange argumentsIntRange, IntRange outputsIntRange, + List defaultOutputNames, boolean hasCondition) { + this.arityRange = arityIntRange; + this.argumentsRange = argumentsIntRange; + this.outputsRange = outputsIntRange; + this.prettyName = prettyName; + this.defaultOutputNames = defaultOutputNames; + this.hasCondition = hasCondition; + } + + /** + * The count IntRange of inputs the transform expects. + */ + public IntRange arityRange() { + return this.arityRange; + } + + /** + * The count IntRange of arguments the transform expects. + */ + public IntRange argumentsRange() { + return this.argumentsRange; + } + + /** + * The count IntRange of outputs the transform expects. + */ + public IntRange outputsRange() { + return this.outputsRange; + } + + public String prettyName() { + return prettyName; + } + + public List defaultOutputNames() { + return defaultOutputNames; + } + + public boolean hasCondition() { + return hasCondition; + } + + @Override + public String toString() { + return prettyName(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.value(prettyName); + return builder; + } + + /** + * Get the enum for the given pretty name. + * The static function valueOf() cannot be overridden so use + * this method instead when converting from the pretty name + * to enum. + */ + public static TransformType fromString(String prettyName) throws IllegalArgumentException { + Set all = EnumSet.allOf(TransformType.class); + + for (TransformType type : all) { + if (type.prettyName().equals(prettyName)) { + return type; + } + } + + throw new IllegalArgumentException("Unknown [transformType]: [" + prettyName + "]"); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/ArgumentVerifier.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/ArgumentVerifier.java new file mode 100644 index 00000000000..702c47acf54 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/ArgumentVerifier.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform.verification; + + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; + +@FunctionalInterface +public interface ArgumentVerifier +{ + void verify(String argument, TransformConfig tc) throws ElasticsearchParseException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexExtractVerifier.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexExtractVerifier.java new file mode 100644 index 00000000000..85b53fb3d91 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexExtractVerifier.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform.verification; + + +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; + +import java.util.List; +import java.util.regex.Pattern; + +public class RegexExtractVerifier implements ArgumentVerifier +{ + @Override + public void verify(String arg, TransformConfig tc) { + new RegexPatternVerifier().verify(arg, tc); + + Pattern pattern = Pattern.compile(arg); + int groupCount = pattern.matcher("").groupCount(); + List outputs = tc.getOutputs(); + int outputCount = outputs == null ? 0 : outputs.size(); + if (groupCount != outputCount) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_EXTRACT_GROUPS_SHOULD_MATCH_OUTPUT_COUNT, + tc.getTransform(), outputCount, arg, groupCount); + throw new IllegalArgumentException(msg); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexPatternVerifier.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexPatternVerifier.java new file mode 100644 index 00000000000..6515726dbe9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/RegexPatternVerifier.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform.verification; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; + +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public class RegexPatternVerifier implements ArgumentVerifier +{ + @Override + public void verify(String arg, TransformConfig tc) throws ElasticsearchParseException { + try { + Pattern.compile(arg); + } catch (PatternSyntaxException e) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_INVALID_ARGUMENT, tc.getTransform(), arg); + throw new IllegalArgumentException(msg); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigVerifier.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigVerifier.java new file mode 100644 index 00000000000..bd5f7efcd1f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigVerifier.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform.verification; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.IntRange; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; + +import java.util.List; + +public final class TransformConfigVerifier +{ + private TransformConfigVerifier() + { + // Hide default constructor + } + + /** + * Checks the transform configuration is valid + *
    + *
  1. Checks there are the correct number of inputs for a given transform + * type and that those inputs are not empty strings
    2. + *
  2. Check the number of arguments is correct for the transform type and + * verify the argument (i.e. is is a valid regex)
    3. + *
  3. Check there is a valid number of ouputs for the transform type and + * those outputs are not empty strings
    4. + *
  4. If the transform has a condition verify it
    5. + *
+ */ + public static boolean verify(TransformConfig tc) throws ElasticsearchParseException { + TransformType type; + try { + type = tc.type(); + } catch (IllegalArgumentException e) { + throw new ElasticsearchParseException(Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_UNKNOWN_TYPE, tc.getTransform())); + } + + checkCondition(tc, type); + checkInputs(tc, type); + checkArguments(tc, type); + checkOutputs(tc, type); + + return true; + } + + private static void checkCondition(TransformConfig tc, TransformType type) { + if (type.hasCondition()) { + if (tc.getCondition() == null) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_CONDITION_REQUIRED, type.prettyName())); + } + } + } + + private static void checkInputs(TransformConfig tc, TransformType type) { + List inputs = tc.getInputs(); + checkValidInputCount(tc, type, inputs); + checkInputsAreNonEmptyStrings(tc, inputs); + } + + private static void checkValidInputCount(TransformConfig tc, TransformType type, List inputs) { + int inputsSize = (inputs == null) ? 0 : inputs.size(); + if (!type.arityRange().contains(inputsSize)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_INVALID_INPUT_COUNT, + tc.getTransform(), rangeAsString(type.arityRange()), inputsSize); + throw new IllegalArgumentException(msg); + } + } + + private static void checkInputsAreNonEmptyStrings(TransformConfig tc, List inputs) { + if (containsEmptyString(inputs)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_INPUTS_CONTAIN_EMPTY_STRING, tc.getTransform()); + throw new IllegalArgumentException(msg); + } + } + + private static boolean containsEmptyString(List strings) { + return strings.stream().anyMatch(s -> s.trim().isEmpty()); + } + + private static void checkArguments(TransformConfig tc, TransformType type) { + checkArgumentsCountValid(tc, type); + checkArgumentsValid(tc, type); + } + + private static void checkArgumentsCountValid(TransformConfig tc, TransformType type) { + List arguments = tc.getArguments(); + int argumentsSize = (arguments == null) ? 0 : arguments.size(); + if (!type.argumentsRange().contains(argumentsSize)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_INVALID_ARGUMENT_COUNT, + tc.getTransform(), rangeAsString(type.argumentsRange()), argumentsSize); + throw new IllegalArgumentException(msg); + } + } + + private static void checkArgumentsValid(TransformConfig tc, TransformType type) { + if (tc.getArguments() != null) { + ArgumentVerifier av = argumentVerifierForType(type); + for (String argument : tc.getArguments()) { + av.verify(argument, tc); + } + } + } + + private static ArgumentVerifier argumentVerifierForType(TransformType type) { + switch (type) { + case REGEX_EXTRACT: + return new RegexExtractVerifier(); + case REGEX_SPLIT: + return new RegexPatternVerifier(); + default: + return (argument, config) -> {}; + } + } + + + private static void checkOutputs(TransformConfig tc, TransformType type) { + List outputs = tc.getOutputs(); + checkValidOutputCount(tc, type, outputs); + checkOutputsAreNonEmptyStrings(tc, outputs); + } + + private static void checkValidOutputCount(TransformConfig tc, TransformType type, List outputs) { + int outputsSize = (outputs == null) ? 0 : outputs.size(); + if (!type.outputsRange().contains(outputsSize)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_INVALID_OUTPUT_COUNT, + tc.getTransform(), rangeAsString(type.outputsRange()), outputsSize); + throw new IllegalArgumentException(msg); + } + } + + private static void checkOutputsAreNonEmptyStrings(TransformConfig tc, List outputs) { + if (containsEmptyString(outputs)) { + String msg = Messages.getMessage( + Messages.JOB_CONFIG_TRANSFORM_OUTPUTS_CONTAIN_EMPTY_STRING, tc.getTransform()); + throw new IllegalArgumentException(msg); + } + } + + private static String rangeAsString(IntRange range) { + if (range.hasLowerBound() && range.hasUpperBound() && range.lower() == range.upper()) { + return String.valueOf(range.lower()); + } + return range.toString(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigsVerifier.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigsVerifier.java new file mode 100644 index 00000000000..3bb6b9dff89 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/job/transform/verification/TransformConfigsVerifier.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.transform.verification; + + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class TransformConfigsVerifier +{ + private TransformConfigsVerifier() + { + } + + /** + * Checks the transform configurations are valid + *
    + *
  1. Call {@linkplain TransformConfigVerifier#verify(TransformConfig)} ()} on each transform
    2. + *
  2. Check all the transform output field names are unique
    3. + *
  3. Check there are no circular dependencies in the transforms
    4. + *
+ */ + public static boolean verify(List transforms) throws ElasticsearchParseException + { + for (TransformConfig tr : transforms) + { + TransformConfigVerifier.verify(tr); + } + + String duplicatedName = outputNamesAreUnique(transforms); + if (duplicatedName != null) + { + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_OUTPUT_NAME_USED_MORE_THAN_ONCE, duplicatedName); + throw new IllegalArgumentException(msg); + } + + // Check for circular dependencies + int index = checkForCircularDependencies(transforms); + if (index >= 0) + { + TransformConfig tc = transforms.get(index); + String msg = Messages.getMessage(Messages.JOB_CONFIG_TRANSFORM_CIRCULAR_DEPENDENCY, tc.type(), tc.getInputs()); + throw new IllegalArgumentException(msg); + } + + return true; + } + + + /** + * return null if all transform ouput names are + * unique or the first duplicate name if there are + * duplications + */ + private static String outputNamesAreUnique(List transforms) + { + Set fields = new HashSet<>(); + for (TransformConfig t : transforms) + { + for (String output : t.getOutputs()) + { + if (fields.contains(output)) + { + return output; + } + fields.add(output); + } + } + + return null; + } + + + + /** + * Find circular dependencies in the list of transforms. + * This might be because a transform's input is its output + * or because of a transitive dependency. + * + * If there is a circular dependency the index of the transform + * in the transforms list at the start of the chain + * is returned else -1 + * + * @return -1 if no circular dependencies else the index of the + * transform at the start of the circular chain + */ + public static int checkForCircularDependencies(List transforms) + { + for (int i=0; i chain = new HashSet(); + chain.add(new Integer(i)); + + TransformConfig tc = transforms.get(i); + if (checkCircularDependenciesRecursive(tc, transforms, chain) == false) + { + return i; + } + } + + return -1; + } + + + private static boolean checkCircularDependenciesRecursive(TransformConfig transform, + List transforms, + Set chain) + { + boolean result = true; + + for (int i=0; i UPDATE_INTERVAL_SETTING = Setting.longSetting("usage.update.interval", 300, 0, Property.NodeScope); + + private final String jobId; + private final Logger logger; + + private long bytesReadSinceLastReport; + private long fieldsReadSinceLastReport; + private long recordsReadSinceLastReport; + + private long lastUpdateTimeMs; + private long updateIntervalMs; + + private final UsagePersister persister; + + public UsageReporter(Settings settings, String jobId, UsagePersister persister, Logger logger) { + bytesReadSinceLastReport = 0; + fieldsReadSinceLastReport = 0; + recordsReadSinceLastReport = 0; + + this.jobId = jobId; + this.persister = persister; + this.logger = logger; + + lastUpdateTimeMs = System.currentTimeMillis(); + + long interval = UPDATE_INTERVAL_SETTING.get(settings); + updateIntervalMs = interval * 1000; + this.logger.info("Setting usage update interval to " + interval + " seconds"); + } + + /** + * Add bytesRead to the running total + */ + public void addBytesRead(long bytesRead) { + bytesReadSinceLastReport += bytesRead; + + long now = System.currentTimeMillis(); + + if (now - lastUpdateTimeMs > updateIntervalMs) { + reportUsage(now); + } + } + + public void addFieldsRecordsRead(long fieldsRead) { + fieldsReadSinceLastReport += fieldsRead; + ++recordsReadSinceLastReport; + } + + public long getBytesReadSinceLastReport() { + return bytesReadSinceLastReport; + } + + public long getFieldsReadSinceLastReport() { + return fieldsReadSinceLastReport; + } + + public long getRecordsReadSinceLastReport() { + return recordsReadSinceLastReport; + } + + + public String getJobId() { + return jobId; + } + + public Logger getLogger() { + return logger; + } + + /** + * Logs total bytes written and calls {@linkplain UsagePersister#persistUsage(String, long, long, long)} + * bytesReadSinceLastReport, fieldsReadSinceLastReport and + * recordsReadSinceLastReport are reset to 0 after this has been called. + */ + public void reportUsage() { + reportUsage(System.currentTimeMillis()); + } + + /** + * See {@linkplain #reportUsage()} + * + * @param epochMs The time now - saved as the last update time + */ + private void reportUsage(long epochMs) { + logger.info(String.format(Locale.ROOT, "An additional %dKiB, %d fields and %d records read by job %s", + bytesReadSinceLastReport >> 10, fieldsReadSinceLastReport, recordsReadSinceLastReport, jobId)); + + try { + persister.persistUsage(jobId, bytesReadSinceLastReport, fieldsReadSinceLastReport, recordsReadSinceLastReport); + } catch (ElasticsearchException e) { + logger.error("Error persisting usage for job " + jobId, e); + } + + lastUpdateTimeMs = epochMs; + + bytesReadSinceLastReport = 0; + fieldsReadSinceLastReport = 0; + recordsReadSinceLastReport = 0; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/lists/ListDocument.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/lists/ListDocument.java new file mode 100644 index 00000000000..3a1705cc27e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/lists/ListDocument.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.lists; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcherSupplier; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +public class ListDocument extends ToXContentToBytes implements Writeable { + public static final ParseField TYPE = new ParseField("list"); + public static final ParseField ID = new ParseField("id"); + public static final ParseField ITEMS = new ParseField("items"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new ListDocument((String) a[0], (List) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), ITEMS); + } + + private final String id; + private final List items; + + public ListDocument(String id, List items) { + this.id = Objects.requireNonNull(id, ID.getPreferredName() + " must not be null"); + this.items = Objects.requireNonNull(items, ITEMS.getPreferredName() + " must not be null"); + } + + public ListDocument(StreamInput in) throws IOException { + id = in.readString(); + items = Arrays.asList(in.readStringArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringArray(items.toArray(new String[items.size()])); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + builder.field(ITEMS.getPreferredName(), items); + builder.endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getItems() { + return new ArrayList<>(items); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof ListDocument)) { + return false; + } + + ListDocument other = (ListDocument) obj; + return id.equals(other.id) && items.equals(other.items); + } + + @Override + public int hashCode() { + return Objects.hash(id, items); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataAction.java new file mode 100644 index 00000000000..66760e1de87 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.data; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PostDataAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestPostDataAction extends BaseRestHandler { + + private static final boolean DEFAULT_IGNORE_DOWNTIME = false; + private static final String DEFAULT_RESET_START = ""; + private static final String DEFAULT_RESET_END = ""; + + private final PostDataAction.TransportAction transportPostDataAction; + + @Inject + public RestPostDataAction(Settings settings, RestController controller, PostDataAction.TransportAction transportPostDataAction) { + super(settings); + this.transportPostDataAction = transportPostDataAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "data/{jobId}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + PostDataAction.Request request = new PostDataAction.Request(restRequest.param(Job.ID.getPreferredName())); + request.setIgnoreDowntime( + restRequest.paramAsBoolean(PostDataAction.Request.IGNORE_DOWNTIME.getPreferredName(), DEFAULT_IGNORE_DOWNTIME)); + request.setResetStart(restRequest.param(PostDataAction.Request.RESET_START.getPreferredName(), DEFAULT_RESET_START)); + request.setResetEnd(restRequest.param(PostDataAction.Request.RESET_END.getPreferredName(), DEFAULT_RESET_END)); + request.setContent(restRequest.content()); + + return channel -> transportPostDataAction.execute(request, new RestStatusToXContentListener<>(channel)); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataCloseAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataCloseAction.java new file mode 100644 index 00000000000..4667aa93d0b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataCloseAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.data; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PostDataCloseAction; + +import java.io.IOException; + +public class RestPostDataCloseAction extends BaseRestHandler { + + private static final ParseField JOB_ID = new ParseField("jobId"); + + private final PostDataCloseAction.TransportAction transportPostDataCloseAction; + + @Inject + public RestPostDataCloseAction(Settings settings, RestController controller, + PostDataCloseAction.TransportAction transportPostDataCloseAction) { + super(settings); + this.transportPostDataCloseAction = transportPostDataCloseAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "data/{jobId}/_close", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + PostDataCloseAction.Request postDataCloseRequest = new PostDataCloseAction.Request(restRequest.param(JOB_ID.getPreferredName())); + + return channel -> transportPostDataCloseAction.execute(postDataCloseRequest, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataFlushAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataFlushAction.java new file mode 100644 index 00000000000..fc93cb8a282 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/data/RestPostDataFlushAction.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.data; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PostDataFlushAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestPostDataFlushAction extends BaseRestHandler { + + private final boolean DEFAULT_CALC_INTERIM = false; + private final String DEFAULT_START = ""; + private final String DEFAULT_END = ""; + private final String DEFAULT_ADVANCE_TIME = ""; + + private final PostDataFlushAction.TransportAction transportPostDataFlushAction; + + @Inject + public RestPostDataFlushAction(Settings settings, RestController controller, + PostDataFlushAction.TransportAction transportPostDataFlushAction) { + super(settings); + this.transportPostDataFlushAction = transportPostDataFlushAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "data/{" + Job.ID.getPreferredName() + "}/_flush", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + final PostDataFlushAction.Request request; + if (RestActions.hasBodyContent(restRequest)) { + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + request = PostDataFlushAction.Request.parseRequest(jobId, parser, () -> parseFieldMatcher); + } else { + request = new PostDataFlushAction.Request(restRequest.param(Job.ID.getPreferredName())); + request.setCalcInterim(restRequest.paramAsBoolean(PostDataFlushAction.Request.CALC_INTERIM.getPreferredName(), + DEFAULT_CALC_INTERIM)); + request.setStart(restRequest.param(PostDataFlushAction.Request.START.getPreferredName(), DEFAULT_START)); + request.setEnd(restRequest.param(PostDataFlushAction.Request.END.getPreferredName(), DEFAULT_END)); + request.setAdvanceTime(restRequest.param(PostDataFlushAction.Request.ADVANCE_TIME.getPreferredName(), DEFAULT_ADVANCE_TIME)); + } + + return channel -> transportPostDataFlushAction.execute(request, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/influencers/RestGetInfluencersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/influencers/RestGetInfluencersAction.java new file mode 100644 index 00000000000..66cd25ddcad --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/influencers/RestGetInfluencersAction.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.influencers; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetInfluencersAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetInfluencersAction extends BaseRestHandler { + + private final GetInfluencersAction.TransportAction transportAction; + + @Inject + public RestGetInfluencersAction(Settings settings, RestController controller, GetInfluencersAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + "}/influencers", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + String start = restRequest.param(GetInfluencersAction.Request.START.getPreferredName()); + String end = restRequest.param(GetInfluencersAction.Request.END.getPreferredName()); + BytesReference bodyBytes = restRequest.content(); + final GetInfluencersAction.Request request; + if (bodyBytes != null && bodyBytes.length() > 0) { + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + request = GetInfluencersAction.Request.parseRequest(jobId, start, end, parser, () -> parseFieldMatcher); + } else { + request = new GetInfluencersAction.Request(jobId, start, end); + request.setIncludeInterim(restRequest.paramAsBoolean(GetInfluencersAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), 0), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), 100))); + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetInfluencersAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); + request.setSort(restRequest.param(GetInfluencersAction.Request.SORT_FIELD.getPreferredName(), + Influencer.ANOMALY_SCORE.getPreferredName())); + request.setDecending(restRequest.paramAsBoolean(GetInfluencersAction.Request.DESCENDING_SORT.getPreferredName(), false)); + } + + return channel -> transportAction.execute(request, new RestToXContentListener(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestDeleteJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestDeleteJobAction.java new file mode 100644 index 00000000000..a4b871dc642 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestDeleteJobAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.DeleteJobAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestDeleteJobAction extends BaseRestHandler { + + private final DeleteJobAction.TransportAction transportDeleteJobAction; + + @Inject + public RestDeleteJobAction(Settings settings, RestController controller, DeleteJobAction.TransportAction transportDeleteJobAction) { + super(settings); + this.transportDeleteJobAction = transportDeleteJobAction; + controller.registerHandler(RestRequest.Method.DELETE, PrelertPlugin.BASE_PATH + "jobs/{" + Job.ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + DeleteJobAction.Request deleteJobRequest = new DeleteJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + return channel -> transportDeleteJobAction.execute(deleteJobRequest, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobAction.java new file mode 100644 index 00000000000..598fd1bb1c9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobAction.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetJobAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; +import java.util.Set; + +public class RestGetJobAction extends BaseRestHandler { + + private final GetJobAction.TransportAction transportGetJobAction; + + @Inject + public RestGetJobAction(Settings settings, RestController controller, GetJobAction.TransportAction transportGetJobAction) { + super(settings); + this.transportGetJobAction = transportGetJobAction; + controller.registerHandler(RestRequest.Method.GET, PrelertPlugin.BASE_PATH + "jobs/{" + Job.ID.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "jobs/{" + Job.ID.getPreferredName() + "}/{metric}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetJobAction.Request getJobRequest = new GetJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + Set stats = Strings.splitStringByCommaToSet(restRequest.param("metric", "config")); + if (stats.contains("_all")) { + getJobRequest.all(); + } + else { + getJobRequest.config(stats.contains("config")); + getJobRequest.dataCounts(stats.contains("data_counts")); + getJobRequest.modelSizeStats(stats.contains("model_size_stats")); + } + + return channel -> transportGetJobAction.execute(getJobRequest, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobsAction.java new file mode 100644 index 00000000000..f5df60522a2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestGetJobsAction.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetJobsAction; +import org.elasticsearch.xpack.prelert.action.GetJobsAction.Response; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetJobsAction extends BaseRestHandler { + private static final int DEFAULT_FROM = 0; + private static final int DEFAULT_SIZE = 100; + + private final GetJobsAction.TransportAction transportGetJobsAction; + + @Inject + public RestGetJobsAction(Settings settings, RestController controller, GetJobsAction.TransportAction transportGetJobsAction) { + super(settings); + this.transportGetJobsAction = transportGetJobsAction; + controller.registerHandler(RestRequest.Method.GET, PrelertPlugin.BASE_PATH + "jobs", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + final GetJobsAction.Request request; + if (RestActions.hasBodyContent(restRequest)) { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + request = GetJobsAction.Request.PARSER.apply(parser, () -> parseFieldMatcher); + } else { + request = new GetJobsAction.Request(); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), DEFAULT_SIZE))); + } + return channel -> transportGetJobsAction.execute(request, new RestToXContentListener(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPauseJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPauseJobAction.java new file mode 100644 index 00000000000..9ca9678bc78 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPauseJobAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PauseJobAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestPauseJobAction extends BaseRestHandler { + + private final PauseJobAction.TransportAction transportPauseJobAction; + + @Inject + public RestPauseJobAction(Settings settings, RestController controller, PauseJobAction.TransportAction transportPauseJobAction) { + super(settings); + this.transportPauseJobAction = transportPauseJobAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "jobs/{" + Job.ID.getPreferredName() + "}/_pause", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + PauseJobAction.Request request = new PauseJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + return channel -> transportPauseJobAction.execute(request, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPutJobsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPutJobsAction.java new file mode 100644 index 00000000000..d9ba9cfa88c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestPutJobsAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PutJobAction; + +import java.io.IOException; + +public class RestPutJobsAction extends BaseRestHandler { + + private final PutJobAction.TransportAction transportPutJobAction; + + @Inject + public RestPutJobsAction(Settings settings, RestController controller, PutJobAction.TransportAction transportPutJobAction) { + super(settings); + this.transportPutJobAction = transportPutJobAction; + controller.registerHandler(RestRequest.Method.PUT, PrelertPlugin.BASE_PATH + "jobs", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = XContentFactory.xContent(restRequest.content()).createParser(restRequest.content()); + PutJobAction.Request putJobRequest = PutJobAction.Request.parseRequest(parser, () -> parseFieldMatcher); + boolean overwrite = restRequest.paramAsBoolean("overwrite", false); + putJobRequest.setOverwrite(overwrite); + return channel -> transportPutJobAction.execute(putJobRequest, new RestToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestResumeJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestResumeJobAction.java new file mode 100644 index 00000000000..5490de4ce3d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/job/RestResumeJobAction.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.ResumeJobAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestResumeJobAction extends BaseRestHandler { + + private final ResumeJobAction.TransportAction transportResumeJobAction; + + @Inject + public RestResumeJobAction(Settings settings, RestController controller, ResumeJobAction.TransportAction transportResumeJobAction) { + super(settings); + this.transportResumeJobAction = transportResumeJobAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "jobs/{" + Job.ID.getPreferredName() + "}/_resume", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + ResumeJobAction.Request request = new ResumeJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + return channel -> transportResumeJobAction.execute(request, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestGetListAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestGetListAction.java new file mode 100644 index 00000000000..bf2ce3e9360 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestGetListAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.list; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetListAction; +import org.elasticsearch.xpack.prelert.lists.ListDocument; + +import java.io.IOException; + +public class RestGetListAction extends BaseRestHandler { + + private final GetListAction.TransportAction transportGetListAction; + + @Inject + public RestGetListAction(Settings settings, RestController controller, GetListAction.TransportAction transportGetListAction) { + super(settings); + this.transportGetListAction = transportGetListAction; + controller.registerHandler(RestRequest.Method.GET, PrelertPlugin.BASE_PATH + "lists/{" + ListDocument.ID.getPreferredName() + "}", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetListAction.Request getListRequest = new GetListAction.Request(restRequest.param(ListDocument.ID.getPreferredName())); + return channel -> transportGetListAction.execute(getListRequest, new RestStatusToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestPutListAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestPutListAction.java new file mode 100644 index 00000000000..b76555961d2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/list/RestPutListAction.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.list; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PutListAction; + +import java.io.IOException; + +public class RestPutListAction extends BaseRestHandler { + + private final PutListAction.TransportAction transportCreateListAction; + + @Inject + public RestPutListAction(Settings settings, RestController controller, PutListAction.TransportAction transportCreateListAction) { + super(settings); + this.transportCreateListAction = transportCreateListAction; + controller.registerHandler(RestRequest.Method.PUT, PrelertPlugin.BASE_PATH + "lists", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + PutListAction.Request putListRequest = PutListAction.Request.parseRequest(parser, () -> parseFieldMatcher); + return channel -> transportCreateListAction.execute(putListRequest, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestDeleteModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestDeleteModelSnapshotAction.java new file mode 100644 index 00000000000..7788af632a9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestDeleteModelSnapshotAction.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.DeleteModelSnapshotAction; + +import java.io.IOException; + +public class RestDeleteModelSnapshotAction extends BaseRestHandler { + + private static final ParseField JOB_ID = new ParseField("jobId"); + private static final ParseField SNAPSHOT_ID = new ParseField("snapshotId"); + + private final DeleteModelSnapshotAction.TransportAction transportAction; + + @Inject + public RestDeleteModelSnapshotAction(Settings settings, RestController controller, + DeleteModelSnapshotAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.DELETE, PrelertPlugin.BASE_PATH + "modelsnapshots/{jobId}/{snapshotId}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + DeleteModelSnapshotAction.Request deleteModelSnapshot = new DeleteModelSnapshotAction.Request( + restRequest.param(JOB_ID.getPreferredName()), restRequest.param(SNAPSHOT_ID.getPreferredName())); + + return channel -> transportAction.execute(deleteModelSnapshot, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestGetModelSnapshotsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestGetModelSnapshotsAction.java new file mode 100644 index 00000000000..0a8afd228b4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestGetModelSnapshotsAction.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetModelSnapshotsAction; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetModelSnapshotsAction extends BaseRestHandler { + + private static final ParseField JOB_ID = new ParseField("jobId"); + private static final ParseField SORT = new ParseField("sort"); + private static final ParseField DESC_ORDER = new ParseField("desc"); + private static final ParseField SIZE = new ParseField("size"); + private static final ParseField FROM = new ParseField("from"); + private static final ParseField START = new ParseField("start"); + private static final ParseField END = new ParseField("end"); + private static final ParseField DESCRIPTION = new ParseField("description"); + + // Even though these are null, setting up the defaults in case + // we want to change them later + private final String DEFAULT_SORT = null; + private final String DEFAULT_START = null; + private final String DEFAULT_END = null; + private final String DEFAULT_DESCRIPTION = null; + private final boolean DEFAULT_DESC_ORDER = true; + private final int DEFAULT_FROM = 0; + private final int DEFAULT_SIZE = 100; + + private final GetModelSnapshotsAction.TransportAction transportGetModelSnapshotsAction; + + @Inject + public RestGetModelSnapshotsAction(Settings settings, RestController controller, + GetModelSnapshotsAction.TransportAction transportGetModelSnapshotsAction) { + super(settings); + this.transportGetModelSnapshotsAction = transportGetModelSnapshotsAction; + controller.registerHandler(RestRequest.Method.GET, PrelertPlugin.BASE_PATH + "modelsnapshots/{jobId}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(JOB_ID.getPreferredName()); + GetModelSnapshotsAction.Request getModelSnapshots; + if (RestActions.hasBodyContent(restRequest)) { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + getModelSnapshots = GetModelSnapshotsAction.Request.parseRequest(jobId, parser, () -> parseFieldMatcher); + } else { + getModelSnapshots = new GetModelSnapshotsAction.Request(jobId); + getModelSnapshots.setSort(restRequest.param(SORT.getPreferredName(), DEFAULT_SORT)); + if (restRequest.hasParam(START.getPreferredName())) { + getModelSnapshots.setStart(restRequest.param(START.getPreferredName(), DEFAULT_START)); + } + if (restRequest.hasParam(END.getPreferredName())) { + getModelSnapshots.setEnd(restRequest.param(END.getPreferredName(), DEFAULT_END)); + } + if (restRequest.hasParam(DESCRIPTION.getPreferredName())) { + getModelSnapshots.setDescriptionString(restRequest.param(DESCRIPTION.getPreferredName(), DEFAULT_DESCRIPTION)); + } + getModelSnapshots.setDescOrder(restRequest.paramAsBoolean(DESC_ORDER.getPreferredName(), DEFAULT_DESC_ORDER)); + getModelSnapshots.setPageParams(new PageParams(restRequest.paramAsInt(FROM.getPreferredName(), DEFAULT_FROM), + restRequest.paramAsInt(SIZE.getPreferredName(), DEFAULT_SIZE))); + } + + return channel -> transportGetModelSnapshotsAction.execute(getModelSnapshots, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestPutModelSnapshotDescriptionAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestPutModelSnapshotDescriptionAction.java new file mode 100644 index 00000000000..2548696e888 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestPutModelSnapshotDescriptionAction.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.PutModelSnapshotDescriptionAction; +import java.io.IOException; + +public class RestPutModelSnapshotDescriptionAction extends BaseRestHandler { + + private static final ParseField JOB_ID = new ParseField("jobId"); + private static final ParseField SNAPSHOT_ID = new ParseField("snapshotId"); + + private final PutModelSnapshotDescriptionAction.TransportAction transportAction; + + @Inject + public RestPutModelSnapshotDescriptionAction(Settings settings, RestController controller, + PutModelSnapshotDescriptionAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + + // NORELEASE: should be a POST action + controller.registerHandler(RestRequest.Method.PUT, PrelertPlugin.BASE_PATH + "modelsnapshots/{jobId}/{snapshotId}/description", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + PutModelSnapshotDescriptionAction.Request getModelSnapshots = PutModelSnapshotDescriptionAction.Request.parseRequest( + restRequest.param(JOB_ID.getPreferredName()), + restRequest.param(SNAPSHOT_ID.getPreferredName()), + parser, () -> parseFieldMatcher + ); + + return channel -> transportAction.execute(getModelSnapshots, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestRevertModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestRevertModelSnapshotAction.java new file mode 100644 index 00000000000..dfbc7623aeb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/modelsnapshots/RestRevertModelSnapshotAction.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.prelert.job.Job; + +import java.io.IOException; + +public class RestRevertModelSnapshotAction extends BaseRestHandler { + + private final RevertModelSnapshotAction.TransportAction transportAction; + + private final String TIME_DEFAULT = null; + private final String SNAPSHOT_ID_DEFAULT = null; + private final String DESCRIPTION_DEFAULT = null; + private final boolean DELETE_INTERVENING_DEFAULT = false; + + @Inject + public RestRevertModelSnapshotAction(Settings settings, RestController controller, + RevertModelSnapshotAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.POST, + PrelertPlugin.BASE_PATH + "modelsnapshots/{" + Job.ID.getPreferredName() + "}/_revert", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + RevertModelSnapshotAction.Request request; + if (RestActions.hasBodyContent(restRequest)) { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + request = RevertModelSnapshotAction.Request.parseRequest(jobId, parser, () -> parseFieldMatcher); + } else { + request = new RevertModelSnapshotAction.Request(jobId); + request.setTime(restRequest.param(RevertModelSnapshotAction.Request.TIME.getPreferredName(), TIME_DEFAULT)); + request.setSnapshotId(restRequest.param(RevertModelSnapshotAction.Request.SNAPSHOT_ID.getPreferredName(), SNAPSHOT_ID_DEFAULT)); + request.setDescription( + restRequest.param(RevertModelSnapshotAction.Request.DESCRIPTION.getPreferredName(), DESCRIPTION_DEFAULT)); + request.setDeleteInterveningResults(restRequest + .paramAsBoolean(RevertModelSnapshotAction.Request.DELETE_INTERVENING.getPreferredName(), DELETE_INTERVENING_DEFAULT)); + } + return channel -> transportAction.execute(request, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetBucketAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetBucketAction.java new file mode 100644 index 00000000000..45a289e6487 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetBucketAction.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetBucketAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetBucketAction extends BaseRestHandler { + + private final GetBucketAction.TransportAction transportAction; + + @Inject + public RestGetBucketAction(Settings settings, RestController controller, GetBucketAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + + "}/bucket/{" + Bucket.TIMESTAMP.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.POST, + PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + + "}/bucket/{" + Bucket.TIMESTAMP.getPreferredName() + "}", this); + + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + "}/bucket", this); + controller.registerHandler(RestRequest.Method.POST, + PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + "}/bucket", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + BytesReference bodyBytes = restRequest.content(); + final GetBucketAction.Request request; + if (bodyBytes != null && bodyBytes.length() > 0) { + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + request = GetBucketAction.Request.parseRequest(jobId, parser, () -> parseFieldMatcher); + } else { + request = new GetBucketAction.Request(jobId); + String timestamp = restRequest.param(GetBucketAction.Request.TIMESTAMP.getPreferredName()); + String start = restRequest.param(GetBucketAction.Request.START.getPreferredName()); + String end = restRequest.param(GetBucketAction.Request.END.getPreferredName()); + + // Single bucket + if (timestamp != null && !timestamp.isEmpty()) { + request.setTimestamp(timestamp); + request.setExpand(restRequest.paramAsBoolean(GetBucketAction.Request.EXPAND.getPreferredName(), false)); + request.setIncludeInterim(restRequest.paramAsBoolean(GetBucketAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + } else if (start != null && !start.isEmpty() && end != null && !end.isEmpty()) { + // Multiple buckets + request.setStart(start); + request.setEnd(end); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), 0), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), 100))); + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetBucketAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); + request.setMaxNormalizedProbability( + Double.parseDouble(restRequest.param( + GetBucketAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); + if (restRequest.hasParam(GetBucketAction.Request.PARTITION_VALUE.getPreferredName())) { + request.setPartitionValue(restRequest.param(GetBucketAction.Request.PARTITION_VALUE.getPreferredName())); + } + } else { + throw new IllegalArgumentException("Either [timestamp] or [start, end] parameters must be set."); + } + + // Common options + request.setExpand(restRequest.paramAsBoolean(GetBucketAction.Request.EXPAND.getPreferredName(), false)); + request.setIncludeInterim(restRequest.paramAsBoolean(GetBucketAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + } + + return channel -> transportAction.execute(request, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetCategoryAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetCategoryAction.java new file mode 100644 index 00000000000..f249d41ce9a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetCategoryAction.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetCategoryDefinitionAction; +import org.elasticsearch.xpack.prelert.action.GetCategoryDefinitionAction.Request; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetCategoryAction extends BaseRestHandler { + + private final GetCategoryDefinitionAction.TransportAction transportAction; + + @Inject + public RestGetCategoryAction(Settings settings, RestController controller, + GetCategoryDefinitionAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "results/{jobId}/categorydefinition/{categoryId}", this); + controller.registerHandler(RestRequest.Method.GET, + PrelertPlugin.BASE_PATH + "results/{jobId}/categorydefinition", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + Request request = new Request(restRequest.param(Job.ID.getPreferredName())); + + String categoryId = restRequest.param(Request.CATEGORY_ID.getPreferredName()); + if (categoryId != null && !categoryId.isEmpty()) { + request.setCategoryId(categoryId); + } else { + PageParams pageParams = new PageParams( + restRequest.paramAsInt(Request.FROM.getPreferredName(), 0), + restRequest.paramAsInt(Request.SIZE.getPreferredName(), 100) + ); + request.setPageParams(pageParams); + } + + return channel -> transportAction.execute(request, new RestToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetRecordsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetRecordsAction.java new file mode 100644 index 00000000000..554fbcdff6f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/results/RestGetRecordsAction.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.GetRecordsAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.PageParams; + +import java.io.IOException; + +public class RestGetRecordsAction extends BaseRestHandler { + + private final GetRecordsAction.TransportAction transportAction; + + @Inject + public RestGetRecordsAction(Settings settings, RestController controller, GetRecordsAction.TransportAction transportAction) { + super(settings); + this.transportAction = transportAction; + controller.registerHandler(RestRequest.Method.GET, PrelertPlugin.BASE_PATH + "results/{" + Job.ID.getPreferredName() + "}/records", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetRecordsAction.Request request = new GetRecordsAction.Request(restRequest.param(Job.ID.getPreferredName()), + restRequest.param(GetRecordsAction.Request.START.getPreferredName()), + restRequest.param(GetRecordsAction.Request.END.getPreferredName())); + request.setIncludeInterim(restRequest.paramAsBoolean(GetRecordsAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), 0), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), 100))); + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetRecordsAction.Request.ANOMALY_SCORE_FILTER.getPreferredName(), "0.0"))); + request.setSort(restRequest.param(GetRecordsAction.Request.SORT.getPreferredName(), + AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName())); + request.setDecending(restRequest.paramAsBoolean(GetRecordsAction.Request.DESCENDING.getPreferredName(), false)); + request.setMaxNormalizedProbability( + Double.parseDouble(restRequest.param(GetRecordsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); + String partitionValue = restRequest.param(GetRecordsAction.Request.PARTITION_VALUE.getPreferredName()); + if (partitionValue != null) { + request.setPartitionValue(partitionValue); + } + + return channel -> transportAction.execute(request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStartJobSchedulerAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStartJobSchedulerAction.java new file mode 100644 index 00000000000..a667250b2b7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStartJobSchedulerAction.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.schedulers; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.StartJobSchedulerAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; + +public class RestStartJobSchedulerAction extends BaseRestHandler { + + private static final String DEFAULT_START = "0"; + + private final StartJobSchedulerAction.TransportAction transportJobSchedulerAction; + + @Inject + public RestStartJobSchedulerAction(Settings settings, RestController controller, + StartJobSchedulerAction.TransportAction transportJobSchedulerAction) { + super(settings); + this.transportJobSchedulerAction = transportJobSchedulerAction; + controller.registerHandler(RestRequest.Method.POST, + PrelertPlugin.BASE_PATH + "schedulers/{" + Job.ID.getPreferredName() + "}/_start", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + StartJobSchedulerAction.Request jobSchedulerRequest; + if (RestActions.hasBodyContent(restRequest)) { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + jobSchedulerRequest = StartJobSchedulerAction.Request.parseRequest(jobId, parser, () -> parseFieldMatcher); + } else { + long startTimeMillis = parseDateOrThrow(restRequest.param(SchedulerState.START_TIME_MILLIS.getPreferredName(), DEFAULT_START), + SchedulerState.START_TIME_MILLIS.getPreferredName()); + Long endTimeMillis = null; + if (restRequest.hasParam(SchedulerState.END_TIME_MILLIS.getPreferredName())) { + endTimeMillis = parseDateOrThrow(restRequest.param(SchedulerState.END_TIME_MILLIS.getPreferredName()), + SchedulerState.END_TIME_MILLIS.getPreferredName()); + } + SchedulerState schedulerState = new SchedulerState(JobSchedulerStatus.STARTING, startTimeMillis, endTimeMillis); + jobSchedulerRequest = new StartJobSchedulerAction.Request(jobId, schedulerState); + } + return channel -> transportJobSchedulerAction.execute(jobSchedulerRequest, new AcknowledgedRestListener<>(channel)); + } + + static long parseDateOrThrow(String date, String paramName) { + try { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseMillis(date); + } catch (IllegalArgumentException e) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg, e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStopJobSchedulerAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStopJobSchedulerAction.java new file mode 100644 index 00000000000..eb86c45b7e6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/schedulers/RestStopJobSchedulerAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.schedulers; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.StopJobSchedulerAction; + +import java.io.IOException; + +public class RestStopJobSchedulerAction extends BaseRestHandler { + + private static final ParseField JOB_ID = new ParseField("jobId"); + + private final StopJobSchedulerAction.TransportAction transportJobSchedulerAction; + + @Inject + public RestStopJobSchedulerAction(Settings settings, RestController controller, + StopJobSchedulerAction.TransportAction transportJobSchedulerAction) { + super(settings); + this.transportJobSchedulerAction = transportJobSchedulerAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "schedulers/{jobId}/_stop", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + StopJobSchedulerAction.Request jobSchedulerRequest = new StopJobSchedulerAction.Request( + restRequest.param(JOB_ID.getPreferredName())); + return channel -> transportJobSchedulerAction.execute(jobSchedulerRequest, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateDetectorAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateDetectorAction.java new file mode 100644 index 00000000000..09f915f4d2f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateDetectorAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.validate; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.ValidateDetectorAction; + +import java.io.IOException; + +public class RestValidateDetectorAction extends BaseRestHandler { + + private ValidateDetectorAction.TransportAction transportValidateAction; + + @Inject + public RestValidateDetectorAction(Settings settings, RestController controller, + ValidateDetectorAction.TransportAction transportValidateAction) { + super(settings); + this.transportValidateAction = transportValidateAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "_validate/detector", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + ValidateDetectorAction.Request validateDetectorRequest = ValidateDetectorAction.Request.parseRequest(parser, + () -> parseFieldMatcher); + return channel -> transportValidateAction.execute(validateDetectorRequest, + new AcknowledgedRestListener(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformAction.java new file mode 100644 index 00000000000..3e836262aca --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.validate; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.ValidateTransformAction; + +import java.io.IOException; + +public class RestValidateTransformAction extends BaseRestHandler { + + private ValidateTransformAction.TransportAction transportValidateAction; + + @Inject + public RestValidateTransformAction(Settings settings, RestController controller, + ValidateTransformAction.TransportAction transportValidateAction) { + super(settings); + this.transportValidateAction = transportValidateAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "_validate/transform", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + ValidateTransformAction.Request validateDetectorRequest = ValidateTransformAction.Request.parseRequest(parser, + () -> parseFieldMatcher); + return channel -> transportValidateAction.execute(validateDetectorRequest, + new AcknowledgedRestListener(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformsAction.java new file mode 100644 index 00000000000..893d419b232 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/rest/validate/RestValidateTransformsAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.rest.validate; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.action.ValidateTransformsAction; + +import java.io.IOException; + +public class RestValidateTransformsAction extends BaseRestHandler { + + private ValidateTransformsAction.TransportAction transportValidateAction; + + @Inject + public RestValidateTransformsAction(Settings settings, RestController controller, + ValidateTransformsAction.TransportAction transportValidateAction) { + super(settings); + this.transportValidateAction = transportValidateAction; + controller.registerHandler(RestRequest.Method.POST, PrelertPlugin.BASE_PATH + "_validate/transforms", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + BytesReference bodyBytes = RestActions.getRestContent(restRequest); + XContentParser parser = XContentFactory.xContent(bodyBytes).createParser(bodyBytes); + ValidateTransformsAction.Request validateDetectorRequest = ValidateTransformsAction.Request.PARSER.apply(parser, + () -> parseFieldMatcher); + return channel -> transportValidateAction.execute(validateDetectorRequest, + new AcknowledgedRestListener(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Concat.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Concat.java new file mode 100644 index 00000000000..b1690bf8c41 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Concat.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.StringJoiner; + +import org.apache.logging.log4j.Logger; + + +/** + * Concatenate input fields + */ +public class Concat extends Transform { + private static final String EMPTY_STRING = ""; + + private final String delimiter; + + public Concat(List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + delimiter = EMPTY_STRING; + } + + public Concat(String join, List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + delimiter = join; + } + + public String getDelimiter() { + return delimiter; + } + + /** + * Concat has only 1 output field + */ + @Override + public TransformResult transform(String[][] readWriteArea) + throws TransformException { + if (writeIndexes.isEmpty()) { + return TransformResult.FAIL; + } + + TransformIndex writeIndex = writeIndexes.get(0); + + StringJoiner joiner = new StringJoiner(delimiter); + for (TransformIndex i : readIndexes) { + joiner.add(readWriteArea[i.array][i.index]); + } + readWriteArea[writeIndex.array][writeIndex.index] = joiner.toString(); + + return TransformResult.OK; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/DependencySorter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/DependencySorter.java new file mode 100644 index 00000000000..654df4c2418 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/DependencySorter.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; + +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; + +/** + * Transform inputs and outputs can be chained together this class provides + * methods for finding the chains of dependencies is a list of transforms. The + * results are ordered list of transforms that should be executed in order + * starting at index 0 + */ +public final class DependencySorter { + /** + * Hide public constructor + */ + private DependencySorter() { + + } + + /** + * For the input field get the chain of transforms that must be executed to + * get that field. The returned list is ordered so that the ones at the end + * of the list are dependent on those at the beginning. + *

+ * Note if there is a circular dependency in the list of transforms this + * will cause a stack overflow. Check with + * {@linkplain org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigsVerifier#checkForCircularDependencies(List)} + * first. + * + * @return List of transforms ordered by dependencies + */ + public static List findDependencies(String input, List transforms) { + return findDependencies(Arrays.asList(input), transforms); + } + + /** + * For the list of input fields get the chain of transforms that must be + * executed to get those fields. The returned list is ordered so that the + * ones at the end of the list are dependent on those at the beginning + *

+ * Note if there is a circular dependency in the list of transforms this + * will cause a stack overflow. Check with + * {@linkplain org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigsVerifier#checkForCircularDependencies(List)} + * first. + * + * @return List of transforms ordered by dependencies + */ + public static List findDependencies(List inputs, List transforms) { + List dependencies = new LinkedList<>(); + + ListIterator itr = transforms.listIterator(); + while (itr.hasNext()) { + TransformConfig tc = itr.next(); + for (String input : inputs) { + if (tc.getOutputs().contains(input)) { + findDependenciesRecursive(tc, transforms, dependencies); + } + } + + } + return dependencies; + } + + /** + * Recursively find the transform dependencies and add them to the + * dependency list + * + */ + private static void findDependenciesRecursive(TransformConfig transform, List transforms, + List dependencies) { + int index = dependencies.indexOf(transform); + if (index >= 0) { + return; + } + + ListIterator itr = transforms.listIterator(); + while (itr.hasNext()) { + TransformConfig tc = itr.next(); + + for (String input : transform.getInputs()) { + if (tc.getOutputs().contains(input)) { + findDependenciesRecursive(tc, transforms, dependencies); + } + } + } + + dependencies.add(transform); + } + + /** + * Return an ordered list of transforms (the same size as the input list) + * that sorted in terms of dependencies. + *

+ * Note if there is a circular dependency in the list of transforms this + * will cause a stack overflow. Check with + * {@linkplain org.elasticsearch.xpack.prelert.job.transform.verification.TransformConfigsVerifier#checkForCircularDependencies(List)} + * first. + * + * @return List of transforms ordered by dependencies + */ + public static List sortByDependency(List transforms) { + List orderedDependencies = new LinkedList<>(); + List transformsCopy = new LinkedList<>(transforms); + + transformsCopy = orderDependenciesRecursive(transformsCopy, orderedDependencies); + while (transformsCopy.isEmpty() == false) { + transformsCopy = orderDependenciesRecursive(transformsCopy, orderedDependencies); + } + + return orderedDependencies; + } + + /** + * Find the dependencies of the head of the transforms list + * adding them to the dependencies list. The returned list is a + * copy of the input transforms with the dependent transforms + * (i.e. those that have been ordered and add to dependencies) + * removed. + *

+ * In the case where the input transforms list contains + * multiple chains of dependencies this function should be called multiple + * times using its return value as the input transforms + * parameter + *

+ * To avoid concurrent modification of the transforms list a new copy is + * made for each recursive call and a new modified list returned + * + * @param dependencies + * Transforms are added to this list + * @return As transforms are moved from transforms to + * dependencies this list is a new copy of the + * transforms input with the moved transforms removed. + */ + private static List orderDependenciesRecursive(List transforms, List dependencies) { + if (transforms.isEmpty()) { + return transforms; + } + + ListIterator itr = transforms.listIterator(); + TransformConfig transform = itr.next(); + itr.remove(); + + int index = dependencies.indexOf(transform); + if (index >= 0) { + return transforms; + } + + while (itr.hasNext()) { + TransformConfig tc = itr.next(); + + for (String input : transform.getInputs()) { + if (tc.getOutputs().contains(input)) { + transforms = orderDependenciesRecursive(new LinkedList(transforms), dependencies); + + itr = transforms.listIterator(); + } + } + } + + dependencies.add(transform); + return transforms; + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilter.java new file mode 100644 index 00000000000..883034273d9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilter.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.condition.Condition; + + +/** + * Abstract base class for exclude filters + */ +public abstract class ExcludeFilter extends Transform { + private final Condition condition; + + /** + * The condition should have been verified by now and it must have a + * valid value & operator + */ + public ExcludeFilter(Condition condition, List readIndexes, + List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + this.condition = condition; + } + + public Condition getCondition() { + return condition; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterNumeric.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterNumeric.java new file mode 100644 index 00000000000..d9f9c216bfb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterNumeric.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; + + +/** + * Parses a numeric value from a field and compares it against a hard + * value using a certain {@link Operator} + */ +public class ExcludeFilterNumeric extends ExcludeFilter { + private final double filterValue; + + /** + * The condition should have been verified by now but if they are not valid + * then the default of < (less than) and filter of 0.0 are used meaning + * that no values are excluded. + */ + public ExcludeFilterNumeric(Condition condition, List readIndexes, + List writeIndexes, Logger logger) { + super(condition, readIndexes, writeIndexes, logger); + + filterValue = parseFilterValue(getCondition().getValue()); + } + + /** + * If no condition then the default is < (less than) and filter value of + * 0.0 are used meaning that only -ve values are excluded. + */ + public ExcludeFilterNumeric(List readIndexes, + List writeIndexes, Logger logger) { + super(new Condition(Operator.LT, "0.0"), + readIndexes, writeIndexes, logger); + filterValue = 0.0; + } + + private double parseFilterValue(String fieldValue) { + double result = 0.0; + try { + result = Double.parseDouble(fieldValue); + } catch (NumberFormatException e) { + logger.warn("Exclude transform cannot parse a number from field '" + fieldValue + "'. Using default 0.0"); + } + + return result; + } + + /** + * Returns {@link TransformResult#EXCLUDE} if the value should be excluded + */ + @Override + public TransformResult transform(String[][] readWriteArea) + throws TransformException { + TransformResult result = TransformResult.OK; + for (TransformIndex readIndex : readIndexes) { + String field = readWriteArea[readIndex.array][readIndex.index]; + + try { + double value = Double.parseDouble(field); + + if (getCondition().getOperator().test(value, filterValue)) { + result = TransformResult.EXCLUDE; + break; + } + } catch (NumberFormatException e) { + + } + } + + return result; + } + + public double filterValue() { + return filterValue; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterRegex.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterRegex.java new file mode 100644 index 00000000000..f02ba044300 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/ExcludeFilterRegex.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.condition.Condition; + +/** + * Matches a field against a regex + */ +public class ExcludeFilterRegex extends ExcludeFilter { + private final Pattern pattern; + + public ExcludeFilterRegex(Condition condition, List readIndexes, + List writeIndexes, Logger logger) { + super(condition, readIndexes, writeIndexes, logger); + + pattern = Pattern.compile(getCondition().getValue()); + } + + /** + * Returns {@link TransformResult#EXCLUDE} if the record matches the regex + */ + @Override + public TransformResult transform(String[][] readWriteArea) + throws TransformException { + TransformResult result = TransformResult.OK; + for (TransformIndex readIndex : readIndexes) { + String field = readWriteArea[readIndex.array][readIndex.index]; + Matcher match = pattern.matcher(field); + + if (match.matches()) { + result = TransformResult.EXCLUDE; + break; + } + } + + return result; + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/HighestRegisteredDomain.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/HighestRegisteredDomain.java new file mode 100644 index 00000000000..e0af3cbbb66 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/HighestRegisteredDomain.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; + +import org.apache.logging.log4j.Logger; + + +/** + * Split a hostname into Highest Registered Domain and sub domain. + * TODO Reimplement porting the code from C++ + */ +public class HighestRegisteredDomain extends Transform { + /** + * Immutable class for the domain split results + */ + public static class DomainSplit { + private String subDomain; + private String highestRegisteredDomain; + + private DomainSplit(String subDomain, String highestRegisteredDomain) { + this.subDomain = subDomain; + this.highestRegisteredDomain = highestRegisteredDomain; + } + + public String getSubDomain() { + return subDomain; + } + + public String getHighestRegisteredDomain() { + return highestRegisteredDomain; + } + } + + public HighestRegisteredDomain(List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + } + + @Override + public TransformResult transform(String[][] readWriteArea) { + return TransformResult.FAIL; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexExtract.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexExtract.java new file mode 100644 index 00000000000..25806294871 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexExtract.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.Logger; + +public class RegexExtract extends Transform { + private final Pattern pattern; + + public RegexExtract(String regex, List readIndexes, + List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + + pattern = Pattern.compile(regex); + } + + @Override + public TransformResult transform(String[][] readWriteArea) + throws TransformException { + TransformIndex readIndex = readIndexes.get(0); + String field = readWriteArea[readIndex.array][readIndex.index]; + + Matcher match = pattern.matcher(field); + + if (match.find()) { + int maxMatches = Math.min(writeIndexes.size(), match.groupCount()); + for (int i = 0; i < maxMatches; i++) { + TransformIndex index = writeIndexes.get(i); + readWriteArea[index.array][index.index] = match.group(i + 1); + } + + return TransformResult.OK; + } else { + logger.warn("Transform 'extract' failed to match field: " + field); + } + + return TransformResult.FAIL; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexSplit.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexSplit.java new file mode 100644 index 00000000000..99084c083b7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/RegexSplit.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.Logger; + + +public class RegexSplit extends Transform { + private final Pattern pattern; + + public RegexSplit(String regex, List readIndexes, + List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + + pattern = Pattern.compile(regex); + } + + @Override + public TransformResult transform(String[][] readWriteArea) + throws TransformException { + TransformIndex readIndex = readIndexes.get(0); + String field = readWriteArea[readIndex.array][readIndex.index]; + + String[] split = pattern.split(field); + + warnIfOutputCountIsNotMatched(split.length, field); + + int count = Math.min(split.length, writeIndexes.size()); + for (int i = 0; i < count; i++) { + TransformIndex index = writeIndexes.get(i); + readWriteArea[index.array][index.index] = split[i]; + } + + return TransformResult.OK; + } + + private void warnIfOutputCountIsNotMatched(int splitCount, String field) { + if (splitCount != writeIndexes.size()) { + String warning = String.format(Locale.ROOT, + "Transform 'split' has %d output(s) but splitting value '%s' resulted to %d part(s)", + writeIndexes.size(), field, splitCount); + logger.warn(warning); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/StringTransform.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/StringTransform.java new file mode 100644 index 00000000000..ff8d0696ba2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/StringTransform.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.Locale; +import java.util.function.Function; + +import org.apache.logging.log4j.Logger; + +public class StringTransform extends Transform { + private final Function convertFunction; + + private StringTransform(Function convertFunction, + List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + this.convertFunction = convertFunction; + if (readIndexes.size() != 1 || writeIndexes.size() != 1) { + throw new IllegalArgumentException(); + } + } + + @Override + public TransformResult transform(String[][] readWriteArea) throws TransformException { + TransformIndex readIndex = readIndexes.get(0); + TransformIndex writeIndex = writeIndexes.get(0); + String input = readWriteArea[readIndex.array][readIndex.index]; + readWriteArea[writeIndex.array][writeIndex.index] = convertFunction.apply(input); + return TransformResult.OK; + } + + public static StringTransform createLowerCase(List readIndexes, + List writeIndexes, Logger logger) { + return new StringTransform(s -> s.toLowerCase(Locale.ROOT), readIndexes, writeIndexes, logger); + } + + public static StringTransform createUpperCase(List readIndexes, + List writeIndexes, Logger logger) { + return new StringTransform(s -> s.toUpperCase(Locale.ROOT), readIndexes, writeIndexes, logger); + } + + public static StringTransform createTrim(List readIndexes, + List writeIndexes, Logger logger) { + return new StringTransform(s -> s.trim(), readIndexes, writeIndexes, logger); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Transform.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Transform.java new file mode 100644 index 00000000000..7f80859793e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/Transform.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.List; +import java.util.Objects; + +import org.apache.logging.log4j.Logger; + +/** + * Abstract transform class. + * Instances are created with maps telling it which field(s) + * to read from in the input array and where to write to. + * The read/write area is passed in the {@linkplain #transform(String[][])} + * function. + *

+ * Some transforms may fail and we will continue processing for + * others a failure is terminal meaning the record should not be + * processed further + */ +public abstract class Transform { + /** + * OK means the transform was successful, + * FAIL means the transform failed but it's ok to continue processing + * EXCLUDE means the no further processing should take place and the record discarded + */ + public enum TransformResult { + OK, FAIL, EXCLUDE + } + + public static class TransformIndex { + public final int array; + public final int index; + + public TransformIndex(int a, int b) { + this.array = a; + this.index = b; + } + + @Override + public int hashCode() { + return Objects.hash(array, index); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TransformIndex other = (TransformIndex) obj; + return Objects.equals(this.array, other.array) + && Objects.equals(this.index, other.index); + } + } + + protected final Logger logger; + protected final List readIndexes; + protected final List writeIndexes; + + /** + * @param readIndexes Read inputs from these indexes + * @param writeIndexes Outputs are written to these indexes + * @param logger Transform results go into these indexes + */ + public Transform(List readIndexes, List writeIndexes, Logger logger) { + this.logger = logger; + + this.readIndexes = readIndexes; + this.writeIndexes = writeIndexes; + } + + /** + * The indexes for the inputs + */ + public final List getReadIndexes() { + return readIndexes; + } + + /** + * The write output indexes + */ + public final List getWriteIndexes() { + return writeIndexes; + } + + /** + * Transform function. + * The read write array of arrays area typically contains an input array, + * scratch area array and the output array. The scratch area is used in the + * case where the transform is chained so reads/writes to an intermediate area + */ + public abstract TransformResult transform(String[][] readWriteArea) + throws TransformException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformException.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformException.java new file mode 100644 index 00000000000..aa6ca5e9be1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformException.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +public abstract class TransformException extends Exception { + + public TransformException(String message) { + super(message); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformFactory.java new file mode 100644 index 00000000000..fdebffd1eda --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/TransformFactory.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; +import org.elasticsearch.xpack.prelert.transforms.Transform.TransformIndex; + +/** + * Create transforms from the configuration object. + * Transforms need to know where to read strings from and where + * write the output to hence input and output maps required by the + * create method. + */ +public class TransformFactory { + public static final int INPUT_ARRAY_INDEX = 0; + public static final int SCRATCH_ARRAY_INDEX = 1; + public static final int OUTPUT_ARRAY_INDEX = 2; + + public Transform create(TransformConfig transformConfig, + Map inputIndexesMap, + Map scratchAreaIndexesMap, + Map outputIndexesMap, + Logger logger) { + int[] input = new int[transformConfig.getInputs().size()]; + fillIndexArray(transformConfig.getInputs(), inputIndexesMap, input); + + List readIndexes = new ArrayList<>(); + for (String field : transformConfig.getInputs()) { + Integer index = inputIndexesMap.get(field); + if (index != null) { + readIndexes.add(new TransformIndex(INPUT_ARRAY_INDEX, index)); + } else { + index = scratchAreaIndexesMap.get(field); + if (index != null) { + readIndexes.add(new TransformIndex(SCRATCH_ARRAY_INDEX, index)); + } else if (outputIndexesMap.containsKey(field)) // also check the outputs array for this input + { + index = outputIndexesMap.get(field); + readIndexes.add(new TransformIndex(SCRATCH_ARRAY_INDEX, index)); + } else { + throw new IllegalStateException("Transform input '" + field + + "' cannot be found"); + } + } + } + + List writeIndexes = new ArrayList<>(); + for (String field : transformConfig.getOutputs()) { + Integer index = outputIndexesMap.get(field); + if (index != null) { + writeIndexes.add(new TransformIndex(OUTPUT_ARRAY_INDEX, index)); + } else { + index = scratchAreaIndexesMap.get(field); + if (index != null) { + writeIndexes.add(new TransformIndex(SCRATCH_ARRAY_INDEX, index)); + } + } + } + + TransformType type = transformConfig.type(); + + switch (type) { + case DOMAIN_SPLIT: + return new HighestRegisteredDomain(readIndexes, writeIndexes, logger); + case CONCAT: + if (transformConfig.getArguments().isEmpty()) { + return new Concat(readIndexes, writeIndexes, logger); + } else { + return new Concat(transformConfig.getArguments().get(0), + readIndexes, writeIndexes, logger); + } + case REGEX_EXTRACT: + return new RegexExtract(transformConfig.getArguments().get(0), readIndexes, + writeIndexes, logger); + case REGEX_SPLIT: + return new RegexSplit(transformConfig.getArguments().get(0), readIndexes, + writeIndexes, logger); + case EXCLUDE: + if (transformConfig.getCondition().getOperator().expectsANumericArgument()) { + return new ExcludeFilterNumeric(transformConfig.getCondition(), + readIndexes, writeIndexes, logger); + } else { + return new ExcludeFilterRegex(transformConfig.getCondition(), readIndexes, + writeIndexes, logger); + } + case LOWERCASE: + return StringTransform.createLowerCase(readIndexes, writeIndexes, logger); + case UPPERCASE: + return StringTransform.createUpperCase(readIndexes, writeIndexes, logger); + case TRIM: + return StringTransform.createTrim(readIndexes, writeIndexes, logger); + default: + // This code will never be hit - it's to + // keep the compiler happy. + throw new IllegalArgumentException("Unknown transform type " + type); + } + } + + /** + * For each field fill the indexArray + * with the index from the indexes map. + */ + private static void fillIndexArray(List fields, Map indexes, + int[] indexArray) { + int i = 0; + for (String field : fields) { + Integer index = indexes.get(field); + if (index != null) { + indexArray[i++] = index; + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateFormatTransform.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateFormatTransform.java new file mode 100644 index 00000000000..497113b56a1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateFormatTransform.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms.date; + +import java.time.ZoneId; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Locale; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.transforms.TransformException; +import org.elasticsearch.xpack.prelert.utils.time.DateTimeFormatterTimestampConverter; +import org.elasticsearch.xpack.prelert.utils.time.TimestampConverter; + +/** + * A transform that attempts to parse a String timestamp + * according to a timeFormat. It converts that + * to a long that represents the equivalent epoch. + */ +public class DateFormatTransform extends DateTransform { + private final String timeFormat; + private final TimestampConverter dateToEpochConverter; + + public DateFormatTransform(String timeFormat, ZoneId defaultTimezone, + List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + + this.timeFormat = timeFormat; + dateToEpochConverter = DateTimeFormatterTimestampConverter.ofPattern(timeFormat, defaultTimezone); + } + + @Override + protected long toEpochMs(String field) throws TransformException { + try { + return dateToEpochConverter.toEpochMillis(field); + } catch (DateTimeParseException pe) { + String message = String.format(Locale.ROOT, "Cannot parse date '%s' with format string '%s'", + field, timeFormat); + + throw new ParseTimestampException(message); + } + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateTransform.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateTransform.java new file mode 100644 index 00000000000..a42ced0ba49 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DateTransform.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms.date; + +import java.util.List; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.transforms.Transform; +import org.elasticsearch.xpack.prelert.transforms.TransformException; + +/** + * Abstract class introduces the {@link #epochMs()} method for + * date transforms + */ +public abstract class DateTransform extends Transform { + protected static final int SECONDS_TO_MS = 1000; + + private long epochMs; + + public DateTransform(List readIndexes, List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + } + + /** + * The epoch time from the last transform + */ + public long epochMs() { + return epochMs; + } + + /** + * Expects 1 input and 1 output. + */ + @Override + public final TransformResult transform(String[][] readWriteArea) throws TransformException { + if (readIndexes.isEmpty()) { + throw new ParseTimestampException("Cannot parse null string"); + } + + if (writeIndexes.isEmpty()) { + throw new ParseTimestampException("No write index for the datetime format transform"); + } + + TransformIndex i = readIndexes.get(0); + String field = readWriteArea[i.array][i.index]; + + if (field == null) { + throw new ParseTimestampException("Cannot parse null string"); + } + + epochMs = toEpochMs(field); + TransformIndex writeIndex = writeIndexes.get(0); + readWriteArea[writeIndex.array][writeIndex.index] = Long.toString(epochMs / SECONDS_TO_MS); + return TransformResult.OK; + } + + protected abstract long toEpochMs(String field) throws TransformException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DoubleDateTransform.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DoubleDateTransform.java new file mode 100644 index 00000000000..91969608a90 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/DoubleDateTransform.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms.date; + +import java.util.List; +import java.util.Locale; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.xpack.prelert.transforms.TransformException; + +/** + * A transformer that attempts to parse a String timestamp + * as a double and convert that to a long that represents + * an epoch time in seconds. + * If isMillisecond is true, it assumes the number represents + * time in milli-seconds and will convert to seconds + */ +public class DoubleDateTransform extends DateTransform { + private final boolean isMillisecond; + + public DoubleDateTransform(boolean isMillisecond, List readIndexes, + List writeIndexes, Logger logger) { + super(readIndexes, writeIndexes, logger); + this.isMillisecond = isMillisecond; + } + + @Override + protected long toEpochMs(String field) throws TransformException { + try { + long longValue = Double.valueOf(field).longValue(); + return isMillisecond ? longValue : longValue * SECONDS_TO_MS; + } catch (NumberFormatException e) { + String message = String.format(Locale.ROOT, "Cannot parse timestamp '%s' as epoch value", field); + throw new ParseTimestampException(message); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/ParseTimestampException.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/ParseTimestampException.java new file mode 100644 index 00000000000..c1af0d2de23 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/transforms/date/ParseTimestampException.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.transforms.date; + +import org.elasticsearch.xpack.prelert.transforms.TransformException; + +public class ParseTimestampException extends TransformException { + + public ParseTimestampException(String message) { + super(message); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/CloseableIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/CloseableIterator.java new file mode 100644 index 00000000000..5072ff9d85f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/CloseableIterator.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * An interface for iterators that can have resources that will be automatically cleaned up + * if iterator is created in a try-with-resources block. + */ +public interface CloseableIterator extends Iterator, Closeable { + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/ExceptionsHelper.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/ExceptionsHelper.java new file mode 100644 index 00000000000..24970c9d717 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/ExceptionsHelper.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +// NORELEASE: add cause exceptions! +public class ExceptionsHelper { + + public static ResourceNotFoundException missingJobException(String jobId) { + return new ResourceNotFoundException(Messages.getMessage(Messages.JOB_UNKNOWN_ID, jobId)); + } + + public static ResourceAlreadyExistsException jobAlreadyExists(String jobId) { + throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_CONFIG_ID_ALREADY_TAKEN, jobId)); + } + + public static ElasticsearchException serverError(String msg) { + return new ElasticsearchException(msg); + } + + public static ElasticsearchException serverError(String msg, Throwable cause) { + return new ElasticsearchException(msg, cause); + } + + public static ElasticsearchStatusException conflictStatusException(String msg) { + return new ElasticsearchStatusException(msg, RestStatus.CONFLICT); + } + + /** + * A more REST-friendly Object.requireNonNull() + */ + public static T requireNonNull(T obj, String paramName) { + if (obj == null) { + throw new IllegalArgumentException("[" + paramName + "] must not be null."); + } + return obj; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelper.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelper.java new file mode 100644 index 00000000000..7e46d78d3a4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/NamedPipeHelper.java @@ -0,0 +1,317 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.env.Environment; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.time.Duration; + + +/** + * Opens named pipes that are created elsewhere. + * + * In production, these will have been created in C++ code, as the procedure for creating them is + * platform dependent and uses native OS calls that are not easily available in Java. + * + * Once the named pipes have been created elsewhere Java can open them like normal files, however, + * there are complications: + * - On *nix, when opening a pipe for output Java will create a normal file of the requested name + * if the named pipe doesn't already exist. To avoid this, named pipes are only opened for + * for output once the expected file name exists in the file system. + * - On Windows, the server end of a pipe needs to reset it between client connects. Methods like + * File.isFile() and File.exists() on Windows internally call the Win32 API function CreateFile() + * followed by GetFileInformationByHandle(), and if the CreateFile() succeeds it counts as opening + * the named pipe, requiring it to be reset on the server side before subsequent access. To avoid + * this, the check for whether a given path represents a named pipe is done using simple string + * comparison on Windows. + */ +public class NamedPipeHelper { + + /** + * Try this often to open named pipes that we're waiting on another process to create. + */ + private static final long PAUSE_TIME_MS = 20; + + /** + * On Windows named pipes are ALWAYS accessed via this path; it is impossible to put them + * anywhere else. + */ + private static final String WIN_PIPE_PREFIX = "\\\\.\\pipe\\"; + + public NamedPipeHelper() { + // Do nothing - the only reason there's a constructor is to allow mocking + } + + /** + * The default path where named pipes will be created. On *nix they can be created elsewhere + * (subject to security manager constraints), but on Windows this is the ONLY place they can + * be created. + * @return The directory prefix as a string. + */ + public String getDefaultPipeDirectoryPrefix(Environment env) { + // The return type is String because we don't want any (too) clever path processing removing + // the seemingly pointless . in the path used on Windows. + if (Constants.WINDOWS) { + return WIN_PIPE_PREFIX; + } + // Use the Java temporary directory. The Elasticsearch bootstrap sets up the security + // manager to allow this to be read from and written to. Also, the code that spawns our + // daemon passes on this location to the C++ code using the $TMPDIR environment variable. + // All these factors need to align for everything to work in production. If any changes + // are made here then CNamedPipeFactory::defaultPath() in the C++ code will probably + // also need to be changed. + return env.tmpFile().toString() + PathUtils.getDefaultFileSystem().getSeparator(); + } + + /** + * Open a named pipe created elsewhere for input. + * + * @param path + * Path of named pipe to open. + * @param timeout + * How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException + * if the named pipe cannot be opened. + */ + @SuppressForbidden(reason = "Environment doesn't have path for Windows named pipes") + public InputStream openNamedPipeInputStream(String path, Duration timeout) throws IOException { + return openNamedPipeInputStream(PathUtils.get(path), timeout); + } + + /** + * Open a named pipe created elsewhere for input. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + public InputStream openNamedPipeInputStream(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Can't use Files.isRegularFile() on on named pipes on Windows, as it renders them unusable, + // but luckily there's an even simpler check (that's not possible on *nix) + if (Constants.WINDOWS && !file.toString().startsWith(WIN_PIPE_PREFIX)) { + throw new IOException(file + " is not a named pipe"); + } + + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + // Try to open the file periodically until the timeout expires, then, if + // it's still not available throw the exception from FileInputStream + while (true) { + // On Windows Files.isRegularFile() will render a genuine named pipe unusable + if (!Constants.WINDOWS && Files.isRegularFile(file)) { + throw new IOException(file + " is not a named pipe"); + } + try { + PrivilegedInputPipeOpener privilegedInputPipeOpener = new PrivilegedInputPipeOpener(file); + return AccessController.doPrivileged(privilegedInputPipeOpener); + } catch (RuntimeException e) { + if (timeoutMillisRemaining <= 0) { + propagatePrivilegedException(e); + } + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + propagatePrivilegedException(e); + } + } + } + } + + /** + * Open a named pipe created elsewhere for output. + * + * @param path + * Path of named pipe to open. + * @param timeout + * How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException + * if the named pipe cannot be opened. + */ + @SuppressForbidden(reason = "Environment doesn't have path for Windows named pipes") + public OutputStream openNamedPipeOutputStream(String path, Duration timeout) throws IOException { + return openNamedPipeOutputStream(PathUtils.get(path), timeout); + } + + /** + * Open a named pipe created elsewhere for output. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + public OutputStream openNamedPipeOutputStream(Path file, Duration timeout) throws IOException { + if (Constants.WINDOWS) { + return openNamedPipeOutputStreamWindows(file, timeout); + } + return openNamedPipeOutputStreamUnix(file, timeout); + } + + /** + * The logic here is very similar to that of opening an input stream, because on Windows + * Java cannot create a regular file when asked to open a named pipe that doesn't exist. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + private OutputStream openNamedPipeOutputStreamWindows(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Can't use File.isFile() on Windows, but luckily there's an even simpler check (that's not possible on *nix) + if (!file.toString().startsWith(WIN_PIPE_PREFIX)) { + throw new IOException(file + " is not a named pipe"); + } + + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + // Try to open the file periodically until the timeout expires, then, if + // it's still not available throw the exception from FileOutputStream + while (true) { + try { + PrivilegedOutputPipeOpener privilegedOutputPipeOpener = new PrivilegedOutputPipeOpener(file); + return AccessController.doPrivileged(privilegedOutputPipeOpener); + } catch (RuntimeException e) { + if (timeoutMillisRemaining <= 0) { + propagatePrivilegedException(e); + } + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + propagatePrivilegedException(e); + } + } + } + } + + /** + * This has to use different logic to the input pipe case to avoid the danger of creating + * a regular file when the named pipe does not exist when the method is first called. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + private OutputStream openNamedPipeOutputStreamUnix(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Periodically check whether the file exists until the timeout expires, then, if + // it's still not available throw a FileNotFoundException + while (timeoutMillisRemaining > 0 && !Files.exists(file)) { + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + + if (Files.isRegularFile(file)) { + throw new IOException(file + " is not a named pipe"); + } + + if (!Files.exists(file)) { + throw new FileNotFoundException("Cannot open " + file + " (No such file or directory)"); + } + + // There's a race condition here in that somebody could delete the named pipe at this point + // causing the line below to create a regular file. Not sure what can be done about this + // without using low level OS calls... + + return Files.newOutputStream(file); + } + + /** + * To work around the limitation that privileged actions cannot throw checked exceptions the classes + * below wrap IOExceptions in RuntimeExceptions. If such an exception needs to be propagated back + * to a user of this class then it's nice if they get the original IOException rather than having + * it wrapped in a RuntimeException. However, the privileged calls could also possibly throw other + * RuntimeExceptions, so this method accounts for this case too. + */ + private void propagatePrivilegedException(RuntimeException e) throws IOException { + Throwable ioe = ExceptionsHelper.unwrap(e, IOException.class); + if (ioe != null) { + throw (IOException)ioe; + } + throw e; + } + + /** + * Used to work around the limitation that privileged actions cannot throw checked exceptions. + */ + private static class PrivilegedInputPipeOpener implements PrivilegedAction { + + private final Path file; + + public PrivilegedInputPipeOpener(Path file) { + this.file = file; + } + + @SuppressForbidden(reason = "Files.newInputStream doesn't work with Windows named pipes") + public InputStream run() { + try { + return new FileInputStream(file.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } + + /** + * Used to work around the limitation that privileged actions cannot throw checked exceptions. + */ + private static class PrivilegedOutputPipeOpener implements PrivilegedAction { + + private final Path file; + + public PrivilegedOutputPipeOpener(Path file) { + this.file = file; + } + + @SuppressForbidden(reason = "Files.newOutputStream doesn't work with Windows named pipes") + public OutputStream run() { + try { + return new FileOutputStream(file.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/PrelertStrings.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/PrelertStrings.java new file mode 100644 index 00000000000..b8f9e770f18 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/PrelertStrings.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils; + +import java.util.regex.Pattern; + +/** + * Another String utilities class. Class name is prefixed with Prelert to avoid confusion + * with one of the myriad String utility classes out there. + */ +public final class PrelertStrings +{ + private static final Pattern NEEDS_QUOTING = Pattern.compile("\\W"); + + private PrelertStrings() + { + // do nothing + } + + /** + * Surrounds with double quotes the given {@code input} if it contains + * any non-word characters. Any double quotes contained in {@code input} + * will be escaped. + * + * @param input any non null string + * @return {@code input} when it does not contain non-word characters, or a new string + * that contains {@code input} surrounded by double quotes otherwise + */ + public static String doubleQuoteIfNotAlphaNumeric(String input) + { + if (!NEEDS_QUOTING.matcher(input).find()) + { + return input; + } + + StringBuilder quoted = new StringBuilder(); + quoted.append('\"'); + + for (int i = 0; i < input.length(); ++i) + { + char c = input.charAt(i); + if (c == '\"' || c == '\\') + { + quoted.append('\\'); + } + quoted.append(c); + } + + quoted.append('\"'); + return quoted.toString(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/SingleDocument.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/SingleDocument.java new file mode 100644 index 00000000000..00c16318091 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/SingleDocument.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.StatusToXContent; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Objects; + +/** + * Generic wrapper class for returning a single document requested through + * the REST API. If the requested document does not exist {@link #isExists()} + * will be false and {@link #getDocument()} will return null. + */ +public class SingleDocument extends ToXContentToBytes implements Writeable, StatusToXContent { + + public static final ParseField DOCUMENT = new ParseField("document"); + public static final ParseField EXISTS = new ParseField("exists"); + public static final ParseField TYPE = new ParseField("type"); + + private final boolean exists; + private final String type; + + @Nullable + private final T document; + + /** + * Constructor for a SingleDocument with an existing doc + * + * @param type + * the document type + * @param document + * the document (non-null) + */ + public SingleDocument(String type, T document) { + this.exists = document != null; + this.type = type; + this.document = document; + } + + public SingleDocument(StreamInput in, Reader documentReader) throws IOException { + this.exists = in.readBoolean(); + this.type = in.readString(); + if (in.readBoolean()) { + document = documentReader.read(in); + } else { + document = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBoolean(exists); + out.writeString(type); + boolean hasDocument = document != null; + out.writeBoolean(hasDocument); + if (hasDocument) { + document.writeTo(out); + } + } + + /** + * Return true if the requested document exists + * + * @return true is document exists + */ + public boolean isExists() { + return exists; + } + + /** + * The type of the requested document + * @return The document type + */ + public String getType() { + return type; + } + + /** + * Get the requested document or null + * + * @return The document or null + */ + @Nullable + public T getDocument() { + return document; + } + + @Override + public RestStatus status() { + return exists ? RestStatus.OK : RestStatus.NOT_FOUND; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(EXISTS.getPreferredName(), exists); + builder.field(TYPE.getPreferredName(), type); + if (document != null) { + builder.field(DOCUMENT.getPreferredName(), document); + } + return builder; + } + + /** + * Creates an empty document with the given type + * @param type the document type + * @return The empty SingleDocument + */ + public static SingleDocument empty(String type) { + return new SingleDocument(type, (T) null); + } + + @Override + public int hashCode() { + return Objects.hash(document, type, exists); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + @SuppressWarnings("unchecked") + SingleDocument other = (SingleDocument) obj; + return Objects.equals(exists, other.exists) && + Objects.equals(type, other.type) && + Objects.equals(document, other.document); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/DateTimeFormatterTimestampConverter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/DateTimeFormatterTimestampConverter.java new file mode 100644 index 00000000000..920a4b057f7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/DateTimeFormatterTimestampConverter.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils.time; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; + +/** + *

This class implements {@link TimestampConverter} using the {@link DateTimeFormatter} + * of the Java 8 time API for parsing timestamps and other classes of that API for converting + * timestamps to epoch times. + * + *

Objects of this class are immutable and thread-safe + * + */ +public class DateTimeFormatterTimestampConverter implements TimestampConverter +{ + private final DateTimeFormatter formatter; + private final boolean hasTimeZone; + private final ZoneId defaultZoneId; + + private DateTimeFormatterTimestampConverter(DateTimeFormatter dateTimeFormatter, + boolean hasTimeZone, ZoneId defaultTimezone) + { + formatter = dateTimeFormatter; + this.hasTimeZone = hasTimeZone; + defaultZoneId = defaultTimezone; + } + + /** + * Creates a formatter according to the given pattern. The system's default timezone + * is used for dates without timezone information. + * @param pattern the pattern to be used by the formatter, not null. + * See {@link DateTimeFormatter} for the syntax of the accepted patterns + * @return a {@code TimestampConverter} + * @throws IllegalArgumentException if the pattern is invalid or cannot produce a full timestamp + * (e.g. contains a date but not a time) + */ + public static TimestampConverter ofPattern(String pattern) + { + return ofPattern(pattern, ZoneOffset.systemDefault()); + } + + /** + * Creates a formatter according to the given pattern + * @param pattern the pattern to be used by the formatter, not null. + * See {@link DateTimeFormatter} for the syntax of the accepted patterns + * @param defaultTimezone the timezone to be used for dates without timezone information. + * @return a {@code TimestampConverter} + * @throws IllegalArgumentException if the pattern is invalid or cannot produce a full timestamp + * (e.g. contains a date but not a time) + */ + public static TimestampConverter ofPattern(String pattern, ZoneId defaultTimezone) + { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .parseLenient() + .appendPattern(pattern) + .parseDefaulting(ChronoField.YEAR_OF_ERA, LocalDate.now(defaultTimezone).getYear()) + .toFormatter(); + + String now = formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochSecond(0), ZoneOffset.UTC)); + try + { + TemporalAccessor parsed = formatter.parse(now); + boolean hasTimeZone = parsed.isSupported(ChronoField.INSTANT_SECONDS); + if (hasTimeZone) + { + Instant.from(parsed); + } + else + { + LocalDateTime.from(parsed); + } + return new DateTimeFormatterTimestampConverter(formatter, hasTimeZone, defaultTimezone); + } + catch (DateTimeException e) + { + throw new IllegalArgumentException("Timestamp cannot be derived from pattern: " + pattern); + } + } + + @Override + public long toEpochSeconds(String timestamp) + { + return toInstant(timestamp).getEpochSecond(); + } + + @Override + public long toEpochMillis(String timestamp) + { + return toInstant(timestamp).toEpochMilli(); + } + + private Instant toInstant(String timestamp) + { + TemporalAccessor parsed = formatter.parse(timestamp); + if (hasTimeZone) + { + return Instant.from(parsed); + } + return LocalDateTime.from(parsed).atZone(defaultZoneId).toInstant(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimeUtils.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimeUtils.java new file mode 100644 index 00000000000..cb29b393a97 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimeUtils.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils.time; + +import org.elasticsearch.index.mapper.DateFieldMapper; + +public final class TimeUtils { + private TimeUtils() { + // Do nothing + } + + /** + * First tries to parse the date first as a Long and convert that to an + * epoch time. If the long number has more than 10 digits it is considered a + * time in milliseconds else if 10 or less digits it is in seconds. If that + * fails it tries to parse the string using + * {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER} + * + * If the date string cannot be parsed -1 is returned. + * + * @return The epoch time in milliseconds or -1 if the date cannot be + * parsed. + */ + public static long dateStringToEpoch(String date) { + try { + long epoch = Long.parseLong(date); + if (date.trim().length() <= 10) // seconds + { + return epoch * 1000; + } else { + return epoch; + } + } catch (NumberFormatException nfe) { + // not a number + } + + try { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseMillis(date); + } catch (IllegalArgumentException e) { + } + // Could not do the conversion + return -1; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimestampConverter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimestampConverter.java new file mode 100644 index 00000000000..6917fc9961c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/prelert/utils/time/TimestampConverter.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.utils.time; + +import java.time.format.DateTimeParseException; + +/** + * A converter that enables conversions of textual timestamps to epoch seconds + * or milliseconds according to a given pattern. + */ +public interface TimestampConverter +{ + /** + * Converts the a textual timestamp into an epoch in seconds + * + * @param timestamp the timestamp to convert, not null. The timestamp is expected to + * be formatted according to the pattern of the formatter. In addition, the pattern is + * assumed to contain both date and time information. + * @return the epoch in seconds for the given timestamp + * @throws DateTimeParseException if unable to parse the given timestamp + */ + long toEpochSeconds(String timestamp); + + /** + * Converts the a textual timestamp into an epoch in milliseconds + * + * @param timestamp the timestamp to convert, not null. The timestamp is expected to + * be formatted according to the pattern of the formatter. In addition, the pattern is + * assumed to contain both date and time information. + * @return the epoch in milliseconds for the given timestamp + * @throws DateTimeParseException if unable to parse the given timestamp + */ + long toEpochMillis(String timestamp); +} diff --git a/elasticsearch/src/main/plugin-metadata/plugin-security.policy b/elasticsearch/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 00000000000..e3d950a5ac2 --- /dev/null +++ b/elasticsearch/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,4 @@ +grant { + // needed for Windows named pipes + permission java.io.FilePermission "\\\\.\\pipe\\*", "read,write"; +}; diff --git a/elasticsearch/src/main/resources/log4j2.properties b/elasticsearch/src/main/resources/log4j2.properties new file mode 100644 index 00000000000..46877d0de32 --- /dev/null +++ b/elasticsearch/src/main/resources/log4j2.properties @@ -0,0 +1,9 @@ +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console diff --git a/elasticsearch/src/main/resources/org/elasticsearch/xpack/prelert/job/messages/prelert_messages.properties b/elasticsearch/src/main/resources/org/elasticsearch/xpack/prelert/job/messages/prelert_messages.properties new file mode 100644 index 00000000000..c713096e23f --- /dev/null +++ b/elasticsearch/src/main/resources/org/elasticsearch/xpack/prelert/job/messages/prelert_messages.properties @@ -0,0 +1,226 @@ + +# Prelert Engine API messages + +autodetect.flush.timeout = Timed out flushing job. +autodetect.flush.failed.unexpected.death = Flush failed: Unexpected death of the Autodetect process flushing job. + +cpu.limit.jobs = Cannot start job with id ''{0}''. The maximum number of concurrently running jobs is limited as a function of the number of CPU cores see this error code''s help documentation for details of how to elevate the setting + +datastore.error.deleting = Error deleting index ''{0}'' +datastore.error.deleting.missing.index = Cannot delete job - no index with id ''{0}'' in the database +datastore.error.executing.script = Error executing script ''{0}'' + +license.limit.detectors = Cannot create new job - your license limits you to {0,number,integer} detector(s), but you have configured {1,number,integer}. +license.limit.detectors.reactivate = Cannot reactivate job with id ''{0}'' - your license limits you to {1,number,integer} concurrently running detectors. You must close a job before you can reactivate another. +license.limit.jobs = Cannot create new job - your license limits you to {0,number,integer} concurrently running job(s). You must close a job before you can create a new one. +license.limit.jobs.reactivate = Cannot reactivate job with id ''{0}'' - your license limits you to {1,number,integer} concurrently running jobs. You must close a job before you can reactivate another. +license.limit.partitions = Cannot create new job - your license disallows partition fields, but you have configured one. + +logfile.invalid.chars.path = Invalid log file path. ''{0}''. Log file names cannot contain ''\\'' or ''/'' +logfile.invalid.path = Invalid log file path. ''{0}'' is outside the base logs directory +logfile.missing = Cannot read log file ''{0}'' +logfile.missing.directory = Cannot open log file directory ''{0}'' + +job.audit.created = Job created +job.audit.deleted = Job deleted +job.audit.paused = Job paused +job.audit.resumed = Job resumed +job.audit.updated = Job updated: {0} +job.audit.reverted = Job model snapshot reverted to ''{0}'' +job.audit.old.results.deleted = Deleted results prior to {0} +job.audit.snapshot.deleted = Job model snapshot ''{0}'' deleted +job.audit.scheduler.started.from.to = Scheduler started (from: {0} to: {1}) +job.audit.scheduler.started.realtime = Scheduler started in real-time +job.audit.scheduler.continued.realtime = Scheduler continued in real-time +job.audit.scheduler.lookback.completed = Scheduler lookback completed +job.audit.scheduler.stopped = Scheduler stopped +job.audit.scheduler.no.data = Scheduler has been retrieving no data for a while +job.audit.scheduler.data.seen.again = Scheduler has started retrieving data again +job.audit.scheduler.data.analysis.error = Scheduler is encountering errors submitting data for analysis: {0} +job.audit.scheduler.data.extraction.error = Scheduler is encountering errors extracting data: {0} +job.audit.scheduler.recovered = Scheduler has recovered data extraction and analysis + +system.audit.started = System started +system.audit.shutdown = System shut down + +job.cannot.delete.while.scheduler.runs = Cannot delete job {0} while the scheduler is running +job.cannot.pause = Cannot pause job ''{0}'' while its status is {1} +job.cannot.resume = Cannot resume job ''{0}'' while its status is {1} + +job.config.byField.incompatible.function = byFieldName cannot be used with function ''{0}'' +job.config.byField.needs.another = byFieldName must be used in conjunction with fieldName or function +job.config.cannot.encrypt.password = Cannot encrypt password +job.config.categorization.filters.require.categorization.field.name = categorizationFilters require setting categorizationFieldName +job.config.categorization.filters.contains.duplicates = categorizationFilters contain duplicates +job.config.categorization.filter.contains.empty = categorizationFilters are not allowed to contain empty strings +job.config.categorization.filter.contains.invalid.regex = categorizationFilters contains invalid regular expression ''{0}'' +job.config.condition.invalid.operator = Invalid operator for condition +job.config.condition.invalid.value.null = Invalid condition: the value field cannot be null +job.config.condition.invalid.value.numeric = Invalid condition value: cannot parse a double from string ''{0}'' +job.config.condition.invalid.value.regex = Invalid condition value: ''{0}'' is not a valid regular expression +job.config.condition.unknown.operator = Unknown condition operator ''{0}'' +job.config.dataformat.requires.transform = When the data format is {0}, transforms are required. +job.config.detectionrule.condition.categorical.invalid.option = Invalid detector rule: a categorical ruleCondition does not support {0} +job.config.detectionrule.condition.categorical.missing.option = Invalid detector rule: a categorical ruleCondition requires {0} to be set +job.config.detectionrule.condition.invalid.fieldname = Invalid detector rule: fieldName has to be one of {0}; actual was ''{1}'' +job.config.detectionrule.condition.missing.fieldname = Invalid detector rule: missing fieldName in ruleCondition where fieldValue ''{0}'' is set +job.config.detectionrule.condition.numerical.invalid.operator = Invalid detector rule: operator ''{0}'' is not allowed +job.config.detectionrule.condition.numerical.invalid.option = Invalid detector rule: a numerical ruleCondition does not support {0} +job.config.detectionrule.condition.numerical.missing.option = Invalid detector rule: a numerical ruleCondition requires {0} to be set +job.config.detectionrule.condition.numerical.with.fieldname.requires.fieldvalue = Invalid detector rule: a numerical ruleCondition with fieldName requires that fieldValue is set +job.config.detectionrule.invalid.targetfieldname = Invalid detector rule: targetFieldName has to be one of {0}; actual was ''{1}'' +job.config.detectionrule.missing.targetfieldname = Invalid detector rule: missing targetFieldName where targetFieldValue ''{0}'' is set +job.config.detectionrule.not.supported.by.function = Invalid detector rule: function {0} does not support rules +job.config.detectionrule.requires.at.least.one.condition = Invalid detector rule: at least one ruleCondition is required +job.config.fieldname.incompatible.function = fieldName cannot be used with function ''{0}'' +job.config.function.requires.byfield = byFieldName must be set when the ''{0}'' function is used +job.config.function.requires.fieldname = fieldName must be set when the ''{0}'' function is used +job.config.function.requires.overfield = overFieldName must be set when the ''{0}'' function is used +job.config.function.incompatible.presummarized = The ''{0}'' function cannot be used in jobs that will take pre-summarized input +job.config.id.already.taken = The job cannot be created with the Id ''{0}''. The Id is already used. +job.config.id.too.long = The job id cannot contain more than {0,number,integer} characters. +job.config.invalid.fieldname.chars = Invalid fieldname ''{0}''. Fieldnames including over, by and partition fields cannot contain any of these characters: {1} +job.config.invalid.jobid.chars = Invalid job id; must be lowercase alphanumeric and may contain hyphens or underscores +job.config.invalid.timeformat = Invalid Time format string ''{0}'' +job.config.missing.analysisconfig = Either an an AnalysisConfig or job reference id must be set +job.config.model.debug.config.invalid.bounds.percentile = Invalid modelDebugConfig: boundsPercentile must be in the range [0, 100] +job.config.field.value.too.low = {0} cannot be less than {1,number}. Value = {2,number} +job.config.no.analysis.field = One of function, fieldName, byFieldName or overFieldName must be set +job.config.no.analysis.field.not.count = Unless the function is 'count' one of fieldName, byFieldName or overFieldName must be set +job.config.no.detectors = No detectors configured +job.config.overField.incompatible.function = overFieldName cannot be used with function ''{0}'' +job.config.overField.needs.another = overFieldName must be used in conjunction with fieldName or function +job.config.overlapping.buckets.incompatible.function = Overlapping buckets cannot be used with function ''{0}'' +job.config.multiple.bucketspans.require.bucketspan = Multiple bucketSpans require a bucketSpan to be specified +job.config.multiple.bucketspans.must.be.multiple = Multiple bucketSpan ''{0}'' must be a multiple of the main bucketSpan ''{1}'' +job.config.per.partition.normalisation.requires.partition.field = If the job is configured with Per-Partition Normalization enabled a detector must have a partition field +job.config.per.partition.normalisation.cannot.use.influencers = A job configured with Per-Partition Normalization cannot use influencers + +job.config.update.analysis.limits.parse.error = JSON parse error reading the update value for analysisLimits +job.config.update.analysis.limits.cannot.be.null = Invalid update value for analysisLimits: null +job.config.update.analysis.limits.model.memory.limit.cannot.be.decreased = Invalid update value for analysisLimits: modelMemoryLimit cannot be decreased; existing is {0}, update had {1} +job.config.update.categorization.filters.invalid = Invalid update value for categorizationFilters: value must be an array of strings; actual was: {0} +job.config.update.custom.settings.invalid = Invalid update value for customSettings: value must be an object +job.config.update.description.invalid = Invalid update value for job description: value must be a string +job.config.update.detectors.invalid = Invalid update value for detectors: value must be an array +job.config.update.detectors.invalid.detector.index = Invalid index: valid range is [{0}, {1}]; actual was: {2} +job.config.update.detectors.detector.index.should.be.integer = Invalid index: integer expected; actual was: {0} +job.config.update.detectors.missing.params = Invalid update value for detectors: requires {0} and at least one of {1} +job.config.update.detectors.description.should.be.string = Invalid description: string expected; actual was: {0} +job.config.update.detectors.rules.parse.error = JSON parse error reading the update value for detectorRules +job.config.update.failed = Update failed. Please see the logs to trace the cause of the failure. +job.config.update.ignore.downtime.parse.error = Invalid update value for ignoreDowntime: expected one of {0}; actual was: {1} +job.config.update.invalid.key = Invalid key ''{0}'' +job.config.update.job.is.not.closed = Cannot update key ''{0}'' while job is not closed; current status is {1} +job.config.update.model.debug.config.parse.error = JSON parse error reading the update value for ModelDebugConfig +job.config.update.requires.non.empty.object = Update requires JSON that contains a non-empty object +job.config.update.parse.error = JSON parse error reading the job update +job.config.update.background.persist.interval.invalid = Invalid update value for backgroundPersistInterval: value must be an exact number of seconds no less than 3600 +job.config.update.renormalization.window.days.invalid = Invalid update value for renormalizationWindowDays: value must be an exact number of days +job.config.update.model.snapshot.retention.days.invalid = Invalid update value for modelSnapshotRetentionDays: value must be an exact number of days +job.config.update.results.retention.days.invalid = Invalid update value for resultsRetentionDays: value must be an exact number of days +job.config.update.scheduler.config.parse.error = JSON parse error reading the update value for schedulerConfig +job.config.update.scheduler.config.cannot.be.null = Invalid update value for schedulerConfig: null +job.config.update.scheduler.config.data.source.invalid = Invalid update value for schedulerConfig: dataSource cannot be changed; existing is {0}, update had {1} + +job.config.transform.circular.dependency = Transform type {0} with inputs {1} has a circular dependency +job.config.transform.condition.required = A condition must be defined for transform ''{0}'' +job.config.transform.duplicated.output.name = Transform ''{0}'' has an output with the same name as the summary count field. Transform outputs cannot use the summary count field, please review your configuration +job.config.transform.extract.groups.should.match.output.count = Transform ''{0}'' expects {1} output(s) but regex ''{2}'' captures {3} group(s) +job.config.transform.inputs.contain.empty.string = Transform type {0} contains empty input +job.config.transform.invalid.argument = Transform ''{0}'' has invalid argument ''{1}'' +job.config.transform.invalid.argument.count = Transform type {0} expected {1} argument(s), got {2} +job.config.transform.invalid.input.count = Transform type {0} expected {1} input(s), got {2} +job.config.transform.invalid.output.count = Transform type {0} expected {1} output(s), got {2} +job.config.transform.outputs.contain.empty.string = Transform type {0} contains empty output +job.config.transform.outputs.unused = None of the outputs of transform ''{0}'' are used. Please review your configuration +job.config.transform.output.name.used.more.than.once = Transform output name ''{0}'' is used more than once +job.config.transform.unknown.type = Unknown TransformType ''{0}'' +job.config.unknown.function = Unknown function ''{0}'' +job.config.scheduler.unknown.datasource = Unknown scheduler dataSource ''{0}'' +job.config.scheduler.field.not.supported = Scheduler configuration field {0} not supported for dataSource ''{1}'' +job.config.scheduler.invalid.option.value = Invalid {0} value ''{1}'' in scheduler configuration +job.config.scheduler.requires.bucket.span = A job configured with scheduler requires that bucketSpan is specified +job.config.scheduler.elasticsearch.does.not.support.latency = A job configured with an Elasticsearch scheduler cannot support latency +job.config.scheduler.aggregations.requires.summary.count.field = A scheduler job with aggregations for dataSource ''{0}'' must have summaryCountFieldName ''{1}'' +job.config.scheduler.elasticsearch.requires.dataformat.elasticsearch = A job configured with an Elasticsearch scheduler must have dataFormat ''ELASTICSEARCH'' +job.config.scheduler.incomplete.credentials = Both username and password must be specified if either is +job.config.scheduler.multiple.passwords = Both password and encryptedPassword were specified - please just specify one +job.config.scheduler.multiple.aggregations = Both aggregations and aggs were specified - please just specify one + +job.data.concurrent.use.close = Cannot close job {0} while another connection {2}is {1} the job +job.data.concurrent.use.delete = Cannot delete job {0} while another connection {2}is {1} the job +job.data.concurrent.use.flush = Cannot flush job {0} while another connection {2}is {1} the job +job.data.concurrent.use.pause = Cannot pause job {0} while another connection {2}is {1} the job +job.data.concurrent.use.resume = Cannot resume job {0} while another connection {2}is {1} the job +job.data.concurrent.use.revert = Cannot revert model snapshot for job {0} while another connection {2}is {1} the job +job.data.concurrent.use.update = Cannot update job {0} while another connection {2}is {1} the job +job.data.concurrent.use.upload = Cannot write to job {0} while another connection {2}is {1} the job + +job.missing.quantiles = Cannot read persisted quantiles for job ''{0}'' +job.unknown.id = No known job with id ''{0}'' + +job.scheduler.cannot.start = Cannot start scheduler for job ''{0}'' while its status is {1} +job.scheduler.cannot.stop.in.current.state = Cannot stop scheduler for job ''{0}'' while its status is {1} +job.scheduler.cannot.update.in.current.state = Cannot update scheduler for job ''{0}'' while its status is {1} +job.scheduler.failed.to.stop = Failed to stop scheduler +job.scheduler.no.such.scheduled.job = There is no job ''{0}'' with a scheduler configured +job.scheduler.status.started = started +job.scheduler.status.stopping = stopping +job.scheduler.status.stopped = stopped +job.scheduler.status.updating = updating +job.scheduler.status.deleting = deleting + + +json.job.config.mapping.error = JSON mapping error reading the job configuration +json.job.config.parse.error = JSON parse error reading the job configuration + +json.detector.config.mapping.error = JSON mapping error reading the detector configuration +json.detector.config.parse.error = JSON parse error reading the detector configuration + +json.list.document.mapping.error = JSON mapping error reading the list +json.list.document.parse.error = JSON parse error reading the list + +json.transform.config.mapping.error = JSON mapping error reading the transform configuration +json.transform.config.parse.error = JSON parse error reading the transform configuration + +on.host = on host {0} + +rest.action.not.allowed.for.scheduled.job = This action is not allowed for a scheduled job + +rest.alert.invalid.timeout = Invalid timeout parameter. Timeout must be > 0 +rest.alert.invalid.threshold = Invalid alert parameters. {0} must be in the range 0-100 +rest.alert.missing.argument = Missing argument: either 'score' or 'probability' must be specified +rest.alert.invalid.type = The alert type argument ''{0}'' isn''t a recognised type +rest.alert.cant.use.prob = Influencer alerts require an anomaly score argument + +rest.invalid.datetime.params = Query param ''{0}'' with value ''{1}'' cannot be parsed as a date or converted to a number (epoch). +rest.invalid.flush.params.missing.argument = Invalid flush parameters: ''{0}'' has not been specified. +rest.invalid.flush.params.unexpected = Invalid flush parameters: unexpected ''{0}''. +rest.invalid.reset.params = Invalid reset range parameters: ''{0}'' has not been specified. +rest.invalid.from = Parameter 'from' cannot be < 0 +rest.invalid.size = Parameter 'size' cannot be < 0 +rest.invalid.from.size.sum = The sum of parameters ''from'' and ''size'' cannot be higher than {0}. Please use filters to reduce the number of results. +rest.gzip.error = Content-Encoding = gzip but the data is not in gzip format +rest.start.after.end = Invalid time range: end time ''{0}'' is earlier than start time ''{1}''. +rest.reset.bucket.no.latency = Bucket resetting is not supported when no latency is configured. +rest.invalid.revert.params = Cannot revert to a model snapshot as no parameters were specified. +rest.job.not.closed.revert = Can only revert to a model snapshot when the job is closed. +rest.no.such.model.snapshot = No matching model snapshot exists for job ''{0}'' +rest.invalid.description.params = Both snapshot ID and new description must be provided when changing a model snapshot description. +rest.description.already.used = Model snapshot description ''{0}'' has already been used for job ''{1}'' +rest.cannot.delete.highest.priority = Model snapshot ''{0}'' is the active snapshot for job ''{1}'', so cannot be deleted + +process.action.closed.job = closed +process.action.closing.job = closing +process.action.deleting.job = deleting +process.action.flushing.job = flushing +process.action.pausing.job = pausing +process.action.resuming.job = resuming +process.action.reverting.job = reverting the model snapshot for +process.action.sleeping.job = holding +process.action.updating.job = updating +process.action.writing.job = writing to + + +support.bundle.script.error = Error executing the support bundle script ''{0}'' diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/CreateListActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/CreateListActionRequestTests.java new file mode 100644 index 00000000000..9715f70494d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/CreateListActionRequestTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.PutListAction.Request; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class CreateListActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + int size = randomInt(10); + List items = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + items.add(randomAsciiOfLengthBetween(1, 20)); + } + ListDocument listDocument = new ListDocument(randomAsciiOfLengthBetween(1, 20), items); + return new PutListAction.Request(listDocument); + } + + @Override + protected Request createBlankInstance() { + return new PutListAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return PutListAction.Request.parseRequest(parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/DeleteJobRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/DeleteJobRequestTests.java new file mode 100644 index 00000000000..8c1cd845c39 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/DeleteJobRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.DeleteJobAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class DeleteJobRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionRequestTests.java new file mode 100644 index 00000000000..6fe14f7a31d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionRequestTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.GetBucketAction.Request; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class GetBucketActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + GetBucketAction.Request request = new GetBucketAction.Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setExpand(randomBoolean()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setExpand(randomBoolean()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setStart(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setEnd(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setTimestamp(String.valueOf(randomLong())); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new GetBucketAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return GetBucketAction.Request.parseRequest(null, parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionResponseTests.java new file mode 100644 index 00000000000..489dd5f6e2b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetBucketActionResponseTests.java @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetBucketAction.Response; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.PartitionScore; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GetBucketActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String jobId = "foo"; + Bucket bucket = new Bucket(jobId); + if (randomBoolean()) { + bucket.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + int size = randomInt(10); + List bucketInfluencers = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + BucketInfluencer bucketInfluencer = new BucketInfluencer("foo"); + bucketInfluencer.setAnomalyScore(randomDouble()); + bucketInfluencer.setInfluencerFieldName(randomAsciiOfLengthBetween(1, 20)); + bucketInfluencer.setInitialAnomalyScore(randomDouble()); + bucketInfluencer.setProbability(randomDouble()); + bucketInfluencer.setRawAnomalyScore(randomDouble()); + bucketInfluencers.add(bucketInfluencer); + } + bucket.setBucketInfluencers(bucketInfluencers); + } + if (randomBoolean()) { + bucket.setBucketSpan(randomPositiveLong()); + } + if (randomBoolean()) { + bucket.setEventCount(randomPositiveLong()); + } + if (randomBoolean()) { + bucket.setId(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + int size = randomInt(10); + List influencers = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + Influencer influencer = new Influencer(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20)); + influencer.setAnomalyScore(randomDouble()); + influencer.setInitialAnomalyScore(randomDouble()); + influencer.setProbability(randomDouble()); + influencer.setId(randomAsciiOfLengthBetween(1, 20)); + influencer.setInterim(randomBoolean()); + influencer.setTimestamp(new Date(randomLong())); + influencers.add(influencer); + } + bucket.setInfluencers(influencers); + } + if (randomBoolean()) { + bucket.setInitialAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + bucket.setInterim(randomBoolean()); + } + if (randomBoolean()) { + bucket.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + int size = randomInt(10); + List partitionScores = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + partitionScores.add(new PartitionScore(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomDouble(), randomDouble())); + } + bucket.setPartitionScores(partitionScores); + } + if (randomBoolean()) { + int size = randomInt(10); + Map perPartitionMaxProbability = new HashMap<>(size); + for (int i = 0; i < size; i++) { + perPartitionMaxProbability.put(randomAsciiOfLengthBetween(1, 20), randomDouble()); + } + bucket.setPerPartitionMaxProbability(perPartitionMaxProbability); + } + if (randomBoolean()) { + bucket.setProcessingTimeMs(randomLong()); + } + if (randomBoolean()) { + bucket.setRecordCount(randomInt()); + } + if (randomBoolean()) { + int size = randomInt(10); + List records = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + AnomalyRecord anomalyRecord = new AnomalyRecord(jobId); + anomalyRecord.setAnomalyScore(randomDouble()); + anomalyRecord.setActual(Collections.singletonList(randomDouble())); + anomalyRecord.setTypical(Collections.singletonList(randomDouble())); + anomalyRecord.setProbability(randomDouble()); + anomalyRecord.setId(randomAsciiOfLengthBetween(1, 20)); + anomalyRecord.setInterim(randomBoolean()); + anomalyRecord.setTimestamp(new Date(randomLong())); + records.add(anomalyRecord); + } + bucket.setRecords(records); + } + if (randomBoolean()) { + bucket.setTimestamp(new Date(randomLong())); + } + hits.add(bucket); + } + QueryPage buckets = new QueryPage<>(hits, listSize); + return new Response(buckets); + } + + @Override + protected Response createBlankInstance() { + return new GetBucketAction.Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionRequestTests.java new file mode 100644 index 00000000000..216d0469739 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionRequestTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class GetCategoryDefinitionRequestTests extends AbstractStreamableTestCase { + + @Override + protected GetCategoryDefinitionAction.Request createTestInstance() { + String jobId = randomAsciiOfLength(10); + GetCategoryDefinitionAction.Request request = new GetCategoryDefinitionAction.Request(jobId); + if (randomBoolean()) { + request.setCategoryId(randomAsciiOfLength(10)); + } else { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected GetCategoryDefinitionAction.Request createBlankInstance() { + return new GetCategoryDefinitionAction.Request(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionResponseTests.java new file mode 100644 index 00000000000..eac364684a2 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetCategoryDefinitionResponseTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.util.Collections; + +public class GetCategoryDefinitionResponseTests extends AbstractStreamableTestCase { + + @Override + protected GetCategoryDefinitionAction.Response createTestInstance() { + QueryPage queryPage = + new QueryPage<>(Collections.singletonList(new CategoryDefinition(randomAsciiOfLength(10))), 1L); + return new GetCategoryDefinitionAction.Response(queryPage); + } + + @Override + protected GetCategoryDefinitionAction.Response createBlankInstance() { + return new GetCategoryDefinitionAction.Response(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionRequestTests.java new file mode 100644 index 00000000000..5187524f16e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionRequestTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.GetInfluencersAction.Request; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class GetInfluencersActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return GetInfluencersAction.Request.parseRequest(null, null, null, parser, () -> matcher); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDecending(randomBoolean()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionResponseTests.java new file mode 100644 index 00000000000..a08d68fecc2 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetInfluencersActionResponseTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetInfluencersAction.Response; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class GetInfluencersActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + Influencer influencer = new Influencer(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20)); + influencer.setAnomalyScore(randomDouble()); + influencer.setInitialAnomalyScore(randomDouble()); + influencer.setProbability(randomDouble()); + influencer.setId(randomAsciiOfLengthBetween(1, 20)); + influencer.setInterim(randomBoolean()); + influencer.setTimestamp(new Date(randomLong())); + hits.add(influencer); + } + QueryPage buckets = new QueryPage<>(hits, listSize); + return new Response(buckets); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionRequestTests.java new file mode 100644 index 00000000000..0cf0e312a66 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionRequestTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetJobAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class GetJobActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request instance = new Request(randomAsciiOfLengthBetween(1, 20)); + instance.config(randomBoolean()); + instance.dataCounts(randomBoolean()); + instance.modelSizeStats(randomBoolean()); + return instance; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionResponseTests.java new file mode 100644 index 00000000000..7a3fdba9945 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobActionResponseTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetJobAction.Response; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.AnalysisLimits; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.job.IgnoreDowntime; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelDebugConfig; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class GetJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final Response result; + if (randomBoolean()) { + result = new Response(); + } else { + String jobId = randomAsciiOfLength(10); + String description = randomBoolean() ? randomAsciiOfLength(10) : null; + Date createTime = new Date(randomPositiveLong()); + Date finishedTime = randomBoolean() ? new Date(randomPositiveLong()) : null; + Date lastDataTime = randomBoolean() ? new Date(randomPositiveLong()) : null; + long timeout = randomPositiveLong(); + AnalysisConfig analysisConfig = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("metric", "some_field").build())).build(); + AnalysisLimits analysisLimits = new AnalysisLimits(randomPositiveLong(), randomPositiveLong()); + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(SchedulerConfig.DataSource.FILE); + schedulerConfig.setFilePath("/file/path"); + DataDescription dataDescription = randomBoolean() ? new DataDescription.Builder().build() : null; + ModelSizeStats modelSizeStats = randomBoolean() ? new ModelSizeStats.Builder("foo").build() : null; + int numTransformers = randomIntBetween(0, 32); + List transformConfigList = new ArrayList<>(numTransformers); + for (int i = 0; i < numTransformers; i++) { + transformConfigList.add(new TransformConfig(TransformType.UPPERCASE.prettyName())); + } + ModelDebugConfig modelDebugConfig = randomBoolean() ? new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10)) : null; + DataCounts counts = randomBoolean() ? new DataCounts(jobId) : null; + IgnoreDowntime ignoreDowntime = randomFrom(IgnoreDowntime.values()); + Long normalizationWindowDays = randomBoolean() ? randomLong() : null; + Long backgroundPersistInterval = randomBoolean() ? randomLong() : null; + Long modelSnapshotRetentionDays = randomBoolean() ? randomLong() : null; + Long resultsRetentionDays = randomBoolean() ? randomLong() : null; + Map customConfig = randomBoolean() ? Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10)) + : null; + Double averageBucketProcessingTimeMs = randomBoolean() ? randomDouble() : null; + String modelSnapshotId = randomBoolean() ? randomAsciiOfLength(10) : null; + Job job = new Job(jobId, description, createTime, finishedTime, lastDataTime, + timeout, analysisConfig, analysisLimits, schedulerConfig.build(), dataDescription, modelSizeStats, transformConfigList, + modelDebugConfig, counts, ignoreDowntime, normalizationWindowDays, backgroundPersistInterval, + modelSnapshotRetentionDays, resultsRetentionDays, customConfig, averageBucketProcessingTimeMs, modelSnapshotId); + + + DataCounts dataCounts = null; + ModelSizeStats sizeStats = null; + + if (randomBoolean()) { + dataCounts = new DataCounts(randomAsciiOfLength(10), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate()); + } + if (randomBoolean()) { + sizeStats = new ModelSizeStats.Builder("foo").build(); + } + result = new Response(new GetJobAction.Response.JobInfo(job, dataCounts, sizeStats)); + } + + return result; + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionRequestTests.java new file mode 100644 index 00000000000..32f6e6670f6 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionRequestTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.GetJobsAction.Request; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class GetJobsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return GetJobsAction.Request.PARSER.apply(parser, () -> matcher); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(); + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionResponseTests.java new file mode 100644 index 00000000000..fea93db3b38 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetJobsActionResponseTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetJobsAction.Response; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.elasticsearch.xpack.prelert.job.JobTests.randomValidJobId; + +public class GetJobsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + hits.add(buildJobBuilder(randomValidJobId()).build()); + } + QueryPage buckets = new QueryPage<>(hits, listSize); + return new Response(buckets); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionRequestTests.java new file mode 100644 index 00000000000..4a94489b55f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetListAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class GetListActionRequestTests extends AbstractStreamableTestCase { + + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionResponseTests.java new file mode 100644 index 00000000000..7c5c9619f12 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetListActionResponseTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetListAction.Response; +import org.elasticsearch.xpack.prelert.lists.ListDocument; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.prelert.utils.SingleDocument; + +import java.util.Collections; + +public class GetListActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final SingleDocument result; + if (randomBoolean()) { + result = SingleDocument.empty(ListDocument.TYPE.getPreferredName()); + } else { + result = new SingleDocument<>(ListDocument.TYPE.getPreferredName(), + new ListDocument(randomAsciiOfLengthBetween(1, 20), Collections.singletonList(randomAsciiOfLengthBetween(1, 20)))); + } + return new Response(result); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionRequestTests.java new file mode 100644 index 00000000000..8415e06a90e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionRequestTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.GetModelSnapshotsAction.Request; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class GetModelSnapshotsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return GetModelSnapshotsAction.Request.parseRequest(null, parser, () -> matcher); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setDescriptionString(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setEnd(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDescOrder(randomBoolean()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionResponseTests.java new file mode 100644 index 00000000000..3a727a23d5d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetModelSnapshotsActionResponseTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetModelSnapshotsAction.Response; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class GetModelSnapshotsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + ModelSnapshot snapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + snapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + hits.add(snapshot); + } + QueryPage snapshots = new QueryPage<>(hits, listSize); + return new Response(snapshots); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionRequestTests.java new file mode 100644 index 00000000000..de18d5c8b57 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionRequestTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.GetRecordsAction.Request; +import org.elasticsearch.xpack.prelert.job.results.PageParams; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class GetRecordsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return GetRecordsAction.Request.parseRequest(null, parser, () -> matcher); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDecending(randomBoolean()); + } + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionResponseTests.java new file mode 100644 index 00000000000..155bc7a4aaf --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/GetRecordsActionResponseTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.GetRecordsAction.Response; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class GetRecordsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + String jobId = randomAsciiOfLengthBetween(1, 20); + String bucketId = randomAsciiOfLengthBetween(1, 20); + for (int j = 0; j < listSize; j++) { + AnomalyRecord record = new AnomalyRecord(jobId); + record.setId(randomAsciiOfLengthBetween(1, 20)); + hits.add(record); + } + QueryPage snapshots = new QueryPage<>(hits, listSize); + return new Response(snapshots); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PauseJobRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PauseJobRequestTests.java new file mode 100644 index 00000000000..dd50a11fed1 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PauseJobRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.PauseJobAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class PauseJobRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionRequestTests.java new file mode 100644 index 00000000000..8c9a39a3055 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionRequestTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class PostDataActionRequestTests extends AbstractStreamableTestCase { + @Override + protected PostDataAction.Request createTestInstance() { + PostDataAction.Request request = new PostDataAction.Request(randomAsciiOfLengthBetween(1, 20)); + request.setIgnoreDowntime(randomBoolean()); + if (randomBoolean()) { + request.setResetStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setResetEnd(randomAsciiOfLengthBetween(1, 20)); + } + return request; + } + + @Override + protected PostDataAction.Request createBlankInstance() { + return new PostDataAction.Request(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionResponseTests.java new file mode 100644 index 00000000000..d49466a6a49 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataActionResponseTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.joda.time.DateTime; + +public class PostDataActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected PostDataAction.Response createTestInstance() { + DataCounts counts = new DataCounts(randomAsciiOfLength(10), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate()); + + return new PostDataAction.Response(counts); + } + + @Override + protected PostDataAction.Response createBlankInstance() { + return new PostDataAction.Response("foo") ; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataCloseRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataCloseRequestTests.java new file mode 100644 index 00000000000..f938ddf047d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataCloseRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.prelert.action.PostDataCloseAction.Request; + +public class PostDataCloseRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataFlushRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataFlushRequestTests.java new file mode 100644 index 00000000000..cf07f853d7c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PostDataFlushRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.PostDataFlushAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class PostDataFlushRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + request.setCalcInterim(randomBoolean()); + if (randomBoolean()) { + request.setStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setEnd(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setAdvanceTime(randomAsciiOfLengthBetween(1, 20)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionRequestTests.java new file mode 100644 index 00000000000..45e1dd59f11 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionRequestTests.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.PutJobAction.Request; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.elasticsearch.xpack.prelert.job.JobTests.randomValidJobId; + +public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + Job.Builder jobConfiguration = buildJobBuilder(randomValidJobId()); + return new Request(jobConfiguration.build(true)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Request.parseRequest(parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionResponseTests.java new file mode 100644 index 00000000000..af813adefd8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutJobActionResponseTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.PutJobAction.Response; +import org.elasticsearch.xpack.prelert.job.IgnoreDowntime; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.elasticsearch.xpack.prelert.job.JobTests.randomValidJobId; + +public class PutJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + Job.Builder builder = buildJobBuilder(randomValidJobId()); + builder.setIgnoreDowntime(IgnoreDowntime.NEVER); + return new Response(randomBoolean(), builder.build()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionRequestTests.java new file mode 100644 index 00000000000..838ae3a3168 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionRequestTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.PutModelSnapshotDescriptionAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class PutModelSnapshotDescriptionActionRequestTests +extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return PutModelSnapshotDescriptionAction.Request.parseRequest(null, null, parser, () -> matcher); + } + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionResponseTests.java new file mode 100644 index 00000000000..74133cf7779 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/PutModelSnapshotDescriptionActionResponseTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.PutModelSnapshotDescriptionAction.Response; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class PutModelSnapshotDescriptionActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + ModelSnapshot snapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + snapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + return new Response(snapshot); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ResumeJobRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ResumeJobRequestTests.java new file mode 100644 index 00000000000..b0930604883 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ResumeJobRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.ResumeJobAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class ResumeJobRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionRequestTests.java new file mode 100644 index 00000000000..d92db9b437e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionRequestTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.RevertModelSnapshotAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class RevertModelSnapshotActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + RevertModelSnapshotAction.Request request = new RevertModelSnapshotAction.Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setDescription(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setTime(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setSnapshotId(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDeleteInterveningResults(randomBoolean()); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new RevertModelSnapshotAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return RevertModelSnapshotAction.Request.parseRequest(null, parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionResponseTests.java new file mode 100644 index 00000000000..92d367eb3ae --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/RevertModelSnapshotActionResponseTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.RevertModelSnapshotAction.Response; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class RevertModelSnapshotActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + if (randomBoolean()) { + return new Response(); + } else { + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + return new RevertModelSnapshotAction.Response(modelSnapshot); + } + } + + @Override + protected Response createBlankInstance() { + return new RevertModelSnapshotAction.Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ScheduledJobsIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ScheduledJobsIT.java new file mode 100644 index 00000000000..ce59089ce4d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ScheduledJobsIT.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.job.metadata.PrelertMetadata; +import org.elasticsearch.xpack.prelert.job.persistence.ElasticsearchPersister; +import org.junit.After; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +@ESIntegTestCase.ClusterScope(numDataNodes = 1) +public class ScheduledJobsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(PrelertPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void clearPrelertMetadata() throws Exception { + MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); + PrelertMetadata prelertMetadata = metaData.custom(PrelertMetadata.TYPE); + for (Map.Entry entry : prelertMetadata.getJobs().entrySet()) { + String jobId = entry.getKey(); + try { + StopJobSchedulerAction.Response response = + client().execute(StopJobSchedulerAction.INSTANCE, new StopJobSchedulerAction.Request(jobId)).get(); + assertTrue(response.isAcknowledged()); + } catch (Exception e) { + // ignore + } + try { + PostDataCloseAction.Response response = + client().execute(PostDataCloseAction.INSTANCE, new PostDataCloseAction.Request(jobId)).get(); + assertTrue(response.isAcknowledged()); + } catch (Exception e) { + // ignore + } + DeleteJobAction.Response response = + client().execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).get(); + assertTrue(response.isAcknowledged()); + } + } + + public void testLookbackOnly() throws Exception { + client().admin().indices().prepareCreate("data") + .addMapping("type", "time", "type=date") + .get(); + long numDocs = randomIntBetween(32, 2048); + long now = System.currentTimeMillis(); + long lastWeek = now - 604800000; + indexDocs(numDocs, lastWeek, now); + + Job.Builder job = createJob(); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build(true)); + PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + + SchedulerState schedulerState = new SchedulerState(JobSchedulerStatus.STARTING, 0, now); + StartJobSchedulerAction.Request startSchedulerRequest = new StartJobSchedulerAction.Request("_job_id", schedulerState); + StartJobSchedulerAction.Response startJobResponse = client().execute(StartJobSchedulerAction.INSTANCE, startSchedulerRequest) + .get(); + assertTrue(startJobResponse.isAcknowledged()); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts("_job_id"); + assertThat(dataCounts.getInputRecordCount(), equalTo(numDocs)); + + PrelertMetadata prelertMetadata = client().admin().cluster().prepareState().all().get() + .getState().metaData().custom(PrelertMetadata.TYPE); + assertThat(prelertMetadata.getAllocations().get("_job_id").getSchedulerState().getStatus(), + equalTo(JobSchedulerStatus.STOPPED)); + }); + } + + public void testRealtime() throws Exception { + client().admin().indices().prepareCreate("data") + .addMapping("type", "time", "type=date") + .get(); + long numDocs1 = randomIntBetween(32, 2048); + long now = System.currentTimeMillis(); + long lastWeek = System.currentTimeMillis() - 604800000; + indexDocs(numDocs1, lastWeek, now); + + Job.Builder job = createJob(); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build(true)); + PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + + SchedulerState schedulerState = new SchedulerState(JobSchedulerStatus.STARTING, 0, null); + StartJobSchedulerAction.Request startSchedulerRequest = new StartJobSchedulerAction.Request("_job_id", schedulerState); + StartJobSchedulerAction.Response startJobResponse = client().execute(StartJobSchedulerAction.INSTANCE, startSchedulerRequest) + .get(); + assertTrue(startJobResponse.isAcknowledged()); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts("_job_id"); + assertThat(dataCounts.getInputRecordCount(), equalTo(numDocs1)); + }); + + long numDocs2 = randomIntBetween(2, 64); + now = System.currentTimeMillis(); + indexDocs(numDocs2, now + 5000, now + 6000); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts("_job_id"); + assertThat(dataCounts.getInputRecordCount(), equalTo(numDocs1 + numDocs2)); + }, 30, TimeUnit.SECONDS); + + StopJobSchedulerAction.Request stopSchedulerRequest = new StopJobSchedulerAction.Request("_job_id"); + StopJobSchedulerAction.Response stopJobResponse = client().execute(StopJobSchedulerAction.INSTANCE, stopSchedulerRequest).get(); + assertTrue(startJobResponse.isAcknowledged()); + assertBusy(() -> { + PrelertMetadata prelertMetadata = client().admin().cluster().prepareState().all().get() + .getState().metaData().custom(PrelertMetadata.TYPE); + assertThat(prelertMetadata.getAllocations().get("_job_id").getSchedulerState().getStatus(), + equalTo(JobSchedulerStatus.STOPPED)); + }); + } + + private void indexDocs(long numDocs, long start, long end) { + int maxIncrement = (int) ((end - start) / numDocs); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + long timestamp = start; + for (int i = 0; i < numDocs; i++) { + IndexRequest indexRequest = new IndexRequest("data", "type"); + indexRequest.source("time", timestamp); + bulkRequestBuilder.add(indexRequest); + timestamp += randomIntBetween(1, maxIncrement); + } + BulkResponse bulkResponse = bulkRequestBuilder + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + logger.info("Indexed [{}] documents", numDocs); + } + + private Job.Builder createJob() { + SchedulerConfig.Builder scheduler = new SchedulerConfig.Builder(SchedulerConfig.DataSource.ELASTICSEARCH); + scheduler.setQueryDelay(1); + scheduler.setFrequency(2); + InetSocketAddress address = cluster().httpAddresses()[0]; + scheduler.setBaseUrl("http://" + NetworkAddress.format(address.getAddress()) + ":" + address.getPort()); + scheduler.setIndexes(Collections.singletonList("data")); + scheduler.setTypes(Collections.singletonList("type")); + + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + dataDescription.setTimeFormat(DataDescription.EPOCH_MS); + + Detector.Builder d = new Detector.Builder("count", null); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + + Job.Builder builder = new Job.Builder(); + builder.setSchedulerConfig(scheduler); + builder.setId("_job_id"); + + builder.setAnalysisConfig(analysisConfig); + builder.setDataDescription(dataDescription); + return builder; + } + + private DataCounts getDataCounts(String jobId) { + GetResponse getResponse = client().prepareGet(ElasticsearchPersister.getJobIndexName(jobId), + DataCounts.TYPE.getPreferredName(), jobId + "-data-counts").get(); + if (getResponse.isExists() == false) { + return new DataCounts("_job_id"); + } + + try (XContentParser parser = XContentHelper.createParser(getResponse.getSourceAsBytesRef())) { + return DataCounts.PARSER.apply(parser, () -> ParseFieldMatcher.EMPTY); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerActionRequestTests.java new file mode 100644 index 00000000000..4019f00945e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StartJobSchedulerActionRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.StartJobSchedulerAction.Request; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class StartJobSchedulerActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + SchedulerState state = new SchedulerState(JobSchedulerStatus.STARTING, randomLong(), randomLong()); + return new Request(randomAsciiOfLengthBetween(1, 20), state); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Request.parseRequest(null, parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerActionRequestTests.java new file mode 100644 index 00000000000..49d33af1b93 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/StopJobSchedulerActionRequestTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.StopJobSchedulerAction.Request; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class StopJobSchedulerActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusRequestTests.java new file mode 100644 index 00000000000..26344beedc1 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobSchedulerStatusRequestTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.UpdateJobSchedulerStatusAction.Request; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class UpdateJobSchedulerStatusRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20), randomFrom(JobSchedulerStatus.values())); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusRequestTests.java new file mode 100644 index 00000000000..7e5a33c59cd --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/UpdateJobStatusRequestTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.xpack.prelert.action.UpdateJobStatusAction.Request; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableTestCase; + +public class UpdateJobStatusRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20), randomFrom(JobStatus.values())); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorActionRequestTests.java new file mode 100644 index 00000000000..0d9b20ee242 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateDetectorActionRequestTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.ValidateDetectorAction.Request; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class ValidateDetectorActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + Detector.Builder detector; + if (randomBoolean()) { + detector = new Detector.Builder(randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null); + } else { + detector = new Detector.Builder(randomFrom(Detector.FIELD_NAME_FUNCTIONS), randomAsciiOfLengthBetween(1, 20)); + } + return new Request(detector.build()); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Request.parseRequest(parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformActionRequestTests.java new file mode 100644 index 00000000000..4b23c43a19b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformActionRequestTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.ValidateTransformAction.Request; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +public class ValidateTransformActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + TransformType transformType = randomFrom(TransformType.values()); + TransformConfig transform = new TransformConfig(transformType.prettyName()); + return new Request(transform); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Request.parseRequest(parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsActionRequestTests.java new file mode 100644 index 00000000000..bdf563fc909 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/action/ValidateTransformsActionRequestTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.action; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.action.ValidateTransformsAction.Request; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; +import org.elasticsearch.xpack.prelert.support.AbstractStreamableXContentTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class ValidateTransformsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + int size = randomInt(10); + List transforms = new ArrayList<>(); + for (int i = 0; i < size; i++) { + TransformType transformType = randomFrom(TransformType.values()); + TransformConfig transform = new TransformConfig(transformType.prettyName()); + transforms.add(transform); + } + return new Request(transforms); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Request.PARSER.apply(parser, () -> matcher); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertJobIT.java new file mode 100644 index 00000000000..7c8845c17c1 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertJobIT.java @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.integration; + +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.junit.After; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.prelert.integration.ScheduledJobIT.clearPrelertMetadata; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.not; + +public class PrelertJobIT extends ESRestTestCase { + + public void testPutJob_GivenFarequoteConfig() throws Exception { + Response response = createFarequoteJob(); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"jobId\":\"farequote\"")); + } + + public void testGetJob_GivenNoSuchJob() throws Exception { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs/non-existing-job")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("\"exists\":false")); + assertThat(e.getMessage(), containsString("\"type\":\"job\"")); + } + + public void testGetJob_GivenJobExists() throws Exception { + createFarequoteJob(); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs/farequote"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"exists\":true")); + assertThat(responseAsString, containsString("\"type\":\"job\"")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote\"")); + } + + public void testGetJobs_GivenNegativeFrom() throws Exception { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?from=-1")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + assertThat(e.getMessage(), containsString("\"reason\":\"Parameter [from] cannot be < 0\"")); + } + + public void testGetJobs_GivenNegativeSize() throws Exception { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?size=-1")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + assertThat(e.getMessage(), containsString("\"reason\":\"Parameter [size] cannot be < 0\"")); + } + + public void testGetJobs_GivenFromAndSizeSumTo10001() throws Exception { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?from=1000&size=11001")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + assertThat(e.getMessage(), containsString("\"reason\":\"The sum of parameters [from] and [size] cannot be higher than 10000.")); + } + + public void testGetJobs_GivenSingleJob() throws Exception { + createFarequoteJob(); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":1")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote\"")); + } + + public void testGetJobs_GivenMultipleJobs() throws Exception { + createFarequoteJob("farequote_1"); + createFarequoteJob("farequote_2"); + createFarequoteJob("farequote_3"); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":3")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_1\"")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_2\"")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_3\"")); + } + + public void testGetJobs_GivenMultipleJobsAndFromIsOne() throws Exception { + createFarequoteJob("farequote_1"); + createFarequoteJob("farequote_2"); + createFarequoteJob("farequote_3"); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?from=1"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":3")); + assertThat(responseAsString, not(containsString("\"jobId\":\"farequote_1\""))); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_2\"")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_3\"")); + } + + public void testGetJobs_GivenMultipleJobsAndSizeIsOne() throws Exception { + createFarequoteJob("farequote_1"); + createFarequoteJob("farequote_2"); + createFarequoteJob("farequote_3"); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?size=1"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":3")); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_1\"")); + assertThat(responseAsString, not(containsString("\"jobId\":\"farequote_2\""))); + assertThat(responseAsString, not(containsString("\"jobId\":\"farequote_3\""))); + } + + public void testGetJobs_GivenMultipleJobsAndFromIsOneAndSizeIsOne() throws Exception { + createFarequoteJob("farequote_1"); + createFarequoteJob("farequote_2"); + createFarequoteJob("farequote_3"); + + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs?from=1&size=1"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":3")); + assertThat(responseAsString, not(containsString("\"jobId\":\"farequote_1\""))); + assertThat(responseAsString, containsString("\"jobId\":\"farequote_2\"")); + assertThat(responseAsString, not(containsString("\"jobId\":\"farequote_3\""))); + } + + private Response createFarequoteJob() throws Exception { + return createFarequoteJob("farequote"); + } + + private Response createFarequoteJob(String jobId) throws Exception { + String job = "{\n" + " \"jobId\":\"" + jobId + "\",\n" + " \"description\":\"Analysis of response time by airline\",\n" + + " \"analysisConfig\" : {\n" + " \"bucketSpan\":3600,\n" + + " \"detectors\" :[{\"function\":\"metric\",\"fieldName\":\"responsetime\",\"byFieldName\":\"airline\"}]\n" + + " },\n" + " \"dataDescription\" : {\n" + " \"fieldDelimiter\":\",\",\n" + " \"timeField\":\"time\",\n" + + " \"timeFormat\":\"yyyy-MM-dd HH:mm:ssX\"\n" + " }\n" + "}"; + + return client().performRequest("put", PrelertPlugin.BASE_PATH + "jobs", Collections.emptyMap(), new StringEntity(job)); + } + + public void testGetBucketResults() throws Exception { + Map params = new HashMap<>(); + params.put("start", "1200"); // inclusive + params.put("end", "1400"); // exclusive + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "results/1/bucket", params)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id '1'")); + + addBucketResult("1", "1234"); + addBucketResult("1", "1235"); + addBucketResult("1", "1236"); + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "results/1/bucket", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":3")); + + params.put("end", "1235"); + response = client().performRequest("get", PrelertPlugin.BASE_PATH + "results/1/bucket", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"hitCount\":1")); + + e = expectThrows(ResponseException.class, () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "results/2/bucket/1234")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id '2'")); + + e = expectThrows(ResponseException.class, () -> client().performRequest("get", PrelertPlugin.BASE_PATH + "results/1/bucket/1")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + response = client().performRequest("get", PrelertPlugin.BASE_PATH + "results/1/bucket/1234"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(isEmptyString())); + } + + public void testPauseAndResumeJob() throws Exception { + createFarequoteJob(); + + client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_pause"); + + assertBusy(() -> { + try { + Response response = client().performRequest("get", PrelertPlugin.BASE_PATH + "jobs/farequote"); + String responseEntityToString = responseEntityToString(response); + assertThat(responseEntityToString, containsString("\"ignoreDowntime\":\"ONCE\"")); + } catch (Exception e) { + fail(); + } + }, 2, TimeUnit.SECONDS); + + client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_resume"); + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_resume")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(409)); + assertThat(e.getMessage(), containsString("Cannot resume job 'farequote' while its status is CLOSED")); + } + + public void testPauseJob_GivenJobIsPaused() throws Exception { + createFarequoteJob(); + + client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_pause"); + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_pause")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(409)); + assertThat(e.getMessage(), containsString("Cannot pause job 'farequote' while its status is PAUSED")); + } + + public void testResumeJob_GivenJobIsClosed() throws Exception { + createFarequoteJob(); + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("post", PrelertPlugin.BASE_PATH + "jobs/farequote/_resume")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(409)); + assertThat(e.getMessage(), containsString("Cannot resume job 'farequote' while its status is CLOSED")); + } + + private Response addBucketResult(String jobId, String timestamp) throws Exception { + String createIndexBody = "{ \"mappings\": {\"bucket\": { \"properties\": { \"timestamp\": { \"type\" : \"date\" } } } } }"; + try { + client().performRequest("put", "prelertresults-" + jobId, Collections.emptyMap(), new StringEntity(createIndexBody)); + } catch (ResponseException e) { + // it is ok: the index already exists + assertThat(e.getMessage(), containsString("index_already_exists_exception")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + } + + String bucketResult = String.format(Locale.ROOT, "{\"jobId\":\"%s\", \"timestamp\": \"%s\"}", jobId, timestamp); + return client().performRequest("put", "prelertresults-" + jobId + "/bucket/" + timestamp, + Collections.singletonMap("refresh", "true"), new StringEntity(bucketResult)); + } + + private static String responseEntityToString(Response response) throws Exception { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + @After + public void clearPrelertState() throws IOException { + clearPrelertMetadata(adminClient()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertYamlTestSuiteIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertYamlTestSuiteIT.java new file mode 100644 index 00000000000..b99b737fa4a --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/PrelertYamlTestSuiteIT.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.integration; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; +import org.junit.After; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import java.io.IOException; + +import static org.elasticsearch.xpack.prelert.integration.ScheduledJobIT.clearPrelertMetadata; + +/** Rest integration test. Runs against a cluster started by {@code gradle integTest} */ +public class PrelertYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public PrelertYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, ClientYamlTestParseException { + return createParameters(); + } + + @After + public void clearPrelertState() throws IOException { + clearPrelertMetadata(adminClient()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/ScheduledJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/ScheduledJobIT.java new file mode 100644 index 00000000000..1f0f86905a7 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/integration/ScheduledJobIT.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.integration; + +import org.apache.http.HttpHost; +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.prelert.PrelertPlugin; +import org.junit.After; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class ScheduledJobIT extends ESRestTestCase { + + public void testStartJobScheduler_GivenMissingJob() { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("post", PrelertPlugin.BASE_PATH + "schedulers/invalid-job/_start")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + } + + public void testStartJobScheduler_GivenNonScheduledJob() throws Exception { + createNonScheduledJob(); + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("post", PrelertPlugin.BASE_PATH + "schedulers/non-scheduled/_start")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + String responseAsString = responseEntityToString(e.getResponse()); + assertThat(responseAsString, containsString("\"reason\":\"There is no job 'non-scheduled' with a scheduler configured\"")); + } + + @AwaitsFix(bugUrl = "The lookback is sometimes too quick and then we fail to see that the scheduler_state to see is STARTED. " + + "We need to find a different way to assert this.") + public void testStartJobScheduler_GivenLookbackOnly() throws Exception { + createAirlineDataIndex(); + createScheduledJob(); + + Response response = client().performRequest("post", + PrelertPlugin.BASE_PATH + "schedulers/scheduled/_start?start=2016-06-01T00:00:00Z&end=2016-06-02T00:00:00Z"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + + assertBusy(() -> { + try { + Response response2 = client().performRequest("get", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.prelert.allocations.scheduler_state")); + assertThat(responseEntityToString(response2), containsString("\"status\":\"STARTED\"")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + waitForSchedulerToBeStopped(); + } + + public void testStartJobScheduler_GivenRealtime() throws Exception { + createAirlineDataIndex(); + createScheduledJob(); + + Response response = client().performRequest("post", + PrelertPlugin.BASE_PATH + "schedulers/scheduled/_start?start=2016-06-01T00:00:00Z"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + + assertBusy(() -> { + try { + Response response2 = client().performRequest("get", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.prelert.allocations.scheduler_state")); + assertThat(responseEntityToString(response2), containsString("\"status\":\"STARTED\"")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + response = client().performRequest("post", PrelertPlugin.BASE_PATH + "schedulers/scheduled/_stop"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + + waitForSchedulerToBeStopped(); + + } + + private void createAirlineDataIndex() throws Exception { + String airlineDataMappings = "{" + " \"mappings\": {" + " \"response\": {" + " \"properties\": {" + + " \"time\": { \"type\":\"date\"}," + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + " }" + " }" + " }" + "}"; + client().performRequest("put", "airline-data", Collections.emptyMap(), new StringEntity(airlineDataMappings)); + + client().performRequest("put", "airline-data/response/1", Collections.emptyMap(), + new StringEntity("{\"time\":\"2016-10-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}")); + client().performRequest("put", "airline-data/response/2", Collections.emptyMap(), + new StringEntity("{\"time\":\"2016-10-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}")); + + client().performRequest("post", "airline-data/_refresh"); + } + + private Response createNonScheduledJob() throws Exception { + String job = "{\n" + " \"jobId\":\"non-scheduled\",\n" + " \"description\":\"Analysis of response time by airline\",\n" + + " \"analysisConfig\" : {\n" + " \"bucketSpan\":3600,\n" + + " \"detectors\" :[{\"function\":\"mean\",\"fieldName\":\"responsetime\",\"byFieldName\":\"airline\"}]\n" + + " },\n" + " \"dataDescription\" : {\n" + " \"fieldDelimiter\":\",\",\n" + " \"timeField\":\"time\",\n" + + " \"timeFormat\":\"yyyy-MM-dd'T'HH:mm:ssX\"\n" + " }\n" + "}"; + + return client().performRequest("put", PrelertPlugin.BASE_PATH + "jobs", Collections.emptyMap(), new StringEntity(job)); + } + + private Response createScheduledJob() throws Exception { + HttpHost httpHost = getClusterHosts().get(0); + String job = "{\n" + " \"jobId\":\"scheduled\",\n" + " \"description\":\"Analysis of response time by airline\",\n" + + " \"analysisConfig\" : {\n" + " \"bucketSpan\":3600,\n" + + " \"detectors\" :[{\"function\":\"mean\",\"fieldName\":\"responsetime\",\"byFieldName\":\"airline\"}]\n" + + " },\n" + " \"dataDescription\" : {\n" + " \"format\":\"ELASTICSEARCH\",\n" + + " \"timeField\":\"time\",\n" + " \"timeFormat\":\"yyyy-MM-dd'T'HH:mm:ssX\"\n" + " },\n" + + " \"schedulerConfig\" : {\n" + " \"dataSource\":\"ELASTICSEARCH\",\n" + + " \"baseUrl\":\"" + httpHost.toURI() + "\",\n" + " \"indexes\":[\"airline-data\"],\n" + + " \"types\":[\"response\"],\n" + " \"retrieveWholeSource\":true\n" + " }\n" + "}"; + + return client().performRequest("put", PrelertPlugin.BASE_PATH + "jobs", Collections.emptyMap(), new StringEntity(job)); + } + + private static String responseEntityToString(Response response) throws Exception { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + private void waitForSchedulerToBeStopped() throws Exception { + assertBusy(() -> { + try { + Response response = client().performRequest("get", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.prelert.allocations.scheduler_state")); + assertThat(responseEntityToString(response), containsString("\"status\":\"STOPPED\"")); + } catch (Exception e) { + fail(); + } + }, 1500, TimeUnit.MILLISECONDS); + } + + @After + public void clearPrelertState() throws IOException { + clearPrelertMetadata(adminClient()); + } + + public static void clearPrelertMetadata(RestClient client) throws IOException { + Map clusterStateAsMap = entityAsMap(client.performRequest("GET", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.prelert.jobs"))); + @SuppressWarnings("unchecked") + List> jobConfigs = + (List>) XContentMapValues.extractValue("metadata.prelert.jobs", clusterStateAsMap); + if (jobConfigs == null) { + return; + } + + for (Map jobConfig : jobConfigs) { + String jobId = (String) jobConfig.get("jobId"); + try { + client.performRequest("POST", "/_xpack/prelert/schedulers/" + jobId + "/_stop"); + } catch (Exception e) { + // ignore + } + try { + client.performRequest("POST", "/_xpack/prelert/data/" + jobId + "/_close"); + } catch (Exception e) { + // ignore + } + client.performRequest("DELETE", "/_xpack/prelert/jobs/" + jobId); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisConfigTests.java new file mode 100644 index 00000000000..2d39480d7a1 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisConfigTests.java @@ -0,0 +1,804 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.detectionrules.Connective; +import org.elasticsearch.xpack.prelert.job.detectionrules.DetectionRule; +import org.elasticsearch.xpack.prelert.job.detectionrules.RuleCondition; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + + +public class AnalysisConfigTests extends AbstractSerializingTestCase { + + @Override + protected AnalysisConfig createTestInstance() { + List detectors = new ArrayList<>(); + int numDetectors = randomIntBetween(1, 10); + for (int i = 0; i < numDetectors; i++) { + detectors.add(new Detector.Builder("count", null).build()); + } + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors); + + + if (randomBoolean()) { + builder.setBatchSpan(randomPositiveLong()); + } + long bucketSpan = AnalysisConfig.Builder.DEFAULT_BUCKET_SPAN; + if (randomBoolean()) { + bucketSpan = randomIntBetween(1, 1_000_000); + builder.setBucketSpan(bucketSpan); + } + if (randomBoolean()) { + builder.setCategorizationFieldName(randomAsciiOfLength(10)); + builder.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); + } + if (randomBoolean()) { + builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); + } + if (randomBoolean()) { + builder.setLatency(randomPositiveLong()); + } + if (randomBoolean()) { + int numBucketSpans = randomIntBetween(0, 10); + List multipleBucketSpans = new ArrayList<>(); + for (int i = 2; i <= numBucketSpans; i++) { + multipleBucketSpans.add(bucketSpan * i); + } + builder.setMultipleBucketSpans(multipleBucketSpans); + } + if (randomBoolean()) { + builder.setMultivariateByFields(randomBoolean()); + } + if (randomBoolean()) { + builder.setOverlappingBuckets(randomBoolean()); + } + if (randomBoolean()) { + builder.setResultFinalizationWindow(randomPositiveLong()); + } + + builder.setUsePerPartitionNormalization(false); + return builder.build(); + } + + @Override + protected Writeable.Reader instanceReader() { + return AnalysisConfig::new; + } + + @Override + protected AnalysisConfig parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return AnalysisConfig.PARSER.apply(parser, () -> matcher).build(); + } + + public void testFieldConfiguration_singleDetector_notPreSummarised() { + // Single detector, not pre-summarised + Detector.Builder det = new Detector.Builder("metric", "responsetime"); + det.setByFieldName("airline"); + det.setPartitionFieldName("sourcetype"); + AnalysisConfig ac = createConfigWithDetectors(Collections.singletonList(det.build())); + + Set termFields = new TreeSet<>(Arrays.asList(new String[]{ + "airline", "sourcetype"})); + Set analysisFields = new TreeSet<>(Arrays.asList(new String[]{ + "responsetime", "airline", "sourcetype"})); + + assertEquals(termFields.size(), ac.termFields().size()); + assertEquals(analysisFields.size(), ac.analysisFields().size()); + + for (String s : ac.termFields()) { + assertTrue(termFields.contains(s)); + } + + for (String s : termFields) { + assertTrue(ac.termFields().contains(s)); + } + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals(1, ac.fields().size()); + assertTrue(ac.fields().contains("responsetime")); + + assertEquals(1, ac.byFields().size()); + assertTrue(ac.byFields().contains("airline")); + + assertEquals(1, ac.partitionFields().size()); + assertTrue(ac.partitionFields().contains("sourcetype")); + + assertNull(ac.getSummaryCountFieldName()); + + // Single detector, pre-summarised + analysisFields.add("summaryCount"); + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(ac); + builder.setSummaryCountFieldName("summaryCount"); + ac = builder.build(); + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals("summaryCount", ac.getSummaryCountFieldName()); + } + + public void testFieldConfiguration_multipleDetectors_NotPreSummarised() { + // Multiple detectors, not pre-summarised + List detectors = new ArrayList<>(); + + Detector.Builder det = new Detector.Builder("metric", "metric1"); + det.setByFieldName("by_one"); + det.setPartitionFieldName("partition_one"); + detectors.add(det.build()); + + det = new Detector.Builder("metric", "metric2"); + det.setByFieldName("by_two"); + det.setOverFieldName("over_field"); + detectors.add(det.build()); + + det = new Detector.Builder("metric", "metric2"); + det.setByFieldName("by_two"); + det.setPartitionFieldName("partition_two"); + detectors.add(det.build()); + + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors); + builder.setInfluencers(Collections.singletonList("Influencer_Field")); + AnalysisConfig ac = builder.build(); + + + Set termFields = new TreeSet<>(Arrays.asList(new String[]{ + "by_one", "by_two", "over_field", + "partition_one", "partition_two", "Influencer_Field"})); + Set analysisFields = new TreeSet<>(Arrays.asList(new String[]{ + "metric1", "metric2", "by_one", "by_two", "over_field", + "partition_one", "partition_two", "Influencer_Field"})); + + assertEquals(termFields.size(), ac.termFields().size()); + assertEquals(analysisFields.size(), ac.analysisFields().size()); + + for (String s : ac.termFields()) { + assertTrue(s, termFields.contains(s)); + } + + for (String s : termFields) { + assertTrue(s, ac.termFields().contains(s)); + } + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals(2, ac.fields().size()); + assertTrue(ac.fields().contains("metric1")); + assertTrue(ac.fields().contains("metric2")); + + assertEquals(2, ac.byFields().size()); + assertTrue(ac.byFields().contains("by_one")); + assertTrue(ac.byFields().contains("by_two")); + + assertEquals(1, ac.overFields().size()); + assertTrue(ac.overFields().contains("over_field")); + + assertEquals(2, ac.partitionFields().size()); + assertTrue(ac.partitionFields().contains("partition_one")); + assertTrue(ac.partitionFields().contains("partition_two")); + + assertNull(ac.getSummaryCountFieldName()); + } + + public void testFieldConfiguration_multipleDetectors_PreSummarised() { + // Multiple detectors, pre-summarised + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setSummaryCountFieldName("summaryCount"); + AnalysisConfig ac = builder.build(); + + assertTrue(ac.analysisFields().contains("summaryCount")); + assertEquals("summaryCount", ac.getSummaryCountFieldName()); + + builder = createConfigBuilder(); + builder.setBucketSpan(1000L); + builder.setMultipleBucketSpans(Arrays.asList(5000L, 10000L, 24000L)); + ac = builder.build(); + assertTrue(ac.getMultipleBucketSpans().contains(5000L)); + assertTrue(ac.getMultipleBucketSpans().contains(10000L)); + assertTrue(ac.getMultipleBucketSpans().contains(24000L)); + } + + + public void testEquals_GivenSameReference() { + AnalysisConfig config = createFullyPopulatedConfig(); + assertTrue(config.equals(config)); + } + + public void testEquals_GivenDifferentClass() { + + assertFalse(createFullyPopulatedConfig().equals("a string")); + } + + + public void testEquals_GivenNull() { + assertFalse(createFullyPopulatedConfig().equals(null)); + } + + + public void testEquals_GivenEqualConfig() { + AnalysisConfig config1 = createFullyPopulatedConfig(); + AnalysisConfig config2 = createFullyPopulatedConfig(); + + assertTrue(config1.equals(config2)); + assertTrue(config2.equals(config1)); + assertEquals(config1.hashCode(), config2.hashCode()); + } + + + public void testEquals_GivenDifferentBatchSpan() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBatchSpan(86400L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setBatchSpan(0L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentBucketSpan() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBucketSpan(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setBucketSpan(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenCategorizationField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setCategorizationFieldName("foo"); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setCategorizationFieldName("bar"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentDetector() { + AnalysisConfig config1 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "low_count").build())); + + AnalysisConfig config2 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "high_count").build())); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentInfluencers() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setInfluencers(Arrays.asList("foo")); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setInfluencers(Arrays.asList("bar")); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentLatency() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setLatency(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setLatency(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentPeriod() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setPeriod(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setPeriod(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenSummaryCountField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setSummaryCountFieldName("foo"); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setSummaryCountFieldName("bar"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenMultivariateByField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setMultivariateByFields(true); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setMultivariateByFields(false); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentCategorizationFilters() { + AnalysisConfig config1 = createFullyPopulatedConfig(); + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setCategorizationFilters(Arrays.asList("foo", "bar")); + builder.setCategorizationFieldName("cat"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + public void testBucketSpanOrDefault() { + AnalysisConfig config1 = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("min", "count").build())).build(); + assertEquals(AnalysisConfig.Builder.DEFAULT_BUCKET_SPAN, config1.getBucketSpanOrDefault()); + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBucketSpan(100L); + config1 = builder.build(); + assertEquals(100L, config1.getBucketSpanOrDefault()); + } + + public void testExtractReferencedLists() { + DetectionRule rule1 = new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "list1"))); + DetectionRule rule2 = new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "list2"))); + Detector.Builder detector1 = new Detector.Builder("count", null); + detector1.setByFieldName("foo"); + detector1.setDetectorRules(Arrays.asList(rule1)); + Detector.Builder detector2 = new Detector.Builder("count", null); + detector2.setDetectorRules(Arrays.asList(rule2)); + detector2.setByFieldName("foo"); + AnalysisConfig config = new AnalysisConfig.Builder( + Arrays.asList(detector1.build(), detector2.build(), new Detector.Builder("count", null).build())).build(); + + assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), config.extractReferencedLists()); + } + + private static AnalysisConfig createFullyPopulatedConfig() { + AnalysisConfig.Builder builder = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("min", "count").build())); + builder.setBatchSpan(86400L); + builder.setBucketSpan(3600L); + builder.setCategorizationFieldName("cat"); + builder.setCategorizationFilters(Arrays.asList("foo")); + builder.setInfluencers(Arrays.asList("myInfluencer")); + builder.setLatency(3600L); + builder.setPeriod(100L); + builder.setSummaryCountFieldName("sumCount"); + return builder.build(); + } + + private static AnalysisConfig createConfigWithDetectors(List detectors) { + return new AnalysisConfig.Builder(detectors).build(); + } + + private static AnalysisConfig.Builder createConfigBuilder() { + return new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("min", "count").build())); + } + + public void testVerify_throws() { + + // count works with no fields + Detector d = new Detector.Builder("count", null).build(); + new AnalysisConfig.Builder(Collections.singletonList(d)).build(); + + try { + d = new Detector.Builder("distinct_count", null).build(); + new AnalysisConfig.Builder(Collections.singletonList(d)).build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("Unless the function is 'count' one of fieldName, byFieldName or overFieldName must be set", e.getMessage()); + } + + // should work now + Detector.Builder builder = new Detector.Builder("distinct_count", "somefield"); + builder.setOverFieldName("over"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + builder = new Detector.Builder("info_content", "somefield"); + builder.setOverFieldName("over"); + d = builder.build(); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + builder.setByFieldName("by"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + try { + builder = new Detector.Builder("made_up_function", "somefield"); + builder.setOverFieldName("over"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("Unknown function 'made_up_function'", e.getMessage()); + } + + builder = new Detector.Builder("distinct_count", "somefield"); + AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Collections.singletonList(builder.build())); + acBuilder.setBatchSpan(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("batchSpan cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setBatchSpan(10L); + acBuilder.setBucketSpan(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("bucketSpan cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setBucketSpan(3600L); + acBuilder.setPeriod(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("period cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setPeriod(1L); + acBuilder.setLatency(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("latency cannot be less than 0. Value = -1", e.getMessage()); + } + } + + public void testVerify_GivenNegativeBucketSpan() { + AnalysisConfig.Builder config = createValidConfig(); + config.setBucketSpan(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "bucketSpan", 0, -1), e.getMessage()); + } + + public void testVerify_GivenNegativeBatchSpan() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBatchSpan(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "batchSpan", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenNegativeLatency() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setLatency(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "latency", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenNegativePeriod() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setPeriod(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "period", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenDefaultConfig_ShouldBeInvalidDueToNoDetectors() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setDetectors(null); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS), e.getMessage()); + } + + + public void testVerify_GivenValidConfig() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.build(); + } + + + public void testVerify_GivenValidConfigWithCategorizationFieldNameAndCategorizationFilters() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setCategorizationFieldName("myCategory"); + analysisConfig.setCategorizationFilters(Arrays.asList("foo", "bar")); + + analysisConfig.build(); + } + + + public void testVerify_OverlappingBuckets() { + List detectors; + Detector detector; + + boolean onByDefault = false; + + // Uncomment this when overlappingBuckets turned on by default + if (onByDefault) { + // Test overlappingBuckets unset + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("mean", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + AnalysisConfig ac = analysisConfig.build(); + assertTrue(ac.getOverlappingBuckets()); + + // Test overlappingBuckets unset + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("rare", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + + // Test overlappingBuckets unset + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("min", "value").build(); + detectors.add(detector); + detector = new Detector.Builder("max", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + } + + // Test overlappingBuckets set + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + Detector.Builder builder = new Detector.Builder("rare", null); + builder.setByFieldName("value"); + detectors.add(builder.build()); + analysisConfig.setOverlappingBuckets(false); + analysisConfig.setDetectors(detectors); + assertFalse(analysisConfig.build().getOverlappingBuckets()); + + // Test overlappingBuckets set + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setOverlappingBuckets(true); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + builder = new Detector.Builder("rare", null); + builder.setByFieldName("value"); + detectors.add(builder.build()); + analysisConfig.setDetectors(detectors); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, analysisConfig::build); + assertEquals("Overlapping buckets cannot be used with function '[rare]'", e.getMessage()); + + // Test overlappingBuckets set + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setOverlappingBuckets(false); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("mean", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + AnalysisConfig ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + } + + + public void testMultipleBucketsConfig() { + AnalysisConfig.Builder ac = createValidConfig(); + ac.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L, 35L)); + List detectors = new ArrayList<>(); + Detector detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + ac.setDetectors(detectors); + + ac.setBucketSpan(4L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, ac::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, 10, 4), e.getMessage()); + + ac.setBucketSpan(5L); + ac.build(); + + AnalysisConfig.Builder ac2 = createValidConfig(); + ac2.setBucketSpan(5L); + ac2.setDetectors(detectors); + ac2.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L)); + assertFalse(ac.equals(ac2)); + ac2.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L, 35L)); + + ac.setBucketSpan(222L); + ac.setMultipleBucketSpans(Arrays.asList()); + ac.build(); + + ac.setMultipleBucketSpans(Arrays.asList(222L)); + e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> ac.build()); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, 222, 222), e.getMessage()); + + ac.setMultipleBucketSpans(Arrays.asList(-444L, -888L)); + e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> ac.build()); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, -444, 222), e.getMessage()); + } + + + public void testVerify_GivenCategorizationFiltersButNoCategorizationFieldName() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFilters(Arrays.asList("foo")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME), e.getMessage()); + } + + + public void testVerify_GivenDuplicateCategorizationFilters() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "bar", "foo")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES), e.getMessage()); + } + + + public void testVerify_GivenEmptyCategorizationFilter() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY), e.getMessage()); + } + + + public void testCheckDetectorsHavePartitionFields() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setUsePerPartitionNormalization(true); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD), e.getMessage()); + } + + + public void testCheckDetectorsHavePartitionFields_doesntThrowWhenValid() { + AnalysisConfig.Builder config = createValidConfig(); + Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); + builder.setPartitionFieldName("pField"); + config.build().getDetectors().set(0, builder.build()); + config.setUsePerPartitionNormalization(true); + + config.build(); + } + + + public void testCheckNoInfluencersAreSet() { + + AnalysisConfig.Builder config = createValidConfig(); + Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); + builder.setPartitionFieldName("pField"); + config.build().getDetectors().set(0, builder.build()); + config.setInfluencers(Arrays.asList("inf1", "inf2")); + config.setUsePerPartitionNormalization(true); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS), e.getMessage()); + } + + + public void testVerify_GivenCategorizationFiltersContainInvalidRegex() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "(")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX, "("), e.getMessage()); + } + + private static AnalysisConfig.Builder createValidConfig() { + List detectors = new ArrayList<>(); + Detector detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors); + analysisConfig.setBucketSpan(3600L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setLatency(0L); + analysisConfig.setPeriod(0L); + return analysisConfig; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisLimitsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisLimitsTests.java new file mode 100644 index 00000000000..68b717c73d2 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/AnalysisLimitsTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class AnalysisLimitsTests extends AbstractSerializingTestCase { + + @Override + protected AnalysisLimits createTestInstance() { + return new AnalysisLimits(randomBoolean() ? randomLong() : null, randomBoolean() ? randomPositiveLong() : null); + } + + @Override + protected Writeable.Reader instanceReader() { + return AnalysisLimits::new; + } + + @Override + protected AnalysisLimits parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return AnalysisLimits.PARSER.apply(parser, () -> matcher); + } + + public void testEquals_GivenEqual() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 20L); + + assertTrue(analysisLimits1.equals(analysisLimits1)); + assertTrue(analysisLimits1.equals(analysisLimits2)); + assertTrue(analysisLimits2.equals(analysisLimits1)); + } + + + public void testEquals_GivenDifferentModelMemoryLimit() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(11L, 20L); + + assertFalse(analysisLimits1.equals(analysisLimits2)); + assertFalse(analysisLimits2.equals(analysisLimits1)); + } + + + public void testEquals_GivenDifferentCategorizationExamplesLimit() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 21L); + + assertFalse(analysisLimits1.equals(analysisLimits2)); + assertFalse(analysisLimits2.equals(analysisLimits1)); + } + + + public void testHashCode_GivenEqual() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(5555L, 3L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(5555L, 3L); + + assertEquals(analysisLimits1.hashCode(), analysisLimits2.hashCode()); + } + + public void testVerify_GivenNegativeCategorizationExamplesLimit() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisLimits(1L, -1L)); + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT, 0, -1L); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenValid() { + new AnalysisLimits(0L, 0L); + new AnalysisLimits(1L, null); + new AnalysisLimits(1L, 1L); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataCountsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataCountsTests.java new file mode 100644 index 00000000000..ed9ef6a4cb4 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataCountsTests.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; +import org.joda.time.DateTime; + +import java.util.Date; + +import static org.hamcrest.Matchers.greaterThan; + +public class DataCountsTests extends AbstractSerializingTestCase { + + @Override + protected DataCounts createTestInstance() { + return new DataCounts(randomAsciiOfLength(10), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate()); + } + + @Override + protected Writeable.Reader instanceReader() { + return DataCounts::new; + } + + @Override + protected DataCounts parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return DataCounts.PARSER.apply(parser, () -> matcher); + } + + public void testCountsEquals_GivenEqualCounts() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9); + DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9); + + assertTrue(counts1.equals(counts2)); + assertTrue(counts2.equals(counts1)); + } + + public void testCountsHashCode_GivenEqualCounts() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9); + DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9); + + assertEquals(counts1.hashCode(), counts2.hashCode()); + } + + public void testCountsCopyConstructor() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9); + DataCounts counts2 = new DataCounts(counts1); + + assertEquals(counts1.hashCode(), counts2.hashCode()); + } + + public void testCountCreatedZero() throws Exception { + DataCounts counts = new DataCounts(randomAsciiOfLength(16)); + assertAllFieldsEqualZero(counts); + } + + public void testCountCopyCreatedFieldsNotZero() throws Exception { + DataCounts counts1 = createCounts(1, 200, 400, 3, 4, 5, 6, 1479211200000L, 1479384000000L); + assertAllFieldsGreaterThanZero(counts1); + + DataCounts counts2 = new DataCounts(counts1); + assertAllFieldsGreaterThanZero(counts2); + } + + public void testIncrements() { + DataCounts counts = new DataCounts(randomAsciiOfLength(16)); + + counts.incrementInputBytes(15); + assertEquals(15, counts.getInputBytes()); + + counts.incrementInvalidDateCount(20); + assertEquals(20, counts.getInvalidDateCount()); + + counts.incrementMissingFieldCount(25); + assertEquals(25, counts.getMissingFieldCount()); + + counts.incrementOutOfOrderTimeStampCount(30); + assertEquals(30, counts.getOutOfOrderTimeStampCount()); + + counts.incrementProcessedRecordCount(40); + assertEquals(40, counts.getProcessedRecordCount()); + } + + public void testGetInputRecordCount() { + DataCounts counts = new DataCounts(randomAsciiOfLength(16)); + counts.incrementProcessedRecordCount(5); + assertEquals(5, counts.getInputRecordCount()); + + counts.incrementOutOfOrderTimeStampCount(2); + assertEquals(7, counts.getInputRecordCount()); + + counts.incrementInvalidDateCount(1); + assertEquals(8, counts.getInputRecordCount()); + } + + public void testCalcProcessedFieldCount() { + DataCounts counts = new DataCounts(randomAsciiOfLength(16), 10L, 0L, 0L, 0L, 0L, 0L, 0L, new Date(), new Date()); + counts.calcProcessedFieldCount(3); + + assertEquals(30, counts.getProcessedFieldCount()); + + counts = new DataCounts(randomAsciiOfLength(16), 10L, 0L, 0L, 0L, 0L, 5L, 0L, new Date(), new Date()); + counts.calcProcessedFieldCount(3); + assertEquals(25, counts.getProcessedFieldCount()); + } + + public void testEquals() { + DataCounts counts1 = new DataCounts( + randomAsciiOfLength(16), 10L, 5000L, 2000L, 300L, 6L, 15L, 0L, new Date(), new Date(1435000000L)); + DataCounts counts2 = new DataCounts(counts1); + + assertEquals(counts1, counts2); + counts2.incrementInputBytes(1); + assertFalse(counts1.equals(counts2)); + } + + public void testSetEarliestRecordTimestamp_doesnotOverwrite() { + DataCounts counts = new DataCounts(randomAsciiOfLength(12)); + counts.setEarliestRecordTimeStamp(new Date(100L)); + + ESTestCase.expectThrows(IllegalStateException.class, () -> counts.setEarliestRecordTimeStamp(new Date(200L))); + assertEquals(new Date(100L), counts.getEarliestRecordTimeStamp()); + } + + private void assertAllFieldsEqualZero(DataCounts stats) throws Exception { + assertEquals(0L, stats.getProcessedRecordCount()); + assertEquals(0L, stats.getProcessedFieldCount()); + assertEquals(0L, stats.getInputBytes()); + assertEquals(0L, stats.getInputFieldCount()); + assertEquals(0L, stats.getInputRecordCount()); + assertEquals(0L, stats.getInvalidDateCount()); + assertEquals(0L, stats.getMissingFieldCount()); + assertEquals(0L, stats.getOutOfOrderTimeStampCount()); + } + + private void assertAllFieldsGreaterThanZero(DataCounts stats) throws Exception { + assertThat(stats.getProcessedRecordCount(), greaterThan(0L)); + assertThat(stats.getProcessedFieldCount(), greaterThan(0L)); + assertThat(stats.getInputBytes(), greaterThan(0L)); + assertThat(stats.getInputFieldCount(), greaterThan(0L)); + assertThat(stats.getInputRecordCount(), greaterThan(0L)); + assertThat(stats.getInputRecordCount(), greaterThan(0L)); + assertThat(stats.getInvalidDateCount(), greaterThan(0L)); + assertThat(stats.getMissingFieldCount(), greaterThan(0L)); + assertThat(stats.getOutOfOrderTimeStampCount(), greaterThan(0L)); + assertThat(stats.getLatestRecordTimeStamp().getTime(), greaterThan(0L)); + } + + private static DataCounts createCounts( + long processedRecordCount, long processedFieldCount, long inputBytes, long inputFieldCount, + long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, long earliestRecordTime, long latestRecordTime) { + + DataCounts counts = new DataCounts("foo", processedRecordCount, processedFieldCount, inputBytes, + inputFieldCount, invalidDateCount, missingFieldCount, outOfOrderTimeStampCount, + new Date(earliestRecordTime), new Date(latestRecordTime)); + + return counts; + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataDescriptionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataDescriptionTests.java new file mode 100644 index 00000000000..cbb483798fc --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataDescriptionTests.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.DataDescription.DataFormat; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; + +public class DataDescriptionTests extends AbstractSerializingTestCase { + + public void testVerify_GivenValidFormat() { + DataDescription.Builder description = new DataDescription.Builder(); + description.setTimeFormat("epoch"); + description.setTimeFormat("epoch_ms"); + description.setTimeFormat("yyyy-MM-dd HH"); + String goodFormat = "yyyy.MM.dd G 'at' HH:mm:ss z"; + description.setTimeFormat(goodFormat); + } + + public void testVerify_GivenInValidFormat() { + DataDescription.Builder description = new DataDescription.Builder(); + expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat(null)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("invalid")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, "invalid"), e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, ""), e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("y-M-dd")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, "y-M-dd"), e.getMessage()); + expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("YYY-mm-UU hh:mm:ssY")); + } + + public void testTransform_GivenDelimitedAndEpoch() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setFormat(DataFormat.DELIMITED); + dd.setTimeFormat("epoch"); + assertFalse(dd.build().transform()); + } + + public void testTransform_GivenDelimitedAndEpochMs() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setFormat(DataFormat.DELIMITED); + dd.setTimeFormat("epoch_ms"); + assertTrue(dd.build().transform()); + } + + public void testIsTransformTime_GivenTimeFormatIsEpoch() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("epoch"); + assertFalse(dd.build().isTransformTime()); + } + + public void testIsTransformTime_GivenTimeFormatIsEpochMs() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("epoch_ms"); + assertTrue(dd.build().isTransformTime()); + } + + public void testIsTransformTime_GivenTimeFormatPattern() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("yyyy-MM-dd HH:mm:ss.SSSZ"); + assertTrue(dd.build().isTransformTime()); + } + + public void testEquals_GivenDifferentDateFormat() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.DELIMITED); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentQuoteCharacter() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('\''); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentTimeField() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("time"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentTimeFormat() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch_ms"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentFieldDelimiter() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(';'); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testInvalidDataFormat() throws Exception { + BytesArray json = new BytesArray("{ \"format\":\"INEXISTENT_FORMAT\" }"); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT)); + assertThat(ex.getMessage(), containsString("[dataDescription] failed to parse field [format]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), + containsString("No enum constant org.elasticsearch.xpack.prelert.job.DataDescription.DataFormat.INEXISTENT_FORMAT")); + } + + public void testInvalidFieldDelimiter() throws Exception { + BytesArray json = new BytesArray("{ \"fieldDelimiter\":\",,\" }"); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT)); + assertThat(ex.getMessage(), containsString("[dataDescription] failed to parse field [fieldDelimiter]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), + containsString("String must be a single character, found [,,]")); + } + + public void testInvalidQuoteCharacter() throws Exception { + BytesArray json = new BytesArray("{ \"quoteCharacter\":\"''\" }"); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT)); + assertThat(ex.getMessage(), containsString("[dataDescription] failed to parse field [quoteCharacter]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("String must be a single character, found ['']")); + } + + @Override + protected DataDescription createTestInstance() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + if (randomBoolean()) { + dataDescription.setFormat(randomFrom(DataFormat.values())); + } + if (randomBoolean()) { + dataDescription.setTimeField(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + String format; + if (randomBoolean()) { + format = DataDescription.EPOCH; + } else if (randomBoolean()) { + format = DataDescription.EPOCH_MS; + } else { + format = "yyy.MM.dd G 'at' HH:mm:ss z"; + } + dataDescription.setTimeFormat(format); + } + if (randomBoolean()) { + dataDescription.setFieldDelimiter(randomAsciiOfLength(1).charAt(0)); + } + if (randomBoolean()) { + dataDescription.setQuoteCharacter(randomAsciiOfLength(1).charAt(0)); + } + return dataDescription.build(); + } + + @Override + protected Reader instanceReader() { + return DataDescription::new; + } + + @Override + protected DataDescription parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return DataDescription.PARSER.apply(parser, () -> matcher).build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataFormatTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataFormatTests.java new file mode 100644 index 00000000000..22e3e129237 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DataFormatTests.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.DataDescription.DataFormat; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class DataFormatTests extends ESTestCase { + + public void testDataFormatForString() { + assertEquals(DataFormat.DELIMITED, DataFormat.forString("delineated")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("DELINEATED")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("delimited")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("DELIMITED")); + + assertEquals(DataFormat.JSON, DataFormat.forString("json")); + assertEquals(DataFormat.JSON, DataFormat.forString("JSON")); + + assertEquals(DataFormat.SINGLE_LINE, DataFormat.forString("single_line")); + assertEquals(DataFormat.SINGLE_LINE, DataFormat.forString("SINGLE_LINE")); + } + + public void testValidOrdinals() { + assertThat(DataFormat.JSON.ordinal(), equalTo(0)); + assertThat(DataFormat.DELIMITED.ordinal(), equalTo(1)); + assertThat(DataFormat.SINGLE_LINE.ordinal(), equalTo(2)); + assertThat(DataFormat.ELASTICSEARCH.ordinal(), equalTo(3)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.JSON.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.DELIMITED.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.SINGLE_LINE.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.ELASTICSEARCH.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.JSON)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.DELIMITED)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.SINGLE_LINE)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.ELASTICSEARCH)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + DataFormat.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown DataFormat ordinal [")); + } + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DetectorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DetectorTests.java new file mode 100644 index 00000000000..6573da2baf2 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/DetectorTests.java @@ -0,0 +1,643 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; +import org.elasticsearch.xpack.prelert.job.detectionrules.Connective; +import org.elasticsearch.xpack.prelert.job.detectionrules.DetectionRule; +import org.elasticsearch.xpack.prelert.job.detectionrules.RuleCondition; +import org.elasticsearch.xpack.prelert.job.detectionrules.RuleConditionType; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class DetectorTests extends AbstractSerializingTestCase { + + public void testEquals_GivenEqual() { + Detector.Builder builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector1 = builder.build(); + + builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector2 = builder.build(); + + assertTrue(detector1.equals(detector2)); + assertTrue(detector2.equals(detector1)); + assertEquals(detector1.hashCode(), detector2.hashCode()); + } + + public void testEquals_GivenDifferentDetectorDescription() { + Detector detector1 = createDetector().build(); + Detector.Builder builder = createDetector(); + builder.setDetectorDescription("bar"); + Detector detector2 = builder.build(); + + assertFalse(detector1.equals(detector2)); + } + + public void testEquals_GivenDifferentByFieldName() { + Detector detector1 = createDetector().build(); + Detector detector2 = createDetector().build(); + + assertEquals(detector1, detector2); + + Detector.Builder builder = new Detector.Builder(detector2); + builder.setByFieldName("by2"); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + detector2 = builder.build(); + assertFalse(detector1.equals(detector2)); + } + + public void testEquals_GivenDifferentRules() { + Detector detector1 = createDetector().build(); + Detector.Builder builder = new Detector.Builder(detector1); + DetectionRule rule = new DetectionRule(builder.getDetectorRules().get(0).getTargetFieldName(), + builder.getDetectorRules().get(0).getTargetFieldValue(), Connective.OR, + builder.getDetectorRules().get(0).getRuleConditions()); + builder.getDetectorRules().set(0, rule); + Detector detector2 = builder.build(); + + assertFalse(detector1.equals(detector2)); + assertFalse(detector2.equals(detector1)); + } + + public void testExtractAnalysisFields() { + Detector detector = createDetector().build(); + assertEquals(Arrays.asList("by", "over", "partition"), detector.extractAnalysisFields()); + Detector.Builder builder = new Detector.Builder(detector); + builder.setPartitionFieldName(null); + detector = builder.build(); + assertEquals(Arrays.asList("by", "over"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + builder.setByFieldName(null); + detector = builder.build(); + assertEquals(Arrays.asList("over"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); + rule = new DetectionRule(null, null, Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + builder.setOverFieldName(null); + detector = builder.build(); + assertTrue(detector.extractAnalysisFields().isEmpty()); + } + + public void testExtractReferencedLists() { + Detector.Builder builder = createDetector(); + builder.setDetectorRules(Arrays.asList( + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by", "list1"))), + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by", "list2"))))); + + Detector detector = builder.build(); + assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), detector.extractReferencedLists()); + } + + private Detector.Builder createDetector() { + Detector.Builder detector = new Detector.Builder("mean", "field"); + detector.setByFieldName("by"); + detector.setOverFieldName("over"); + detector.setPartitionFieldName("partition"); + detector.setUseNull(true); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by", "val", condition, null))); + detector.setDetectorRules(Arrays.asList(rule)); + return detector; + } + + @Override + protected Detector createTestInstance() { + String function; + Detector.Builder detector; + if (randomBoolean()) { + detector = new Detector.Builder(function = randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null); + } else { + Set functions = new HashSet<>(Detector.FIELD_NAME_FUNCTIONS); + functions.removeAll(Detector.Builder.FUNCTIONS_WITHOUT_RULE_SUPPORT); + detector = new Detector.Builder(function = randomFrom(functions), randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + detector.setDetectorDescription(randomAsciiOfLengthBetween(1, 20)); + } + String fieldName = null; + if (randomBoolean()) { + detector.setPartitionFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } else if (randomBoolean() && Detector.NO_OVER_FIELD_NAME_FUNCTIONS.contains(function) == false) { + detector.setOverFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } else if (randomBoolean() && Detector.NO_BY_FIELD_NAME_FUNCTIONS.contains(function) == false) { + detector.setByFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + detector.setExcludeFrequent(randomFrom(Detector.ExcludeFrequent.values())); + } + if (randomBoolean()) { + int size = randomInt(10); + List detectorRules = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random DetectionRule (it is already tested) + Condition condition = new Condition(Operator.GT, "5"); + detectorRules.add(new DetectionRule(fieldName, null, Connective.OR, Collections.singletonList( + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)) + )); + } + detector.setDetectorRules(detectorRules); + } + if (randomBoolean()) { + detector.setUseNull(randomBoolean()); + } + return detector.build(); + } + + @Override + protected Reader instanceReader() { + return Detector::new; + } + + @Override + protected Detector parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Detector.PARSER.apply(parser, () -> matcher).build(); + } + + public void testVerifyFieldNames_givenInvalidChars() { + Collection testCaseArguments = getCharactersAndValidity(); + for (Object [] args : testCaseArguments) { + String character = (String) args[0]; + boolean valid = (boolean) args[1]; + Detector.Builder detector = createDetectorWithValidFieldNames(); + verifyFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyByFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyOverFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyPartitionFieldName(detector, character, valid); + } + } + + public void testVerifyFunctionForPreSummariedInput() { + Collection testCaseArguments = getCharactersAndValidity(); + for (Object [] args : testCaseArguments) { + String character = (String) args[0]; + boolean valid = (boolean) args[1]; + Detector.Builder detector = createDetectorWithValidFieldNames(); + verifyFieldNameGivenPresummarised(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyByFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyOverFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyByFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyPartitionFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + } + } + + private static void verifyFieldName(Detector.Builder detector, String character, boolean valid) { + Detector.Builder updated = createDetectorWithSpecificFieldName(detector.build().getFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> updated.build()); + } + } + + private static void verifyByFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setByFieldName(detector.build().getByFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyOverFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setOverFieldName(detector.build().getOverFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyPartitionFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setPartitionFieldName(detector.build().getPartitionFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + Detector.Builder updated = createDetectorWithSpecificFieldName(detector.build().getFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> updated.build(true)); + } + + private static void verifyByFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setByFieldName(detector.build().getByFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static void verifyOverFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setOverFieldName(detector.build().getOverFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static void verifyPartitionFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setPartitionFieldName(detector.build().getPartitionFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static Detector.Builder createDetectorWithValidFieldNames() { + Detector.Builder d = new Detector.Builder("metric", "field"); + d.setByFieldName("by"); + d.setOverFieldName("over"); + d.setPartitionFieldName("partition"); + return d; + } + + private static Detector.Builder createDetectorWithSpecificFieldName(String fieldName) { + Detector.Builder d = new Detector.Builder("metric", fieldName); + d.setByFieldName("by"); + d.setOverFieldName("over"); + d.setPartitionFieldName("partition"); + return d; + } + + private static Collection getCharactersAndValidity() { + return Arrays.asList(new Object[][]{ + // char, isValid? + {"a", true}, + {"[", true}, + {"]", true}, + {"(", true}, + {")", true}, + {"=", true}, + {"-", true}, + {" ", true}, + {"\"", false}, + {"\\", false}, + {"\t", false}, + {"\n", false}, + }); + } + + public void testVerify() throws Exception { + // if nothing else is set the count functions (excluding distinct count) + // are the only allowable functions + new Detector.Builder(Detector.COUNT, null).build(); + new Detector.Builder(Detector.COUNT, null).build(true); + + Set difference = new HashSet(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.COUNT); + difference.remove(Detector.HIGH_COUNT); + difference.remove(Detector.LOW_COUNT); + difference.remove(Detector.NON_ZERO_COUNT); + difference.remove(Detector.NZC); + difference.remove(Detector.LOW_NON_ZERO_COUNT); + difference.remove(Detector.LOW_NZC); + difference.remove(Detector.HIGH_NON_ZERO_COUNT); + difference.remove(Detector.HIGH_NZC); + difference.remove(Detector.TIME_OF_DAY); + difference.remove(Detector.TIME_OF_WEEK); + for (String f : difference) { + try { + new Detector.Builder(f, null).build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + new Detector.Builder(f, null).build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // certain fields aren't allowed with certain functions + // first do the over field + for (String f : new String[]{Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.LOW_NON_ZERO_COUNT, Detector.LOW_NZC, Detector.HIGH_NON_ZERO_COUNT, + Detector.HIGH_NZC}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these functions cannot have just an over field + difference = new HashSet<>(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.COUNT); + difference.remove(Detector.HIGH_COUNT); + difference.remove(Detector.LOW_COUNT); + difference.remove(Detector.TIME_OF_DAY); + difference.remove(Detector.TIME_OF_WEEK); + for (String f : difference) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these functions can have just an over field + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.RARE, Detector.FREQ_RARE}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.setByFieldName("by"); + builder.build(); + builder.build(true); + } + + + // some functions require a fieldname + for (String f : new String[]{Detector.DISTINCT_COUNT, Detector.DC, + Detector.HIGH_DISTINCT_COUNT, Detector.HIGH_DC, Detector.LOW_DISTINCT_COUNT, Detector.LOW_DC, + Detector.INFO_CONTENT, Detector.LOW_INFO_CONTENT, Detector.HIGH_INFO_CONTENT, + Detector.METRIC, Detector.MEAN, Detector.HIGH_MEAN, Detector.LOW_MEAN, Detector.AVG, + Detector.HIGH_AVG, Detector.LOW_AVG, Detector.MAX, Detector.MIN, Detector.SUM, + Detector.LOW_SUM, Detector.HIGH_SUM, Detector.NON_NULL_SUM, + Detector.LOW_NON_NULL_SUM, Detector.HIGH_NON_NULL_SUM, Detector.POPULATION_VARIANCE, + Detector.LOW_POPULATION_VARIANCE, Detector.HIGH_POPULATION_VARIANCE}) { + Detector.Builder builder = new Detector.Builder(f, "f"); + builder.setOverFieldName("over"); + builder.build(); + try { + builder.build(true); + Assert.assertFalse(Detector.METRIC.equals(f)); + } catch (IllegalArgumentException e) { + // "metric" is not allowed as the function for pre-summarised input + Assert.assertEquals(Detector.METRIC, f); + } + } + + // these functions cannot have a field name + difference = new HashSet<>(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.METRIC); + difference.remove(Detector.MEAN); + difference.remove(Detector.LOW_MEAN); + difference.remove(Detector.HIGH_MEAN); + difference.remove(Detector.AVG); + difference.remove(Detector.LOW_AVG); + difference.remove(Detector.HIGH_AVG); + difference.remove(Detector.MEDIAN); + difference.remove(Detector.MIN); + difference.remove(Detector.MAX); + difference.remove(Detector.SUM); + difference.remove(Detector.LOW_SUM); + difference.remove(Detector.HIGH_SUM); + difference.remove(Detector.NON_NULL_SUM); + difference.remove(Detector.LOW_NON_NULL_SUM); + difference.remove(Detector.HIGH_NON_NULL_SUM); + difference.remove(Detector.POPULATION_VARIANCE); + difference.remove(Detector.LOW_POPULATION_VARIANCE); + difference.remove(Detector.HIGH_POPULATION_VARIANCE); + difference.remove(Detector.DISTINCT_COUNT); + difference.remove(Detector.HIGH_DISTINCT_COUNT); + difference.remove(Detector.LOW_DISTINCT_COUNT); + difference.remove(Detector.DC); + difference.remove(Detector.LOW_DC); + difference.remove(Detector.HIGH_DC); + difference.remove(Detector.INFO_CONTENT); + difference.remove(Detector.LOW_INFO_CONTENT); + difference.remove(Detector.HIGH_INFO_CONTENT); + difference.remove(Detector.LAT_LONG); + for (String f : difference) { + Detector.Builder builder = new Detector.Builder(f, "f"); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these can have a by field + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.RARE, + Detector.NON_ZERO_COUNT, Detector.NZC}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setByFieldName("b"); + builder.build(); + builder.build(true); + } + + Detector.Builder builder = new Detector.Builder(Detector.FREQ_RARE, null); + builder.setOverFieldName("over"); + builder.setByFieldName("b"); + builder.build(); + builder.build(true); + builder = new Detector.Builder(Detector.FREQ_RARE, null); + builder.setOverFieldName("over"); + builder.setByFieldName("b"); + builder.build(); + + // some functions require a fieldname + for (String f : new String[]{Detector.METRIC, Detector.MEAN, Detector.HIGH_MEAN, + Detector.LOW_MEAN, Detector.AVG, Detector.HIGH_AVG, Detector.LOW_AVG, + Detector.MEDIAN, Detector.MAX, Detector.MIN, Detector.SUM, Detector.LOW_SUM, + Detector.HIGH_SUM, Detector.NON_NULL_SUM, Detector.LOW_NON_NULL_SUM, + Detector.HIGH_NON_NULL_SUM, Detector.POPULATION_VARIANCE, + Detector.LOW_POPULATION_VARIANCE, Detector.HIGH_POPULATION_VARIANCE, + Detector.DISTINCT_COUNT, Detector.DC, + Detector.HIGH_DISTINCT_COUNT, Detector.HIGH_DC, Detector.LOW_DISTINCT_COUNT, + Detector.LOW_DC, Detector.INFO_CONTENT, Detector.LOW_INFO_CONTENT, + Detector.HIGH_INFO_CONTENT, Detector.LAT_LONG}) { + builder = new Detector.Builder(f, "f"); + builder.setByFieldName("b"); + builder.build(); + try { + builder.build(true); + Assert.assertFalse(Detector.METRIC.equals(f)); + } catch (IllegalArgumentException e) { + // "metric" is not allowed as the function for pre-summarised input + Assert.assertEquals(Detector.METRIC, f); + } + } + + + // these functions don't work with fieldname + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.RARE, Detector.FREQ_RARE, Detector.TIME_OF_DAY, + Detector.TIME_OF_WEEK}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.RARE, Detector.FREQ_RARE, Detector.TIME_OF_DAY, + Detector.TIME_OF_WEEK}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + builder = new Detector.Builder(Detector.FREQ_RARE, "field"); + builder.setByFieldName("b"); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC}) { + builder = new Detector.Builder(f, null); + builder.setByFieldName("by"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + builder = new Detector.Builder(f, null); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.NON_ZERO_COUNT, Detector.NZC}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + } + + public void testVerify_GivenInvalidDetectionRuleTargetFieldName() { + Detector.Builder detector = new Detector.Builder("mean", "metricVale"); + detector.setByFieldName("metricName"); + detector.setPartitionFieldName("instance"); + RuleCondition ruleCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null); + DetectionRule rule = new DetectionRule("instancE", null, Connective.OR, Arrays.asList(ruleCondition)); + detector.setDetectorRules(Arrays.asList(rule)); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, detector::build); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, + "[metricName, instance]", "instancE"), + e.getMessage()); + } + + public void testVerify_GivenValidDetectionRule() { + Detector.Builder detector = new Detector.Builder("mean", "metricVale"); + detector.setByFieldName("metricName"); + detector.setPartitionFieldName("instance"); + RuleCondition ruleCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null); + DetectionRule rule = new DetectionRule("instance", null, Connective.OR, Arrays.asList(ruleCondition)); + detector.setDetectorRules(Arrays.asList(rule)); + detector.build(); + } + + public void testExcludeFrequentForString() { + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("all")); + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("ALL")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("none")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("NONE")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("by")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("BY")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("over")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("OVER")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntimeTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntimeTests.java new file mode 100644 index 00000000000..9f0c4afc689 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/IgnoreDowntimeTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.test.ESTestCase; + +public class IgnoreDowntimeTests extends ESTestCase { + + public void testForString() { + assertEquals(IgnoreDowntime.fromString("always"), IgnoreDowntime.ALWAYS); + assertEquals(IgnoreDowntime.fromString("never"), IgnoreDowntime.NEVER); + assertEquals(IgnoreDowntime.fromString("once"), IgnoreDowntime.ONCE); + } + + public void testValidOrdinals() { + assertEquals(0, IgnoreDowntime.NEVER.ordinal()); + assertEquals(1, IgnoreDowntime.ONCE.ordinal()); + assertEquals(2, IgnoreDowntime.ALWAYS.ordinal()); + } + + public void testFromString_GivenLeadingWhitespace() { + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString(" \t ALWAYS")); + } + + + public void testFromString_GivenTrailingWhitespace() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("NEVER \t ")); + } + + + public void testFromString_GivenExactMatches() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("NEVER")); + assertEquals(IgnoreDowntime.ONCE, IgnoreDowntime.fromString("ONCE")); + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString("ALWAYS")); + } + + + public void testFromString_GivenMixedCaseCharacters() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("nevEr")); + assertEquals(IgnoreDowntime.ONCE, IgnoreDowntime.fromString("oNce")); + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString("always")); + } + + public void testFromString_GivenNonMatchingString() { + ESTestCase.expectThrows(IllegalArgumentException.class, + () -> IgnoreDowntime.fromString("nope")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatusTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatusTests.java new file mode 100644 index 00000000000..18508f621da --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobSchedulerStatusTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.test.ESTestCase; + +public class JobSchedulerStatusTests extends ESTestCase { + + public void testForString() { + assertEquals(JobSchedulerStatus.fromString("starting"), JobSchedulerStatus.STARTING); + assertEquals(JobSchedulerStatus.fromString("started"), JobSchedulerStatus.STARTED); + assertEquals(JobSchedulerStatus.fromString("stopping"), JobSchedulerStatus.STOPPING); + assertEquals(JobSchedulerStatus.fromString("stopped"), JobSchedulerStatus.STOPPED); + } + + public void testValidOrdinals() { + assertEquals(0, JobSchedulerStatus.STARTING.ordinal()); + assertEquals(1, JobSchedulerStatus.STARTED.ordinal()); + assertEquals(2, JobSchedulerStatus.STOPPING.ordinal()); + assertEquals(3, JobSchedulerStatus.STOPPED.ordinal()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobStatusTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobStatusTests.java new file mode 100644 index 00000000000..afcd067bbdb --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobStatusTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.test.ESTestCase; + +public class JobStatusTests extends ESTestCase { + + public void testForString() { + assertEquals(JobStatus.fromString("closed"), JobStatus.CLOSED); + assertEquals(JobStatus.fromString("closing"), JobStatus.CLOSING); + assertEquals(JobStatus.fromString("failed"), JobStatus.FAILED); + assertEquals(JobStatus.fromString("paused"), JobStatus.PAUSED); + assertEquals(JobStatus.fromString("pausing"), JobStatus.PAUSING); + assertEquals(JobStatus.fromString("running"), JobStatus.RUNNING); + } + + public void testValidOrdinals() { + assertEquals(0, JobStatus.RUNNING.ordinal()); + assertEquals(1, JobStatus.CLOSING.ordinal()); + assertEquals(2, JobStatus.CLOSED.ordinal()); + assertEquals(3, JobStatus.FAILED.ordinal()); + assertEquals(4, JobStatus.PAUSING.ordinal()); + assertEquals(5, JobStatus.PAUSED.ordinal()); + } + + public void testIsAnyOf() { + assertFalse(JobStatus.RUNNING.isAnyOf()); + assertFalse(JobStatus.RUNNING.isAnyOf(JobStatus.CLOSED, JobStatus.CLOSING, JobStatus.FAILED, + JobStatus.PAUSED, JobStatus.PAUSING)); + assertFalse(JobStatus.CLOSED.isAnyOf(JobStatus.RUNNING, JobStatus.CLOSING, JobStatus.FAILED, + JobStatus.PAUSED, JobStatus.PAUSING)); + + assertTrue(JobStatus.RUNNING.isAnyOf(JobStatus.RUNNING)); + assertTrue(JobStatus.RUNNING.isAnyOf(JobStatus.RUNNING, JobStatus.CLOSED)); + assertTrue(JobStatus.PAUSED.isAnyOf(JobStatus.PAUSED, JobStatus.PAUSING)); + assertTrue(JobStatus.PAUSING.isAnyOf(JobStatus.PAUSED, JobStatus.PAUSING)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobTests.java new file mode 100644 index 00000000000..824618ee5cd --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/JobTests.java @@ -0,0 +1,647 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig.DataSource; +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.job.transform.TransformConfig; +import org.elasticsearch.xpack.prelert.job.transform.TransformType; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JobTests extends AbstractSerializingTestCase { + + @Override + protected Job createTestInstance() { + return createRandomizedJob(); + } + + @Override + protected Writeable.Reader instanceReader() { + return Job::new; + } + + @Override + protected Job parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Job.PARSER.apply(parser, () -> matcher).build(); + } + + public void testConstructor_GivenEmptyJobConfiguration() { + Job job = buildJobBuilder("foo").build(true); + + assertEquals("foo", job.getId()); + assertNotNull(job.getCreateTime()); + assertEquals(600L, job.getTimeout()); + assertNotNull(job.getAnalysisConfig()); + assertNull(job.getAnalysisLimits()); + assertNull(job.getCustomSettings()); + assertNotNull(job.getDataDescription()); + assertNull(job.getDescription()); + assertNull(job.getFinishedTime()); + assertNull(job.getIgnoreDowntime()); + assertNull(job.getLastDataTime()); + assertNull(job.getModelDebugConfig()); + assertNull(job.getModelSizeStats()); + assertNull(job.getRenormalizationWindowDays()); + assertNull(job.getBackgroundPersistInterval()); + assertNull(job.getModelSnapshotRetentionDays()); + assertNull(job.getResultsRetentionDays()); + assertNull(job.getSchedulerConfig()); + assertEquals(Collections.emptyList(), job.getTransforms()); + assertNotNull(job.allFields()); + assertFalse(job.allFields().isEmpty()); + } + + public void testConstructor_GivenJobConfigurationWithIgnoreDowntime() { + Job.Builder builder = new Job.Builder("foo"); + builder.setIgnoreDowntime(IgnoreDowntime.ONCE); + builder.setAnalysisConfig(createAnalysisConfig()); + Job job = builder.build(); + + assertEquals("foo", job.getId()); + assertEquals(IgnoreDowntime.ONCE, job.getIgnoreDowntime()); + } + + public void testConstructor_GivenJobConfigurationWithElasticsearchScheduler_ShouldFillDefaults() { + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + expectThrows(NullPointerException.class, () -> schedulerConfig.setQuery(null)); + } + + public void testEquals_noId() { + expectThrows(IllegalArgumentException.class, () -> buildJobBuilder("").build(true)); + assertNotNull(buildJobBuilder(null).build(true).getId()); // test auto id generation + } + + public void testEquals_GivenDifferentClass() { + Job job = buildJobBuilder("foo").build(); + assertFalse(job.equals("a string")); + } + + public void testEquals_GivenDifferentIds() { + Date createTime = new Date(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setCreateTime(createTime); + Job job1 = builder.build(); + builder.setId("bar"); + Job job2 = builder.build(); + assertFalse(job1.equals(job2)); + } + + public void testEquals_GivenDifferentRenormalizationWindowDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setRenormalizationWindowDays(3L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setRenormalizationWindowDays(4L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentBackgroundPersistInterval() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setBackgroundPersistInterval(10000L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setBackgroundPersistInterval(8000L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentModelSnapshotRetentionDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setModelSnapshotRetentionDays(10L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setModelSnapshotRetentionDays(8L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentResultsRetentionDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setResultsRetentionDays(30L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setResultsRetentionDays(4L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentCustomSettings() { + Job.Builder jobDetails1 = buildJobBuilder("foo"); + Map customSettings1 = new HashMap<>(); + customSettings1.put("key1", "value1"); + jobDetails1.setCustomSettings(customSettings1); + Job.Builder jobDetails2 = buildJobBuilder("foo"); + Map customSettings2 = new HashMap<>(); + customSettings2.put("key2", "value2"); + jobDetails2.setCustomSettings(customSettings2); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentIgnoreDowntime() { + Job.Builder job1 = new Job.Builder(); + job1.setIgnoreDowntime(IgnoreDowntime.NEVER); + Job.Builder job2 = new Job.Builder(); + job2.setIgnoreDowntime(IgnoreDowntime.ONCE); + + assertFalse(job1.equals(job2)); + assertFalse(job2.equals(job1)); + } + + public void testSetAnalysisLimits() { + Job.Builder builder = new Job.Builder(); + builder.setAnalysisLimits(new AnalysisLimits(42L, null)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> builder.setAnalysisLimits(new AnalysisLimits(41L, null))); + assertEquals("Invalid update value for analysisLimits: modelMemoryLimit cannot be decreased; existing is 42, update had 41", + e.getMessage()); + } + + // JobConfigurationTests: + + /** + * Test the {@link AnalysisConfig#analysisFields()} method which produces a + * list of analysis fields from the detectors + */ + public void testAnalysisConfigRequiredFields() { + Detector.Builder d1 = new Detector.Builder("max", "field"); + d1.setByFieldName("by"); + + Detector.Builder d2 = new Detector.Builder("metric", "field2"); + d2.setOverFieldName("over"); + + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + ac.setSummaryCountFieldName("agg"); + + List analysisFields = ac.build().analysisFields(); + assertTrue(analysisFields.size() == 5); + + assertTrue(analysisFields.contains("agg")); + assertTrue(analysisFields.contains("field")); + assertTrue(analysisFields.contains("by")); + assertTrue(analysisFields.contains("field2")); + assertTrue(analysisFields.contains("over")); + + assertFalse(analysisFields.contains("max")); + assertFalse(analysisFields.contains("")); + assertFalse(analysisFields.contains(null)); + + Detector.Builder d3 = new Detector.Builder("count", null); + d3.setByFieldName("by2"); + d3.setPartitionFieldName("partition"); + + ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build(), d3.build())); + + analysisFields = ac.build().analysisFields(); + assertTrue(analysisFields.size() == 6); + + assertTrue(analysisFields.contains("partition")); + assertTrue(analysisFields.contains("field")); + assertTrue(analysisFields.contains("by")); + assertTrue(analysisFields.contains("by2")); + assertTrue(analysisFields.contains("field2")); + assertTrue(analysisFields.contains("over")); + + assertFalse(analysisFields.contains("count")); + assertFalse(analysisFields.contains("max")); + assertFalse(analysisFields.contains("")); + assertFalse(analysisFields.contains(null)); + } + + // JobConfigurationVerifierTests: + + public void testCheckValidId_IdTooLong() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("averyveryveryaveryveryveryaveryveryveryaveryveryveryaveryveryveryaveryveryverylongid"); + expectThrows(IllegalArgumentException.class, () -> builder.build()); + } + + public void testCheckValidId_GivenAllValidChars() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("abcdefghijklmnopqrstuvwxyz-0123456789_."); + builder.build(); + } + + public void testCheckValidId_ProhibitedChars() { + String invalidChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*()+?\"'~±/\\[]{},<>="; + Job.Builder builder = buildJobBuilder("foo"); + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_INVALID_JOBID_CHARS); + for (char c : invalidChars.toCharArray()) { + builder.setId(Character.toString(c)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + } + + public void testCheckValidId_ControlChars() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("two\nlines"); + expectThrows(IllegalArgumentException.class, builder::build); + } + + public void jobConfigurationTest() { + Job.Builder builder = new Job.Builder(); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("bad id with spaces"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("bad_id_with_UPPERCASE_chars"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("a very very very very very very very very very very very very very very very very very very very very long id"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId(null); + expectThrows(IllegalArgumentException.class, builder::build); + + Detector.Builder d = new Detector.Builder("max", "a"); + d.setByFieldName("b"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + builder.setAnalysisConfig(ac); + builder.build(); + builder.setAnalysisLimits(new AnalysisLimits(-1L, null)); + expectThrows(IllegalArgumentException.class, builder::build); + AnalysisLimits limits = new AnalysisLimits(1000L, 4L); + builder.setAnalysisLimits(limits); + builder.build(); + DataDescription.Builder dc = new DataDescription.Builder(); + dc.setTimeFormat("YYY_KKKKajsatp*"); + builder.setDataDescription(dc); + expectThrows(IllegalArgumentException.class, builder::build); + dc = new DataDescription.Builder(); + builder.setDataDescription(dc); + builder.setTimeout(-1L); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setTimeout(300L); + builder.build(); + } + + public void testCheckTransformOutputIsUsed_throws() { + Job.Builder builder = buildJobBuilder("foo"); + TransformConfig tc = new TransformConfig(TransformType.Names.DOMAIN_SPLIT_NAME); + tc.setInputs(Arrays.asList("dns")); + builder.setTransforms(Arrays.asList(tc)); + expectThrows(IllegalArgumentException.class, builder::build); + Detector.Builder newDetector = new Detector.Builder(); + newDetector.setFunction(Detector.MIN); + newDetector.setFieldName(TransformType.DOMAIN_SPLIT.defaultOutputNames().get(0)); + AnalysisConfig.Builder config = new AnalysisConfig.Builder(Collections.singletonList(newDetector.build())); + builder.setAnalysisConfig(config); + builder.build(); + } + + public void testCheckTransformDuplicatOutput_outputIsSummaryCountField() { + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder config = createAnalysisConfig(); + config.setSummaryCountFieldName("summaryCountField"); + builder.setAnalysisConfig(config); + TransformConfig tc = new TransformConfig(TransformType.Names.DOMAIN_SPLIT_NAME); + tc.setInputs(Arrays.asList("dns")); + tc.setOutputs(Arrays.asList("summaryCountField")); + builder.setTransforms(Arrays.asList(tc)); + expectThrows(IllegalArgumentException.class, builder::build); + } + + public void testCheckTransformOutputIsUsed_outputIsSummaryCountField() { + Job.Builder builder = buildJobBuilder("foo"); + TransformConfig tc = new TransformConfig(TransformType.Names.EXTRACT_NAME); + tc.setInputs(Arrays.asList("dns")); + tc.setOutputs(Arrays.asList("summaryCountField")); + tc.setArguments(Arrays.asList("(.*)")); + builder.setTransforms(Arrays.asList(tc)); + expectThrows(IllegalArgumentException.class, builder::build); + } + + public void testCheckTransformOutputIsUsed_transformHasNoOutput() { + Job.Builder builder = buildJobBuilder("foo"); + // The exclude filter has no output + TransformConfig tc = new TransformConfig(TransformType.Names.EXCLUDE_NAME); + tc.setCondition(new Condition(Operator.MATCH, "whitelisted_host")); + tc.setInputs(Arrays.asList("dns")); + builder.setTransforms(Arrays.asList(tc)); + builder.build(); + } + + public void testVerify_GivenDataFormatIsSingleLineAndNullTransforms() { + String errorMessage = Messages.getMessage( + Messages.JOB_CONFIG_DATAFORMAT_REQUIRES_TRANSFORM, + DataDescription.DataFormat.SINGLE_LINE); + Job.Builder builder = buildJobBuilder("foo"); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.SINGLE_LINE); + builder.setDataDescription(dataDescription); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenDataFormatIsSingleLineAndEmptyTransforms() { + String errorMessage = Messages.getMessage( + Messages.JOB_CONFIG_DATAFORMAT_REQUIRES_TRANSFORM, + DataDescription.DataFormat.SINGLE_LINE); + Job.Builder builder = buildJobBuilder("foo"); + builder.setTransforms(new ArrayList<>()); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.SINGLE_LINE); + builder.setDataDescription(dataDescription); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenDataFormatIsSingleLineAndNonEmptyTransforms() { + ArrayList transforms = new ArrayList<>(); + TransformConfig transform = new TransformConfig("trim"); + transform.setInputs(Arrays.asList("raw")); + transform.setOutputs(Arrays.asList("time")); + transforms.add(transform); + Job.Builder builder = buildJobBuilder("foo"); + builder.setTransforms(transforms); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.SINGLE_LINE); + builder.setDataDescription(dataDescription); + builder.build(); + } + + public void testVerify_GivenNegativeRenormalizationWindowDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + "renormalizationWindowDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setRenormalizationWindowDays(-1L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenNegativeModelSnapshotRetentionDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "modelSnapshotRetentionDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setModelSnapshotRetentionDays(-1L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenLowBackgroundPersistInterval() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "backgroundPersistInterval", 3600, 3599); + Job.Builder builder = buildJobBuilder("foo"); + builder.setBackgroundPersistInterval(3599L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenNegativeResultsRetentionDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + "resultsRetentionDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setResultsRetentionDays(-1L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenElasticsearchSchedulerAndNonZeroLatency() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_ELASTICSEARCH_DOES_NOT_SUPPORT_LATENCY); + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfig(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setLatency(3600L); + builder.setAnalysisConfig(ac); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenElasticsearchSchedulerAndZeroLatency() { + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfig(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + builder.setDataDescription(dataDescription); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setLatency(0L); + builder.setAnalysisConfig(ac); + builder.build(); + } + + public void testVerify_GivenElasticsearchSchedulerAndNoLatency() { + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfig(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + builder.setDataDescription(dataDescription); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBatchSpan(1800L); + ac.setBucketSpan(100L); + builder.setAnalysisConfig(ac); + builder.build(); + } + + public void testVerify_GivenElasticsearchSchedulerWithAggsAndCorrectSummaryCountField() throws IOException { + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfigWithAggs(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + builder.setDataDescription(dataDescription); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setSummaryCountFieldName("doc_count"); + builder.setAnalysisConfig(ac); + builder.build(); + } + + public void testVerify_GivenElasticsearchSchedulerWithAggsAndNoSummaryCountField() throws IOException { + String errorMessage = Messages.getMessage( + Messages.JOB_CONFIG_SCHEDULER_AGGREGATIONS_REQUIRES_SUMMARY_COUNT_FIELD, + DataSource.ELASTICSEARCH.toString(), SchedulerConfig.DOC_COUNT); + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfigWithAggs(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + builder.setDataDescription(dataDescription); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + builder.setAnalysisConfig(ac); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenElasticsearchSchedulerWithAggsAndWrongSummaryCountField() throws IOException { + String errorMessage = Messages.getMessage( + Messages.JOB_CONFIG_SCHEDULER_AGGREGATIONS_REQUIRES_SUMMARY_COUNT_FIELD, + DataSource.ELASTICSEARCH.toString(), SchedulerConfig.DOC_COUNT); + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfigWithAggs(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setSchedulerConfig(schedulerConfig); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.ELASTICSEARCH); + builder.setDataDescription(dataDescription); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setSummaryCountFieldName("wrong"); + builder.setAnalysisConfig(ac); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public static Job.Builder buildJobBuilder(String id) { + Job.Builder builder = new Job.Builder(id); + builder.setCreateTime(new Date()); + AnalysisConfig.Builder ac = createAnalysisConfig(); + DataDescription.Builder dc = new DataDescription.Builder(); + builder.setAnalysisConfig(ac); + builder.setDataDescription(dc); + return builder; + } + + private static SchedulerConfig.Builder createValidElasticsearchSchedulerConfig() { + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + schedulerConfig.setBaseUrl("http://localhost:9200"); + schedulerConfig.setIndexes(Arrays.asList("myIndex")); + schedulerConfig.setTypes(Arrays.asList("myType")); + return schedulerConfig; + } + + private static SchedulerConfig.Builder createValidElasticsearchSchedulerConfigWithAggs() + throws IOException { + SchedulerConfig.Builder schedulerConfig = createValidElasticsearchSchedulerConfig(); + String aggStr = + "{" + + "\"buckets\" : {" + + "\"histogram\" : {" + + "\"field\" : \"time\"," + + "\"interval\" : 3600000" + + "}," + + "\"aggs\" : {" + + "\"byField\" : {" + + "\"terms\" : {" + + "\"field\" : \"airline\"," + + "\"size\" : 0" + + "}," + + "\"aggs\" : {" + + "\"stats\" : {" + + "\"stats\" : {" + + "\"field\" : \"responsetime\"" + + "}" + + "}" + + "}" + + "}" + + "}" + + "} " + + "}"; + XContentParser parser = XContentFactory.xContent(aggStr).createParser(aggStr); + schedulerConfig.setAggs(parser.map()); + return schedulerConfig; + } + + public static String randomValidJobId() { + CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray()); + return generator.ofCodePointsLength(random(), 10, 10); + } + + public static AnalysisConfig.Builder createAnalysisConfig() { + Detector.Builder d1 = new Detector.Builder("info_content", "domain"); + d1.setOverFieldName("client"); + Detector.Builder d2 = new Detector.Builder("min", "field"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + return ac; + } + + public static Job createRandomizedJob() { + String jobId = randomValidJobId(); + Job.Builder builder = new Job.Builder(jobId); + if (randomBoolean()) { + builder.setDescription(randomAsciiOfLength(10)); + } + builder.setCreateTime(new Date(randomPositiveLong())); + if (randomBoolean()) { + builder.setFinishedTime(new Date(randomPositiveLong())); + } + if (randomBoolean()) { + builder.setLastDataTime(new Date(randomPositiveLong())); + } + if (randomBoolean()) { + builder.setTimeout(randomPositiveLong()); + } + AnalysisConfig.Builder analysisConfig = createAnalysisConfig(); + analysisConfig.setBucketSpan(100L); + builder.setAnalysisConfig(analysisConfig); + builder.setAnalysisLimits(new AnalysisLimits(randomPositiveLong(), randomPositiveLong())); + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(SchedulerConfig.DataSource.FILE); + schedulerConfig.setFilePath("/file/path"); + builder.setSchedulerConfig(schedulerConfig); + if (randomBoolean()) { + builder.setDataDescription(new DataDescription.Builder()); + } + if (randomBoolean()) { + builder.setModelSizeStats(new ModelSizeStats.Builder("foo")); + } + String[] outputs; + TransformType[] transformTypes ; + AnalysisConfig ac = analysisConfig.build(); + if (randomBoolean()) { + transformTypes = new TransformType[] {TransformType.TRIM, TransformType.LOWERCASE}; + outputs = new String[] {ac.getDetectors().get(0).getFieldName(), ac.getDetectors().get(0).getOverFieldName()}; + } else { + transformTypes = new TransformType[] {TransformType.TRIM}; + outputs = new String[] {ac.getDetectors().get(0).getFieldName()}; + } + List transformConfigList = new ArrayList<>(transformTypes.length); + for (int i = 0; i < transformTypes.length; i++) { + TransformConfig tc = new TransformConfig(transformTypes[i].prettyName()); + tc.setInputs(Collections.singletonList("input" + i)); + tc.setOutputs(Collections.singletonList(outputs[i])); + transformConfigList.add(tc); + } + builder.setTransforms(transformConfigList); + if (randomBoolean()) { + builder.setModelDebugConfig(new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10))); + } + builder.setCounts(new DataCounts(jobId)); + builder.setIgnoreDowntime(randomFrom(IgnoreDowntime.values())); + if (randomBoolean()) { + builder.setRenormalizationWindowDays(randomPositiveLong()); + } + if (randomBoolean()) { + builder.setBackgroundPersistInterval(randomPositiveLong()); + } + if (randomBoolean()) { + builder.setModelSnapshotRetentionDays(randomPositiveLong()); + } + if (randomBoolean()) { + builder.setResultsRetentionDays(randomPositiveLong()); + } + if (randomBoolean()) { + builder.setCustomSettings(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + if (randomBoolean()) { + builder.setAverageBucketProcessingTimeMs(randomDouble()); + } + if (randomBoolean()) { + builder.setModelSnapshotId(randomAsciiOfLength(10)); + } + return builder.build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/MemoryStatusTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/MemoryStatusTests.java new file mode 100644 index 00000000000..341b2b2e970 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/MemoryStatusTests.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats.MemoryStatus; + +public class MemoryStatusTests extends ESTestCase { + + public void testFromString() { + assertEquals(MemoryStatus.OK, MemoryStatus.fromString(MemoryStatus.OK.getName())); + assertEquals(MemoryStatus.SOFT_LIMIT, MemoryStatus.fromString(MemoryStatus.SOFT_LIMIT.getName())); + assertEquals(MemoryStatus.HARD_LIMIT, MemoryStatus.fromString(MemoryStatus.HARD_LIMIT.getName())); + } + + public void testValidOrdinals() { + assertThat(MemoryStatus.OK.ordinal(), equalTo(0)); + assertThat(MemoryStatus.SOFT_LIMIT.ordinal(), equalTo(1)); + assertThat(MemoryStatus.HARD_LIMIT.ordinal(), equalTo(2)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + MemoryStatus.OK.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + MemoryStatus.SOFT_LIMIT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + MemoryStatus.HARD_LIMIT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(MemoryStatus.readFromStream(in), equalTo(MemoryStatus.OK)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(MemoryStatus.readFromStream(in), equalTo(MemoryStatus.SOFT_LIMIT)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(MemoryStatus.readFromStream(in), equalTo(MemoryStatus.HARD_LIMIT)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(3, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + MemoryStatus.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown MemoryStatus ordinal [")); + } + } + } + + public void testInvalidFromString() { + String statusName = randomAsciiOfLengthBetween(11, 20); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> { + MemoryStatus.fromString(statusName); + }); + assertThat(ex.getMessage(), containsString("Unknown MemoryStatus [" + statusName + "]")); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfigTests.java new file mode 100644 index 00000000000..0d960c991cf --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelDebugConfigTests.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.ModelDebugConfig.DebugDestination; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class ModelDebugConfigTests extends AbstractSerializingTestCase { + + public void testEquals() { + assertFalse(new ModelDebugConfig(0d, null).equals(null)); + assertFalse(new ModelDebugConfig(0d, null).equals("a string")); + assertFalse(new ModelDebugConfig(80.0, "").equals(new ModelDebugConfig(81.0, ""))); + assertFalse(new ModelDebugConfig(80.0, "foo").equals(new ModelDebugConfig(80.0, "bar"))); + assertFalse(new ModelDebugConfig(DebugDestination.FILE, 80.0, "foo") + .equals(new ModelDebugConfig(DebugDestination.DATA_STORE, 80.0, "foo"))); + + ModelDebugConfig modelDebugConfig = new ModelDebugConfig(0d, null); + assertTrue(modelDebugConfig.equals(modelDebugConfig)); + assertTrue(new ModelDebugConfig(0d, null).equals(new ModelDebugConfig(0d, null))); + assertTrue(new ModelDebugConfig(80.0, "foo").equals(new ModelDebugConfig(80.0, "foo"))); + assertTrue(new ModelDebugConfig(DebugDestination.FILE, 80.0, "foo").equals(new ModelDebugConfig(80.0, "foo"))); + assertTrue(new ModelDebugConfig(DebugDestination.DATA_STORE, 80.0, "foo") + .equals(new ModelDebugConfig(DebugDestination.DATA_STORE, 80.0, "foo"))); + } + + public void testHashCode() { + assertEquals(new ModelDebugConfig(80.0, "foo").hashCode(), new ModelDebugConfig(80.0, "foo").hashCode()); + assertEquals(new ModelDebugConfig(DebugDestination.FILE, 80.0, "foo").hashCode(), new ModelDebugConfig(80.0, "foo").hashCode()); + assertEquals(new ModelDebugConfig(DebugDestination.DATA_STORE, 80.0, "foo").hashCode(), + new ModelDebugConfig(DebugDestination.DATA_STORE, 80.0, "foo").hashCode()); + } + + public void testVerify_GivenBoundPercentileLessThanZero() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new ModelDebugConfig(-1.0, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE, ""), e.getMessage()); + } + + public void testVerify_GivenBoundPercentileGreaterThan100() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new ModelDebugConfig(100.1, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE, ""), e.getMessage()); + } + + public void testVerify_GivenValid() { + new ModelDebugConfig(93.0, ""); + new ModelDebugConfig(93.0, "foo,bar"); + } + + @Override + protected ModelDebugConfig createTestInstance() { + return new ModelDebugConfig(randomFrom(DebugDestination.values()), randomDouble(), randomAsciiOfLengthBetween(1, 30)); + } + + @Override + protected Reader instanceReader() { + return ModelDebugConfig::new; + } + + @Override + protected ModelDebugConfig parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return ModelDebugConfig.PARSER.apply(parser, () -> matcher); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSizeStatsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSizeStatsTests.java new file mode 100644 index 00000000000..6f85a84b52f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSizeStatsTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats.MemoryStatus; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import java.util.Date; + +public class ModelSizeStatsTests extends AbstractSerializingTestCase { + + public void testDefaultConstructor() { + ModelSizeStats stats = new ModelSizeStats.Builder("foo").build(); + assertEquals("modelSizeStats", stats.getId()); + assertEquals(0, stats.getModelBytes()); + assertEquals(0, stats.getTotalByFieldCount()); + assertEquals(0, stats.getTotalOverFieldCount()); + assertEquals(0, stats.getTotalPartitionFieldCount()); + assertEquals(0, stats.getBucketAllocationFailuresCount()); + assertEquals(MemoryStatus.OK, stats.getMemoryStatus()); + } + + + public void testSetId() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + + stats.setId("bar"); + + assertEquals("bar", stats.build().getId()); + } + + + public void testSetMemoryStatus_GivenNull() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + + NullPointerException ex = expectThrows(NullPointerException.class, () -> stats.setMemoryStatus(null)); + + assertEquals("[memoryStatus] must not be null", ex.getMessage()); + } + + + public void testSetMemoryStatus_GivenSoftLimit() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + + stats.setMemoryStatus(MemoryStatus.SOFT_LIMIT); + + assertEquals(MemoryStatus.SOFT_LIMIT, stats.build().getMemoryStatus()); + } + + @Override + protected ModelSizeStats createTestInstance() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + if (randomBoolean()) { + stats.setBucketAllocationFailuresCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setModelBytes(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalByFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalOverFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalPartitionFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setLogTime(new Date(randomLong())); + } + if (randomBoolean()) { + stats.setTimestamp(new Date(randomLong())); + } + if (randomBoolean()) { + stats.setMemoryStatus(randomFrom(MemoryStatus.values())); + } + if (randomBoolean()) { + stats.setId(randomAsciiOfLengthBetween(1, 20)); + } + return stats.build(); + } + + @Override + protected Reader instanceReader() { + return ModelSizeStats::new; + } + + @Override + protected ModelSizeStats parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return ModelSizeStats.PARSER.apply(parser, () -> matcher).build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSnapshotTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSnapshotTests.java new file mode 100644 index 00000000000..61de472265d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/ModelSnapshotTests.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats.MemoryStatus; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; + +import java.util.Date; + +public class ModelSnapshotTests extends AbstractSerializingTestCase { + private static final Date DEFAULT_TIMESTAMP = new Date(); + private static final String DEFAULT_DESCRIPTION = "a snapshot"; + private static final String DEFAULT_ID = "my_id"; + private static final long DEFAULT_PRIORITY = 1234L; + private static final int DEFAULT_DOC_COUNT = 7; + private static final Date DEFAULT_LATEST_RESULT_TIMESTAMP = new Date(12345678901234L); + private static final Date DEFAULT_LATEST_RECORD_TIMESTAMP = new Date(12345678904321L); + + + public void testEquals_GivenSameObject() { + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + + assertTrue(modelSnapshot.equals(modelSnapshot)); + } + + + public void testEquals_GivenObjectOfDifferentClass() { + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + + assertFalse(modelSnapshot.equals("a string")); + } + + + public void testEquals_GivenEqualModelSnapshots() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setTimestamp(modelSnapshot1.getTimestamp()); + + assertTrue(modelSnapshot1.equals(modelSnapshot2)); + assertTrue(modelSnapshot2.equals(modelSnapshot1)); + assertEquals(modelSnapshot1.hashCode(), modelSnapshot2.hashCode()); + } + + + public void testEquals_GivenDifferentTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setTimestamp(new Date(modelSnapshot2.getTimestamp().getTime() + 1)); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentDescription() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setDescription(modelSnapshot2.getDescription() + " blah"); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentRestorePriority() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setRestorePriority(modelSnapshot2.getRestorePriority() + 1); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentId() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setSnapshotId(modelSnapshot2.getSnapshotId() + "_2"); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentDocCount() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setSnapshotDocCount(modelSnapshot2.getSnapshotDocCount() + 1); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentModelSizeStats() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + ModelSizeStats.Builder modelSizeStats = new ModelSizeStats.Builder("foo"); + modelSizeStats.setModelBytes(42L); + modelSnapshot2.setModelSizeStats(modelSizeStats); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentQuantiles() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setQuantiles(new Quantiles("foo", modelSnapshot2.getQuantiles().getTimestamp(), "different state")); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentLatestResultTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setLatestResultTimeStamp( + new Date(modelSnapshot2.getLatestResultTimeStamp().getTime() + 1)); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + public void testEquals_GivenDifferentLatestRecordTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated(); + ModelSnapshot modelSnapshot2 = createFullyPopulated(); + modelSnapshot2.setLatestRecordTimeStamp( + new Date(modelSnapshot2.getLatestRecordTimeStamp().getTime() + 1)); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + + private static ModelSnapshot createFullyPopulated() { + ModelSnapshot modelSnapshot = new ModelSnapshot("foo"); + modelSnapshot.setTimestamp(DEFAULT_TIMESTAMP); + modelSnapshot.setDescription(DEFAULT_DESCRIPTION); + modelSnapshot.setRestorePriority(DEFAULT_PRIORITY); + modelSnapshot.setSnapshotId(DEFAULT_ID); + modelSnapshot.setSnapshotDocCount(DEFAULT_DOC_COUNT); + modelSnapshot.setModelSizeStats(new ModelSizeStats.Builder("foo")); + modelSnapshot.setLatestResultTimeStamp(DEFAULT_LATEST_RESULT_TIMESTAMP); + modelSnapshot.setLatestRecordTimeStamp(DEFAULT_LATEST_RECORD_TIMESTAMP); + modelSnapshot.setQuantiles(new Quantiles("foo", DEFAULT_TIMESTAMP, "state")); + return modelSnapshot; + } + + @Override + protected ModelSnapshot createTestInstance() { + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setTimestamp(new Date(TimeUtils.dateStringToEpoch(randomTimeValue()))); + modelSnapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setRestorePriority(randomLong()); + modelSnapshot.setSnapshotId(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setSnapshotDocCount(randomInt()); + ModelSizeStats.Builder stats = new ModelSizeStats.Builder(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + stats.setBucketAllocationFailuresCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setModelBytes(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalByFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalOverFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setTotalPartitionFieldCount(randomPositiveLong()); + } + if (randomBoolean()) { + stats.setLogTime(new Date(randomLong())); + } + if (randomBoolean()) { + stats.setTimestamp(new Date(randomLong())); + } + if (randomBoolean()) { + stats.setMemoryStatus(randomFrom(MemoryStatus.values())); + } + if (randomBoolean()) { + stats.setId(randomAsciiOfLengthBetween(1, 20)); + } + modelSnapshot.setModelSizeStats(stats); + modelSnapshot.setLatestResultTimeStamp(new Date(TimeUtils.dateStringToEpoch(randomTimeValue()))); + modelSnapshot.setLatestRecordTimeStamp(new Date(TimeUtils.dateStringToEpoch(randomTimeValue()))); + Quantiles quantiles = + new Quantiles("foo", new Date(TimeUtils.dateStringToEpoch(randomTimeValue())), randomAsciiOfLengthBetween(0, 1000)); + modelSnapshot.setQuantiles(quantiles); + return modelSnapshot; + } + + @Override + protected Reader instanceReader() { + return ModelSnapshot::new; + } + + @Override + protected ModelSnapshot parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return ModelSnapshot.PARSER.apply(parser, () -> matcher); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerConfigTests.java new file mode 100644 index 00000000000..b6c82c63fbe --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerConfigTests.java @@ -0,0 +1,565 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig.DataSource; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SchedulerConfigTests extends AbstractSerializingTestCase { + + @Override + protected SchedulerConfig createTestInstance() { + DataSource dataSource = randomFrom(DataSource.values()); + SchedulerConfig.Builder builder = new SchedulerConfig.Builder(dataSource); + switch (dataSource) { + case FILE: + builder.setFilePath(randomAsciiOfLength(10)); + builder.setTailFile(randomBoolean()); + break; + case ELASTICSEARCH: + builder.setBaseUrl("http://localhost/" + randomAsciiOfLength(10)); + if (randomBoolean()) { + builder.setQuery(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + boolean retrieveWholeSource = randomBoolean(); + if (retrieveWholeSource) { + builder.setRetrieveWholeSource(randomBoolean()); + } else if (randomBoolean()) { + builder.setScriptFields(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + if (randomBoolean()) { + builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + builder.setUsername(randomAsciiOfLength(10)); + if (randomBoolean()) { + builder.setEncryptedPassword(randomAsciiOfLength(10)); + } else { + builder.setPassword(randomAsciiOfLength(10)); + } + } + builder.setIndexes(randomStringList(1, 10)); + builder.setTypes(randomStringList(1, 10)); + if (randomBoolean()) { + builder.setAggregations(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } else if (randomBoolean()) { + builder.setAggs(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + break; + default: + throw new UnsupportedOperationException(); + } + if (randomBoolean()) { + builder.setFrequency(randomPositiveLong()); + } + if (randomBoolean()) { + builder.setQueryDelay(randomPositiveLong()); + } + return builder.build(); + } + + private static List randomStringList(int min, int max) { + int size = scaledRandomIntBetween(min, max); + List list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add(randomAsciiOfLength(10)); + } + return list; + } + + @Override + protected Writeable.Reader instanceReader() { + return SchedulerConfig::new; + } + + @Override + protected SchedulerConfig parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return SchedulerConfig.PARSER.apply(parser, () -> matcher).build(); + } + + /** + * Test parsing of the opaque {@link SchedulerConfig#getQuery()} object + */ + public void testAnalysisConfigRequiredFields() throws IOException { + Logger logger = Loggers.getLogger(SchedulerConfigTests.class); + + String jobConfigStr = "{" + "\"jobId\":\"farequote\"," + "\"schedulerConfig\" : {" + "\"dataSource\":\"ELASTICSEARCH\"," + + "\"baseUrl\":\"http://localhost:9200/\"," + "\"indexes\":[\"farequote\"]," + "\"types\":[\"farequote\"]," + + "\"query\":{\"match_all\":{} }" + "}," + "\"analysisConfig\" : {" + "\"bucketSpan\":3600," + + "\"detectors\" :[{\"function\":\"metric\",\"fieldName\":\"responsetime\",\"byFieldName\":\"airline\"}]," + + "\"influencers\" :[\"airline\"]" + "}," + "\"dataDescription\" : {" + "\"format\":\"ELASTICSEARCH\"," + + "\"timeField\":\"@timestamp\"," + "\"timeFormat\":\"epoch_ms\"" + "}" + "}"; + + XContentParser parser = XContentFactory.xContent(jobConfigStr).createParser(jobConfigStr); + Job jobConfig = Job.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT).build(); + assertNotNull(jobConfig); + + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(jobConfig.getSchedulerConfig()); + assertNotNull(schedulerConfig); + + Map query = schedulerConfig.getQuery(); + assertNotNull(query); + + String queryAsJson = XContentFactory.jsonBuilder().map(query).string(); + logger.info("Round trip of query is: " + queryAsJson); + assertTrue(query.containsKey("match_all")); + } + + public void testBuildAggregatedFieldList_GivenNoAggregations() { + SchedulerConfig.Builder builder = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + builder.setIndexes(Arrays.asList("index")); + builder.setTypes(Arrays.asList("type")); + builder.setBaseUrl("http://localhost/"); + assertTrue(builder.build().buildAggregatedFieldList().isEmpty()); + } + + /** + * Test parsing of the opaque {@link SchedulerConfig#getAggs()} object + */ + + public void testAggsParse() throws IOException { + Logger logger = Loggers.getLogger(SchedulerConfigTests.class); + + String jobConfigStr = "{" + "\"jobId\":\"farequote\"," + "\"schedulerConfig\" : {" + "\"dataSource\":\"ELASTICSEARCH\"," + + "\"baseUrl\":\"http://localhost:9200/\"," + "\"indexes\":[\"farequote\"]," + "\"types\":[\"farequote\"]," + + "\"query\":{\"match_all\":{} }," + "\"aggs\" : {" + "\"top_level_must_be_time\" : {" + "\"histogram\" : {" + + "\"field\" : \"@timestamp\"," + "\"interval\" : 3600000" + "}," + "\"aggs\" : {" + "\"by_field_in_the_middle\" : { " + + "\"terms\" : {" + "\"field\" : \"airline\"," + "\"size\" : 0" + "}," + "\"aggs\" : {" + "\"stats_last\" : {" + + "\"avg\" : {" + "\"field\" : \"responsetime\"" + "}" + "}" + "} " + "}" + "}" + "}" + "}" + "}," + + "\"analysisConfig\" : {" + "\"summaryCountFieldName\":\"doc_count\"," + "\"bucketSpan\":3600," + + "\"detectors\" :[{\"function\":\"avg\",\"fieldName\":\"responsetime\",\"byFieldName\":\"airline\"}]," + + "\"influencers\" :[\"airline\"]" + "}," + "\"dataDescription\" : {" + "\"format\":\"ELASTICSEARCH\"," + + "\"timeField\":\"@timestamp\"," + "\"timeFormat\":\"epoch_ms\"" + "}" + "}"; + + XContentParser parser = XContentFactory.xContent(jobConfigStr).createParser(jobConfigStr); + Job jobConfig = Job.PARSER.parse(parser, () -> ParseFieldMatcher.STRICT).build(); + assertNotNull(jobConfig); + + SchedulerConfig schedulerConfig = jobConfig.getSchedulerConfig(); + assertNotNull(schedulerConfig); + + Map aggs = schedulerConfig.getAggregationsOrAggs(); + assertNotNull(aggs); + + String aggsAsJson = XContentFactory.jsonBuilder().map(aggs).string(); + logger.info("Round trip of aggs is: " + aggsAsJson); + assertTrue(aggs.containsKey("top_level_must_be_time")); + + List aggregatedFieldList = schedulerConfig.buildAggregatedFieldList(); + assertEquals(3, aggregatedFieldList.size()); + assertEquals("@timestamp", aggregatedFieldList.get(0)); + assertEquals("airline", aggregatedFieldList.get(1)); + assertEquals("responsetime", aggregatedFieldList.get(2)); + } + + public void testFillDefaults_GivenDataSourceIsFile() { + SchedulerConfig.Builder schedulerConfig = new SchedulerConfig.Builder(DataSource.FILE); + schedulerConfig.setFilePath("/some/path"); + SchedulerConfig.Builder expectedSchedulerConfig = new SchedulerConfig.Builder(DataSource.FILE); + expectedSchedulerConfig.setFilePath("/some/path"); + expectedSchedulerConfig.setTailFile(false); + assertEquals(expectedSchedulerConfig.build(), schedulerConfig.build()); + } + + public void testFillDefaults_GivenDataSourceIsElasticsearchAndNothingToFill() { + SchedulerConfig.Builder originalSchedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + originalSchedulerConfig.setBaseUrl("http://localhost:9200/"); + originalSchedulerConfig.setQuery(new HashMap<>()); + originalSchedulerConfig.setQueryDelay(30L); + originalSchedulerConfig.setRetrieveWholeSource(true); + originalSchedulerConfig.setScrollSize(2000); + originalSchedulerConfig.setIndexes(Arrays.asList("index")); + originalSchedulerConfig.setTypes(Arrays.asList("type")); + + SchedulerConfig.Builder defaultedSchedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + defaultedSchedulerConfig.setBaseUrl("http://localhost:9200/"); + defaultedSchedulerConfig.setQuery(new HashMap<>()); + defaultedSchedulerConfig.setQueryDelay(30L); + defaultedSchedulerConfig.setRetrieveWholeSource(true); + defaultedSchedulerConfig.setScrollSize(2000); + defaultedSchedulerConfig.setIndexes(Arrays.asList("index")); + defaultedSchedulerConfig.setTypes(Arrays.asList("type")); + + assertEquals(originalSchedulerConfig.build(), defaultedSchedulerConfig.build()); + } + + public void testFillDefaults_GivenDataSourceIsElasticsearchAndDefaultsAreApplied() { + SchedulerConfig.Builder expectedSchedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + expectedSchedulerConfig.setIndexes(Arrays.asList("index")); + expectedSchedulerConfig.setTypes(Arrays.asList("type")); + expectedSchedulerConfig.setBaseUrl("http://localhost:9200/"); + Map defaultQuery = new HashMap<>(); + defaultQuery.put("match_all", new HashMap()); + expectedSchedulerConfig.setQuery(defaultQuery); + expectedSchedulerConfig.setQueryDelay(60L); + expectedSchedulerConfig.setRetrieveWholeSource(false); + expectedSchedulerConfig.setScrollSize(1000); + SchedulerConfig.Builder defaultedSchedulerConfig = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + defaultedSchedulerConfig.setBaseUrl("http://localhost:9200/"); + defaultedSchedulerConfig.setIndexes(Arrays.asList("index")); + defaultedSchedulerConfig.setTypes(Arrays.asList("type")); + assertEquals(expectedSchedulerConfig.build(), defaultedSchedulerConfig.build()); + } + + public void testEquals_GivenDifferentClass() { + SchedulerConfig.Builder builder = new SchedulerConfig.Builder(DataSource.FILE); + builder.setFilePath("path"); + assertFalse(builder.build().equals("a string")); + } + + public void testEquals_GivenSameRef() { + SchedulerConfig.Builder builder = new SchedulerConfig.Builder(DataSource.FILE); + builder.setFilePath("/some/path"); + SchedulerConfig schedulerConfig = builder.build(); + assertTrue(schedulerConfig.equals(schedulerConfig)); + } + + public void testEquals_GivenEqual() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertTrue(sc1.equals(sc2)); + assertTrue(sc2.equals(sc1)); + assertEquals(sc1.hashCode(), sc2.hashCode()); + } + + public void testEquals_GivenDifferentBaseUrl() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + b2.setBaseUrl("http://localhost:8081"); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentQueryDelay() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + b2.setQueryDelay(120L); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentScrollSize() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + b2.setScrollSize(1); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentFrequency() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + b2.setFrequency(120L); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentIndexes() { + SchedulerConfig.Builder sc1 = createFullyPopulated(); + SchedulerConfig.Builder sc2 = createFullyPopulated(); + sc2.setIndexes(Arrays.asList("thisOtherCrazyIndex")); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + public void testEquals_GivenDifferentTypes() { + SchedulerConfig.Builder sc1 = createFullyPopulated(); + SchedulerConfig.Builder sc2 = createFullyPopulated(); + sc2.setTypes(Arrays.asList("thisOtherCrazyType")); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + public void testEquals_GivenDifferentQuery() { + SchedulerConfig.Builder b1 = createFullyPopulated(); + SchedulerConfig.Builder b2 = createFullyPopulated(); + Map emptyQuery = new HashMap<>(); + b2.setQuery(emptyQuery); + + SchedulerConfig sc1 = b1.build(); + SchedulerConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentAggregations() { + SchedulerConfig.Builder sc1 = createFullyPopulated(); + SchedulerConfig.Builder sc2 = createFullyPopulated(); + Map emptyAggs = new HashMap<>(); + sc2.setAggregations(emptyAggs); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + private static SchedulerConfig.Builder createFullyPopulated() { + SchedulerConfig.Builder sc = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + sc.setBaseUrl("http://localhost:8080"); + sc.setFrequency(60L); + sc.setScrollSize(5000); + sc.setIndexes(Arrays.asList("myIndex")); + sc.setTypes(Arrays.asList("myType1", "myType2")); + Map query = new HashMap<>(); + query.put("foo", new HashMap<>()); + sc.setQuery(query); + Map aggs = new HashMap<>(); + aggs.put("bar", new HashMap<>()); + sc.setAggregations(aggs); + sc.setQueryDelay(90L); + return sc; + } + + public void testCheckValidFile_AllOk() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.FILE); + conf.setFilePath("myfile.csv"); + conf.build(); + } + + public void testCheckValidFile_NoPath() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.FILE); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "filePath", "null"), e.getMessage()); + } + + public void testCheckValidFile_EmptyPath() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.FILE); + conf.setFilePath(""); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "filePath", ""), e.getMessage()); + } + + public void testCheckValidFile_InappropriateField() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.FILE); + conf.setFilePath("myfile.csv"); + conf.setBaseUrl("http://localhost:9200/"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED, "baseUrl", DataSource.FILE), e.getMessage()); + } + + public void testCheckValidElasticsearch_AllOk() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setQueryDelay(90L); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + String json = "{ \"match_all\" : {} }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setQuery(parser.map()); + conf.setScrollSize(2000); + conf.build(); + } + + public void testCheckValidElasticsearch_WithUsernameAndPassword() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setQueryDelay(90L); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + conf.setUsername("dave"); + conf.setPassword("secret"); + String json = "{ \"match_all\" : {} }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setQuery(parser.map()); + SchedulerConfig schedulerConfig = conf.build(); + assertEquals("dave", schedulerConfig.getUsername()); + assertEquals("secret", schedulerConfig.getPassword()); + } + + public void testCheckValidElasticsearch_WithUsernameAndEncryptedPassword() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setQueryDelay(90L); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + conf.setUsername("dave"); + conf.setEncryptedPassword("already_encrypted"); + String json = "{ \"match_all\" : {} }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setQuery(parser.map()); + SchedulerConfig schedulerConfig = conf.build(); + assertEquals("dave", schedulerConfig.getUsername()); + assertEquals("already_encrypted", schedulerConfig.getEncryptedPassword()); + } + + public void testCheckValidElasticsearch_WithPasswordNoUsername() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + conf.setPassword("secret"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INCOMPLETE_CREDENTIALS), e.getMessage()); + } + + public void testCheckValidElasticsearch_BothPasswordAndEncryptedPassword() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + conf.setUsername("dave"); + conf.setPassword("secret"); + conf.setEncryptedPassword("already_encrypted"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_MULTIPLE_PASSWORDS), e.getMessage()); + } + + public void testCheckValidElasticsearch_NoQuery() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + assertEquals(Collections.singletonMap("match_all", new HashMap<>()), conf.build().getQuery()); + } + + public void testCheckValidElasticsearch_InappropriateField() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + String json = "{ \"match_all\" : {} }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setQuery(parser.map()); + conf.setTailFile(true); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_FIELD_NOT_SUPPORTED, "tailFile", DataSource.ELASTICSEARCH), + e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenScriptFieldsNotWholeSource() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + String json = "{ \"twiceresponsetime\" : { \"script\" : { \"lang\" : \"expression\", " + + "\"inline\" : \"doc['responsetime'].value * 2\" } } }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setScriptFields(parser.map()); + conf.setRetrieveWholeSource(false); + assertEquals(1, conf.build().getScriptFields().size()); + } + + public void testCheckValidElasticsearch_GivenScriptFieldsAndWholeSource() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myindex")); + conf.setTypes(Arrays.asList("mytype")); + String json = "{ \"twiceresponsetime\" : { \"script\" : { \"lang\" : \"expression\", " + + "\"inline\" : \"doc['responsetime'].value * 2\" } } }"; + XContentParser parser = XContentFactory.xContent(json).createParser(json); + conf.setScriptFields(parser.map()); + conf.setRetrieveWholeSource(true); + expectThrows(IllegalArgumentException.class, conf::build); + } + + public void testCheckValidElasticsearch_GivenNullIndexes() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + expectThrows(NullPointerException.class, () -> conf.setIndexes(null)); + } + + public void testCheckValidElasticsearch_GivenEmptyIndexes() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Collections.emptyList()); + conf.setTypes(Arrays.asList("mytype")); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "indexes", "[]"), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenIndexesContainsOnlyNulls() throws IOException { + List indexes = new ArrayList<>(); + indexes.add(null); + indexes.add(null); + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(indexes); + conf.setTypes(Arrays.asList("mytype")); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "indexes", "[null, null]"), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenIndexesContainsOnlyEmptyStrings() throws IOException { + List indexes = new ArrayList<>(); + indexes.add(""); + indexes.add(""); + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(indexes); + conf.setTypes(Arrays.asList("mytype")); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "indexes", "[, ]"), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenNegativeQueryDelay() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setQueryDelay(-10L)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "queryDelay", -10L), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenZeroFrequency() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(0L)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "frequency", 0L), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenNegativeFrequency() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(-600L)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "frequency", -600L), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenNegativeScrollSize() throws IOException { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setScrollSize(-1000)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_INVALID_OPTION_VALUE, "scrollSize", -1000L), e.getMessage()); + } + + public void testCheckValidElasticsearch_GivenBothAggregationsAndAggsAreSet() { + SchedulerConfig.Builder conf = new SchedulerConfig.Builder(DataSource.ELASTICSEARCH); + conf.setScrollSize(1000); + conf.setBaseUrl("http://localhost:9200/"); + conf.setIndexes(Arrays.asList("myIndex")); + conf.setTypes(Arrays.asList("mytype")); + Map aggs = new HashMap<>(); + conf.setAggregations(aggs); + conf.setAggs(aggs); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_SCHEDULER_MULTIPLE_AGGREGATIONS), e.getMessage()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerStateTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerStateTests.java new file mode 100644 index 00000000000..32a18a2fe7c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/SchedulerStateTests.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class SchedulerStateTests extends AbstractSerializingTestCase { + + @Override + protected SchedulerState createTestInstance() { + return new SchedulerState(randomFrom(JobSchedulerStatus.values()), randomPositiveLong(), + randomBoolean() ? null : randomPositiveLong()); + } + + @Override + protected Writeable.Reader instanceReader() { + return SchedulerState::new; + } + + @Override + protected SchedulerState parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return SchedulerState.PARSER.apply(parser, () -> matcher); + } + + public void testEquals_GivenDifferentClass() { + + assertFalse(new SchedulerState(JobSchedulerStatus.STARTED, 0, null).equals("a string")); + } + + public void testEquals_GivenSameReference() { + SchedulerState schedulerState = new SchedulerState(JobSchedulerStatus.STARTED, 18L, 42L); + assertTrue(schedulerState.equals(schedulerState)); + } + + public void testEquals_GivenEqualObjects() { + SchedulerState schedulerState1 = new SchedulerState(JobSchedulerStatus.STARTED, 18L, 42L); + SchedulerState schedulerState2 = new SchedulerState(schedulerState1.getStatus(), schedulerState1.getStartTimeMillis(), + schedulerState1.getEndTimeMillis()); + + assertTrue(schedulerState1.equals(schedulerState2)); + assertTrue(schedulerState2.equals(schedulerState1)); + assertEquals(schedulerState1.hashCode(), schedulerState2.hashCode()); + } + + public void testEquals_GivenDifferentStatus() { + SchedulerState schedulerState1 = new SchedulerState(JobSchedulerStatus.STARTED, 18L, 42L); + SchedulerState schedulerState2 = new SchedulerState(JobSchedulerStatus.STOPPED, schedulerState1.getStartTimeMillis(), + schedulerState1.getEndTimeMillis()); + + assertFalse(schedulerState1.equals(schedulerState2)); + assertFalse(schedulerState2.equals(schedulerState1)); + } + + public void testEquals_GivenDifferentStartTimeMillis() { + SchedulerState schedulerState1 = new SchedulerState(JobSchedulerStatus.STARTED, 18L, 42L); + SchedulerState schedulerState2 = new SchedulerState(JobSchedulerStatus.STOPPED, 19L, schedulerState1.getEndTimeMillis()); + + assertFalse(schedulerState1.equals(schedulerState2)); + assertFalse(schedulerState2.equals(schedulerState1)); + } + + public void testEquals_GivenDifferentEndTimeMillis() { + SchedulerState schedulerState1 = new SchedulerState(JobSchedulerStatus.STARTED, 18L, 42L); + SchedulerState schedulerState2 = new SchedulerState(JobSchedulerStatus.STOPPED, schedulerState1.getStartTimeMillis(), 43L); + + assertFalse(schedulerState1.equals(schedulerState2)); + assertFalse(schedulerState2.equals(schedulerState1)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivityTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivityTests.java new file mode 100644 index 00000000000..0f40c3d9e76 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditActivityTests.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; +import org.junit.Before; + +import java.util.Date; + +public class AuditActivityTests extends AbstractSerializingTestCase { + private long startMillis; + + @Before + public void setStartTime() { + startMillis = System.currentTimeMillis(); + } + + public void testDefaultConstructor() { + AuditActivity activity = new AuditActivity(); + assertEquals(0, activity.getTotalJobs()); + assertEquals(0, activity.getTotalDetectors()); + assertEquals(0, activity.getRunningJobs()); + assertEquals(0, activity.getRunningDetectors()); + assertNull(activity.getTimestamp()); + } + + public void testNewActivity() { + AuditActivity activity = AuditActivity.newActivity(10, 100, 5, 50); + assertEquals(10, activity.getTotalJobs()); + assertEquals(100, activity.getTotalDetectors()); + assertEquals(5, activity.getRunningJobs()); + assertEquals(50, activity.getRunningDetectors()); + assertDateBetweenStartAndNow(activity.getTimestamp()); + } + + private void assertDateBetweenStartAndNow(Date timestamp) { + long timestampMillis = timestamp.getTime(); + assertTrue(timestampMillis >= startMillis); + assertTrue(timestampMillis <= System.currentTimeMillis()); + } + + @Override + protected AuditActivity parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return AuditActivity.PARSER.apply(parser, () -> matcher); + } + + @Override + protected AuditActivity createTestInstance() { + AuditActivity message = new AuditActivity(); + if (randomBoolean()) { + message.setRunningJobs(randomInt()); + } + if (randomBoolean()) { + message.setRunningDetectors(randomInt()); + } + if (randomBoolean()) { + message.setTotalJobs(randomInt()); + } + if (randomBoolean()) { + message.setTotalDetectors(randomInt()); + } + if (randomBoolean()) { + message.setTimestamp(new Date(TimeUtils.dateStringToEpoch(randomTimeValue()))); + } + return message; + } + + @Override + protected Reader instanceReader() { + return AuditActivity::new; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessageTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessageTests.java new file mode 100644 index 00000000000..15ee79eabfa --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/AuditMessageTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; +import org.elasticsearch.xpack.prelert.utils.time.TimeUtils; +import org.junit.Before; + +import java.util.Date; + +public class AuditMessageTests extends AbstractSerializingTestCase { + private long startMillis; + + @Before + public void setStartTime() { + startMillis = System.currentTimeMillis(); + } + + public void testDefaultConstructor() { + AuditMessage auditMessage = new AuditMessage(); + assertNull(auditMessage.getMessage()); + assertNull(auditMessage.getLevel()); + assertNull(auditMessage.getTimestamp()); + } + + public void testNewInfo() { + AuditMessage info = AuditMessage.newInfo("foo", "some info"); + assertEquals("foo", info.getJobId()); + assertEquals("some info", info.getMessage()); + assertEquals(Level.INFO, info.getLevel()); + assertDateBetweenStartAndNow(info.getTimestamp()); + } + + public void testNewWarning() { + AuditMessage warning = AuditMessage.newWarning("bar", "some warning"); + assertEquals("bar", warning.getJobId()); + assertEquals("some warning", warning.getMessage()); + assertEquals(Level.WARNING, warning.getLevel()); + assertDateBetweenStartAndNow(warning.getTimestamp()); + } + + + public void testNewError() { + AuditMessage error = AuditMessage.newError("foo", "some error"); + assertEquals("foo", error.getJobId()); + assertEquals("some error", error.getMessage()); + assertEquals(Level.ERROR, error.getLevel()); + assertDateBetweenStartAndNow(error.getTimestamp()); + } + + public void testNewActivity() { + AuditMessage error = AuditMessage.newActivity("foo", "some error"); + assertEquals("foo", error.getJobId()); + assertEquals("some error", error.getMessage()); + assertEquals(Level.ACTIVITY, error.getLevel()); + assertDateBetweenStartAndNow(error.getTimestamp()); + } + + private void assertDateBetweenStartAndNow(Date timestamp) { + long timestampMillis = timestamp.getTime(); + assertTrue(timestampMillis >= startMillis); + assertTrue(timestampMillis <= System.currentTimeMillis()); + } + + @Override + protected AuditMessage parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return AuditMessage.PARSER.apply(parser, () -> matcher); + } + + @Override + protected AuditMessage createTestInstance() { + AuditMessage message = new AuditMessage(); + if (randomBoolean()) { + message.setJobId(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + message.setMessage(randomAsciiOfLengthBetween(1, 200)); + } + if (randomBoolean()) { + message.setLevel(randomFrom(Level.values())); + } + if (randomBoolean()) { + message.setTimestamp(new Date(TimeUtils.dateStringToEpoch(randomTimeValue()))); + } + return message; + } + + @Override + protected Reader instanceReader() { + return AuditMessage::new; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/LevelTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/LevelTests.java new file mode 100644 index 00000000000..7d3443e1438 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/audit/LevelTests.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.audit; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class LevelTests extends ESTestCase { + + public void testForString() { + assertEquals(Level.INFO, Level.forString("info")); + assertEquals(Level.INFO, Level.forString("INFO")); + assertEquals(Level.ACTIVITY, Level.forString("activity")); + assertEquals(Level.ACTIVITY, Level.forString("ACTIVITY")); + assertEquals(Level.WARNING, Level.forString("warning")); + assertEquals(Level.WARNING, Level.forString("WARNING")); + assertEquals(Level.ERROR, Level.forString("error")); + assertEquals(Level.ERROR, Level.forString("ERROR")); + } + + public void testValidOrdinals() { + assertThat(Level.INFO.ordinal(), equalTo(0)); + assertThat(Level.ACTIVITY.ordinal(), equalTo(1)); + assertThat(Level.WARNING.ordinal(), equalTo(2)); + assertThat(Level.ERROR.ordinal(), equalTo(3)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Level.INFO.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Level.ACTIVITY.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Level.WARNING.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Level.ERROR.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Level.readFromStream(in), equalTo(Level.INFO)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Level.readFromStream(in), equalTo(Level.ACTIVITY)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Level.readFromStream(in), equalTo(Level.WARNING)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Level.readFromStream(in), equalTo(Level.ERROR)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + Level.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown Level ordinal [")); + } + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/ConditionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/ConditionTests.java new file mode 100644 index 00000000000..b3c1c4aa286 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/ConditionTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.condition; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; + +public class ConditionTests extends AbstractSerializingTestCase { + + public void testSetValues() { + Condition cond = new Condition(Operator.EQ, "5"); + assertEquals(Operator.EQ, cond.getOperator()); + assertEquals("5", cond.getValue()); + } + + public void testHashCodeAndEquals() { + Condition cond1 = new Condition(Operator.MATCH, "regex"); + Condition cond2 = new Condition(Operator.MATCH, "regex"); + + assertEquals(cond1, cond2); + assertEquals(cond1.hashCode(), cond2.hashCode()); + + Condition cond3 = new Condition(Operator.EQ, "5"); + assertFalse(cond1.equals(cond3)); + assertFalse(cond1.hashCode() == cond3.hashCode()); + } + + @Override + protected Condition createTestInstance() { + Operator op = randomFrom(Operator.values()); + Condition condition; + switch (op) { + case EQ: + case GT: + case GTE: + case LT: + case LTE: + condition = new Condition(op, Double.toString(randomDouble())); + break; + case MATCH: + condition = new Condition(op, randomAsciiOfLengthBetween(1, 20)); + break; + default: + throw new AssertionError("Unknown operator selected: " + op.getName()); + } + return condition; + } + + @Override + protected Reader instanceReader() { + return Condition::new; + } + + @Override + protected Condition parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Condition.PARSER.apply(parser, () -> matcher); + } + + public void testInvalidTransformName() throws Exception { + BytesArray json = new BytesArray("{ \"value\":\"someValue\" }"); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, + () -> Condition.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT)); + assertThat(ex.getMessage(), containsString("Required [operator]")); + } + + public void testVerifyArgsNumericArgs() { + new Condition(Operator.LTE, "100"); + new Condition(Operator.GT, "10.0"); + } + + public void testVerify_GivenEmptyValue() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new Condition(Operator.LT, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, ""), e.getMessage()); + } + + public void testVerify_GivenInvalidRegex() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new Condition(Operator.MATCH, "[*")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, "[*"), e.getMessage()); + } + + public void testVerify_GivenNullRegex() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, + () -> new Condition(Operator.MATCH, null)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL, "[*"), e.getMessage()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/OperatorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/OperatorTests.java new file mode 100644 index 00000000000..13ad557a90e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/condition/OperatorTests.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.condition; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.messages.Messages; + +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class OperatorTests extends ESTestCase { + + + public void testFromString() { + assertEquals(Operator.fromString("eq"), Operator.EQ); + assertEquals(Operator.fromString("gt"), Operator.GT); + assertEquals(Operator.fromString("gte"), Operator.GTE); + assertEquals(Operator.fromString("lte"), Operator.LTE); + assertEquals(Operator.fromString("lt"), Operator.LT); + assertEquals(Operator.fromString("match"), Operator.MATCH); + assertEquals(Operator.fromString("Gt"), Operator.GT); + assertEquals(Operator.fromString("EQ"), Operator.EQ); + assertEquals(Operator.fromString("GTE"), Operator.GTE); + assertEquals(Operator.fromString("Match"), Operator.MATCH); + + } + + + public void testTest() { + assertTrue(Operator.GT.expectsANumericArgument()); + assertTrue(Operator.GT.test(1.0, 0.0)); + assertFalse(Operator.GT.test(0.0, 1.0)); + + assertTrue(Operator.GTE.expectsANumericArgument()); + assertTrue(Operator.GTE.test(1.0, 0.0)); + assertTrue(Operator.GTE.test(1.0, 1.0)); + assertFalse(Operator.GTE.test(0.0, 1.0)); + + assertTrue(Operator.EQ.expectsANumericArgument()); + assertTrue(Operator.EQ.test(0.0, 0.0)); + assertFalse(Operator.EQ.test(1.0, 0.0)); + + assertTrue(Operator.LT.expectsANumericArgument()); + assertTrue(Operator.LT.test(0.0, 1.0)); + assertFalse(Operator.LT.test(0.0, 0.0)); + + assertTrue(Operator.LTE.expectsANumericArgument()); + assertTrue(Operator.LTE.test(0.0, 1.0)); + assertTrue(Operator.LTE.test(1.0, 1.0)); + assertFalse(Operator.LTE.test(1.0, 0.0)); + } + + + public void testMatch() { + assertFalse(Operator.MATCH.expectsANumericArgument()); + assertFalse(Operator.MATCH.test(0.0, 1.0)); + + Pattern pattern = Pattern.compile("^aa.*"); + + assertTrue(Operator.MATCH.match(pattern, "aaaaa")); + assertFalse(Operator.MATCH.match(pattern, "bbaaa")); + } + + public void testValidOrdinals() { + assertThat(Operator.EQ.ordinal(), equalTo(0)); + assertThat(Operator.GT.ordinal(), equalTo(1)); + assertThat(Operator.GTE.ordinal(), equalTo(2)); + assertThat(Operator.LT.ordinal(), equalTo(3)); + assertThat(Operator.LTE.ordinal(), equalTo(4)); + assertThat(Operator.MATCH.ordinal(), equalTo(5)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.EQ.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.GT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.GTE.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.LT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.LTE.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(4)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.MATCH.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(5)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.EQ)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.GT)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.GTE)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.LT)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(4); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.LTE)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(5); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.MATCH)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(7, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + Operator.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown Operator ordinal [")); + } + } + } + + public void testVerify_unknownOp() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> Operator.fromString("bad_op")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_UNKNOWN_OPERATOR, "bad_op"), e.getMessage()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescriptionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescriptionTests.java new file mode 100644 index 00000000000..a9308b546ea --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultDetectorDescriptionTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.config; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.Detector; + +public class DefaultDetectorDescriptionTests extends ESTestCase { + + + public void testOf_GivenOnlyFunctionAndFieldName() { + Detector detector = new Detector.Builder("min", "value").build(); + + assertEquals("min(value)", DefaultDetectorDescription.of(detector)); + } + + + public void testOf_GivenOnlyFunctionAndFieldNameWithNonWordChars() { + Detector detector = new Detector.Builder("min", "val-ue").build(); + + assertEquals("min(\"val-ue\")", DefaultDetectorDescription.of(detector)); + } + + + public void testOf_GivenFullyPopulatedDetector() { + Detector.Builder detector = new Detector.Builder("sum", "value"); + detector.setByFieldName("airline"); + detector.setOverFieldName("region"); + detector.setUseNull(true); + detector.setPartitionFieldName("planet"); + detector.setExcludeFrequent(Detector.ExcludeFrequent.ALL); + + assertEquals("sum(value) by airline over region usenull=true partitionfield=planet excludefrequent=all", + DefaultDetectorDescription.of(detector.build())); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequencyTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequencyTests.java new file mode 100644 index 00000000000..96f07d7b8ad --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/config/DefaultFrequencyTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.config; + +import org.elasticsearch.test.ESTestCase; + +import java.time.Duration; + +public class DefaultFrequencyTests extends ESTestCase { + + public void testCalc_GivenNegative() { + ESTestCase.expectThrows(IllegalArgumentException.class, () -> DefaultFrequency.ofBucketSpan(-1)); + } + + + public void testCalc() { + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(1)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(30)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(60)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(90)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(120)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(121)); + + assertEquals(Duration.ofSeconds(61), DefaultFrequency.ofBucketSpan(122)); + assertEquals(Duration.ofSeconds(75), DefaultFrequency.ofBucketSpan(150)); + assertEquals(Duration.ofSeconds(150), DefaultFrequency.ofBucketSpan(300)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1200)); + + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1201)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1800)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(3600)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(7200)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(12 * 3600)); + + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(12 * 3600 + 1)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(13 * 3600)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(24 * 3600)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(48 * 3600)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerTests.java new file mode 100644 index 00000000000..f0277eff3f3 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerTests.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.data; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DataStreamerTests extends ESTestCase { + + public void testConstructor_GivenNullDataProcessor() { + + ESTestCase.expectThrows(NullPointerException.class, () -> new DataStreamer(null)); + } + + public void testStreamData_GivenNoContentEncodingAndNoPersistBaseDir() throws IOException { + + DataProcessor dataProcessor = mock(DataProcessor.class); + DataStreamer dataStreamer = new DataStreamer(dataProcessor); + InputStream inputStream = mock(InputStream.class); + DataLoadParams params = mock(DataLoadParams.class); + + when(dataProcessor.processData("foo", inputStream, params)).thenReturn(new DataCounts("foo")); + + dataStreamer.streamData("", "foo", inputStream, params); + + verify(dataProcessor).processData("foo", inputStream, params); + Mockito.verifyNoMoreInteractions(dataProcessor); + } + + public void testStreamData_ExpectsGzipButNotCompressed() throws IOException { + DataProcessor dataProcessor = mock(DataProcessor.class); + DataStreamer dataStreamer = new DataStreamer(dataProcessor); + InputStream inputStream = mock(InputStream.class); + DataLoadParams params = mock(DataLoadParams.class); + + try { + dataStreamer.streamData("gzip", "foo", inputStream, params); + fail("content encoding : gzip with uncompressed data should throw"); + } catch (IllegalArgumentException e) { + assertEquals("Content-Encoding = gzip but the data is not in gzip format", e.getMessage()); + } + } + + public void testStreamData_ExpectsGzipUsesGZipStream() throws IOException { + PipedInputStream pipedIn = new PipedInputStream(); + PipedOutputStream pipedOut = new PipedOutputStream(pipedIn); + try (GZIPOutputStream gzip = new GZIPOutputStream(pipedOut)) { + gzip.write("Hello World compressed".getBytes(StandardCharsets.UTF_8)); + + DataProcessor dataProcessor = mock(DataProcessor.class); + DataStreamer dataStreamer = new DataStreamer(dataProcessor); + DataLoadParams params = mock(DataLoadParams.class); + + when(dataProcessor.processData(Mockito.anyString(), + Mockito.any(InputStream.class), + Mockito.any(DataLoadParams.class))) + .thenReturn(new DataCounts("foo")); + + dataStreamer.streamData("gzip", "foo", pipedIn, params); + + // submitDataLoadJob should be called with a GZIPInputStream + ArgumentCaptor streamArg = ArgumentCaptor.forClass(InputStream.class); + + verify(dataProcessor).processData(Mockito.anyString(), + streamArg.capture(), + Mockito.any(DataLoadParams.class)); + + assertTrue(streamArg.getValue() instanceof GZIPInputStream); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThreadTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThreadTests.java new file mode 100644 index 00000000000..f231f0e1d78 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/data/DataStreamerThreadTests.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.data; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.utils.ExceptionsHelper; +import org.junit.After; +import org.junit.Before; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Date; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DataStreamerThreadTests extends ESTestCase { + private static final String JOB_ID = "foo"; + private static final String CONTENT_ENCODING = "application/json"; + + private DataStreamer dataStreamer; + private DataLoadParams params; + private InputStream inputStream; + + private DataStreamerThread dataStreamerThread; + + @Before + public void setUpMocks() { + dataStreamer = Mockito.mock(DataStreamer.class); + params = Mockito.mock(DataLoadParams.class); + inputStream = Mockito.mock(InputStream.class); + dataStreamerThread = new DataStreamerThread(dataStreamer, JOB_ID, CONTENT_ENCODING, params, inputStream); + } + + @After + public void verifyInputStreamClosed() throws IOException { + verify(inputStream).close(); + } + + public void testRun() throws Exception { + DataCounts counts = new DataCounts("foo", 42L, 0L, 0L, 0L, 0L, 0L, 0L, new Date(), new Date()); + when(dataStreamer.streamData(CONTENT_ENCODING, JOB_ID, inputStream, params)).thenReturn(counts); + + dataStreamerThread.run(); + + assertEquals(JOB_ID, dataStreamerThread.getJobId()); + assertEquals(counts, dataStreamerThread.getDataCounts()); + assertFalse(dataStreamerThread.getIOException().isPresent()); + assertFalse(dataStreamerThread.getJobException().isPresent()); + } + + public void testRun_GivenIOException() throws Exception { + when(dataStreamer.streamData(CONTENT_ENCODING, JOB_ID, inputStream, params)).thenThrow(new IOException("prelert")); + + dataStreamerThread.run(); + + assertEquals(JOB_ID, dataStreamerThread.getJobId()); + assertNull(dataStreamerThread.getDataCounts()); + assertEquals("prelert", dataStreamerThread.getIOException().get().getMessage()); + assertFalse(dataStreamerThread.getJobException().isPresent()); + } + + public void testRun_GivenJobException() throws Exception { + when(dataStreamer.streamData(CONTENT_ENCODING, JOB_ID, inputStream, params)).thenThrow(ExceptionsHelper.serverError("job failed")); + + dataStreamerThread.run(); + + assertEquals(JOB_ID, dataStreamerThread.getJobId()); + assertNull(dataStreamerThread.getDataCounts()); + assertFalse(dataStreamerThread.getIOException().isPresent()); + assertEquals("job failed", dataStreamerThread.getJobException().get().getMessage()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/ConnectiveTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/ConnectiveTests.java new file mode 100644 index 00000000000..e5e99c5f62c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/ConnectiveTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +public class ConnectiveTests extends ESTestCase { + + public void testForString() { + assertEquals(Connective.OR, Connective.fromString("or")); + assertEquals(Connective.OR, Connective.fromString("OR")); + assertEquals(Connective.AND, Connective.fromString("and")); + assertEquals(Connective.AND, Connective.fromString("AND")); + } + + public void testValidOrdinals() { + assertThat(Connective.OR.ordinal(), equalTo(0)); + assertThat(Connective.AND.ordinal(), equalTo(1)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Connective.OR.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Connective.AND.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Connective.readFromStream(in), equalTo(Connective.OR)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Connective.readFromStream(in), equalTo(Connective.AND)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + Connective.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown Connective ordinal [")); + } + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRuleTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRuleTests.java new file mode 100644 index 00000000000..f1f0cdc0b78 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/DetectionRuleTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +public class DetectionRuleTests extends AbstractSerializingTestCase { + + public void testExtractReferoencedLists() { + RuleCondition numericalCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "field", "value", new Condition(Operator.GT, "5"), null); + List conditions = Arrays.asList( + numericalCondition, + RuleCondition.createCategorical("foo", "list1"), + RuleCondition.createCategorical("bar", "list2")); + DetectionRule rule = new DetectionRule(null, null, Connective.OR, conditions); + + assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), rule.extractReferencedLists()); + } + + public void testEqualsGivenSameObject() { + DetectionRule rule = createFullyPopulated(); + assertTrue(rule.equals(rule)); + } + + public void testEqualsGivenString() { + assertFalse(createFullyPopulated().equals("a string")); + } + + public void testEqualsGivenDifferentTargetFieldName() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField2", "targetValue", Connective.AND, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenDifferentTargetFieldValue() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue2", Connective.AND, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenDifferentConjunction() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.OR, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenRules() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.AND, createRule("10")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenEqual() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = createFullyPopulated(); + assertTrue(rule1.equals(rule2)); + assertTrue(rule2.equals(rule1)); + assertEquals(rule1.hashCode(), rule2.hashCode()); + } + + private static DetectionRule createFullyPopulated() { + return new DetectionRule("targetField", "targetValue", Connective.AND, createRule("5")); + } + + private static List createRule(String value) { + Condition condition = new Condition(Operator.GT, value); + return Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + } + + @Override + protected DetectionRule createTestInstance() { + String targetFieldName = null; + String targetFieldValue = null; + Connective connective = randomFrom(Connective.values()); + if (randomBoolean()) { + targetFieldName = randomAsciiOfLengthBetween(1, 20); + targetFieldValue = randomAsciiOfLengthBetween(1, 20); + } + int size = 1 + randomInt(20); + List ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random condition (it is already tested) + ruleConditions.addAll(createRule(Double.toString(randomDouble()))); + } + return new DetectionRule(targetFieldName, targetFieldValue, connective, ruleConditions); + } + + @Override + protected Reader instanceReader() { + return DetectionRule::new; + } + + @Override + protected DetectionRule parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return DetectionRule.PARSER.apply(parser, () -> matcher); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleActionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleActionTests.java new file mode 100644 index 00000000000..b901520029c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleActionTests.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import org.elasticsearch.test.ESTestCase; + +public class RuleActionTests extends ESTestCase { + + public void testForString() { + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.forString("filter_results")); + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.forString("FILTER_RESULTS")); + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.forString("fiLTer_Results")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTests.java new file mode 100644 index 00000000000..654bc1e5961 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTests.java @@ -0,0 +1,223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.condition.Condition; +import org.elasticsearch.xpack.prelert.job.condition.Operator; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class RuleConditionTests extends AbstractSerializingTestCase { + + @Override + protected RuleCondition createTestInstance() { + Condition condition = null; + String fieldName = null; + String valueList = null; + String fieldValue = null; + RuleConditionType r = randomFrom(RuleConditionType.values()); + switch (r) { + case CATEGORICAL: + valueList = randomAsciiOfLengthBetween(1, 20); + if (randomBoolean()) { + fieldName = randomAsciiOfLengthBetween(1, 20); + } + break; + default: + // no need to randomize, it is properly randomily tested in + // ConditionTest + condition = new Condition(Operator.LT, Double.toString(randomDouble())); + if (randomBoolean()) { + fieldName = randomAsciiOfLengthBetween(1, 20); + fieldValue = randomAsciiOfLengthBetween(1, 20); + } + break; + } + return new RuleCondition(r, fieldName, fieldValue, condition, valueList); + } + + @Override + protected Reader instanceReader() { + return RuleCondition::new; + } + + @Override + protected RuleCondition parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return RuleCondition.PARSER.apply(parser, () -> matcher); + } + + public void testConstructor() { + RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueList"); + assertEquals(RuleConditionType.CATEGORICAL, condition.getConditionType()); + assertNull(condition.getFieldName()); + assertNull(condition.getFieldValue()); + assertNull(condition.getCondition()); + } + + public void testEqualsGivenSameObject() { + RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueList"); + assertTrue(condition.equals(condition)); + } + + public void testEqualsGivenString() { + assertFalse(new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "list").equals("a string")); + } + + public void testEqualsGivenDifferentType() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueList"); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentFieldName() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricNameaaa", "cpu", + new Condition(Operator.LT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentFieldValue() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpuaaa", + new Condition(Operator.LT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentCondition() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", + new Condition(Operator.GT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentValueList() { + RuleCondition condition1 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myList"); + RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myListaaa"); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + private static RuleCondition createFullyPopulated() { + return new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", new Condition(Operator.LT, "5"), null); + } + + public void testVerify_GivenCategoricalWithCondition() { + Condition condition = new Condition(Operator.MATCH, "text"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, condition, null)); + assertEquals("Invalid detector rule: a categorical ruleCondition does not support condition", e.getMessage()); + } + + public void testVerify_GivenCategoricalWithFieldValue() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, "metric", "CPU", null, null)); + assertEquals("Invalid detector rule: a categorical ruleCondition does not support fieldValue", e.getMessage()); + } + + public void testVerify_GivenCategoricalWithoutValueList() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null)); + assertEquals("Invalid detector rule: a categorical ruleCondition requires valueList to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithValueList() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myList")); + assertEquals("Invalid detector rule: a numerical ruleCondition does not support valueList", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical ruleCondition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithFieldNameButNoFieldValue() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", null, new Condition(Operator.LT, "5"), null)); + assertEquals("Invalid detector rule: a numerical ruleCondition with fieldName requires that fieldValue is set", e.getMessage()); + } + + public void testVerify_GivenNumericalTypicalWithValueList() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myList")); + assertEquals("Invalid detector rule: a numerical ruleCondition does not support valueList", e.getMessage()); + } + + public void testVerify_GivenNumericalTypicalWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical ruleCondition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalDiffAbsWithValueList() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myList")); + assertEquals("Invalid detector rule: a numerical ruleCondition does not support valueList", e.getMessage()); + } + + public void testVerify_GivenNumericalDiffAbsWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical ruleCondition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenFieldValueWithoutFieldName() { + Condition condition = new Condition(Operator.LTE, "5"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, "foo", condition, null)); + assertEquals("Invalid detector rule: missing fieldName in ruleCondition where fieldValue 'foo' is set", e.getMessage()); + } + + public void testVerify_GivenNumericalAndOperatorEquals() { + Condition condition = new Condition(Operator.EQ, "5"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + assertEquals("Invalid detector rule: operator 'EQ' is not allowed", e.getMessage()); + } + + public void testVerify_GivenNumericalAndOperatorMatch() { + Condition condition = new Condition(Operator.MATCH, "aaa"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + assertEquals("Invalid detector rule: operator 'MATCH' is not allowed", e.getMessage()); + } + + public void testVerify_GivenDetectionRuleWithInvalidCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "invalid"), + null)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, "invalid"), e.getMessage()); + } + + public void testVerify_GivenValidCategorical() { + // no validation error: + new RuleCondition(RuleConditionType.CATEGORICAL, "metric", null, null, "myList"); + } + + public void testVerify_GivenValidNumericalActual() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GT, "5"), null); + } + + public void testVerify_GivenValidNumericalTypical() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GTE, "5"), null); + } + + public void testVerify_GivenValidNumericalDiffAbs() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, "metric", "cpu", new Condition(Operator.LT, "5"), null); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTypeTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTypeTests.java new file mode 100644 index 00000000000..c3cc6ff4187 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/detectionrules/RuleConditionTypeTests.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.detectionrules; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class RuleConditionTypeTests extends ESTestCase { + + public void testForString() { + assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.forString("categorical")); + assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.forString("CATEGORICAL")); + assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.forString("numerical_actual")); + assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.forString("NUMERICAL_ACTUAL")); + assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.forString("numerical_typical")); + assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.forString("NUMERICAL_TYPICAL")); + assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.forString("numerical_diff_abs")); + assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.forString("NUMERICAL_DIFF_ABS")); + } + + public void testValidOrdinals() { + assertThat(RuleConditionType.CATEGORICAL.ordinal(), equalTo(0)); + assertThat(RuleConditionType.NUMERICAL_ACTUAL.ordinal(), equalTo(1)); + assertThat(RuleConditionType.NUMERICAL_TYPICAL.ordinal(), equalTo(2)); + assertThat(RuleConditionType.NUMERICAL_DIFF_ABS.ordinal(), equalTo(3)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.CATEGORICAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_ACTUAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_TYPICAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_DIFF_ABS.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.CATEGORICAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_ACTUAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_TYPICAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_DIFF_ABS)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + RuleConditionType.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown RuleConditionType ordinal [")); + } + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandlerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandlerTests.java new file mode 100644 index 00000000000..534f5773c2f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageHandlerTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logging; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class CppLogMessageHandlerTests extends ESTestCase { + + public void testParse() throws IOException { + + String testData = "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\",\"pid\":10211," + + "\"thread\":\"0x7fff7d2a8000\",\"message\":\"uname -a : Darwin Davids-MacBook-Pro.local 15.6.0 Darwin Kernel " + + "Version 15.6.0: Thu Sep 1 15:01:16 PDT 2016; root:xnu-3248.60.11~2/RELEASE_X86_64 x86_64\",\"class\":\"prelert\"," + + "\"method\":\"core::CLogger::reconfigureFromProps\",\"file\":\"CLogger.cc\",\"line\":452}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"DEBUG\",\"pid\":10211,\"thread\":\"0x7fff7d2a8000\"," + + "\"message\":\"Logger is logging to named pipe " + + "/var/folders/k5/5sqcdlps5sg3cvlp783gcz740000h0/T/prelert_controller_log_784\",\"class\":\"prelert\"," + + "\"method\":\"core::CLogger::reconfigureLogToNamedPipe\",\"file\":\"CLogger.cc\",\"line\":333}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261151445,\"level\":\"INFO\",\"pid\":10211,\"thread\":\"0x7fff7d2a8000\"," + + "\"message\":\"prelert_controller (64 bit): Version based on 6.5.0 (Build DEVELOPMENT BUILD by dave) " + + "Copyright (c) Prelert Ltd 2006-2016\",\"method\":\"main\",\"file\":\"Main.cc\",\"line\":123}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261169065,\"level\":\"ERROR\",\"pid\":10211,\"thread\":\"0x7fff7d2a8000\"," + + "\"message\":\"Did not understand verb 'a'\",\"class\":\"prelert\"," + + "\"method\":\"controller::CCommandProcessor::handleCommand\",\"file\":\"CCommandProcessor.cc\",\"line\":100}\n" + + "{\"logger\":\"controller\",\"timestamp\":1478261169065,\"level\":\"DEBUG\",\"pid\":10211,\"thread\":\"0x7fff7d2a8000\"," + + "\"message\":\"Prelert controller exiting\",\"method\":\"main\",\"file\":\"Main.cc\",\"line\":147}\n"; + + InputStream is = new ByteArrayInputStream(testData.getBytes(StandardCharsets.UTF_8)); + + Logger logger = Loggers.getLogger(CppLogMessageHandlerTests.class); + Loggers.setLevel(logger, Level.DEBUG); + + try (CppLogMessageHandler handler = new CppLogMessageHandler(is, logger, 100, 3)) { + handler.tailStream(); + + assertEquals("Did not understand verb 'a'\n", handler.getErrors()); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageTests.java new file mode 100644 index 00000000000..82d0aac7a67 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logging/CppLogMessageTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logging; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class CppLogMessageTests extends AbstractSerializingTestCase { + + public void testDefaultConstructor() { + CppLogMessage msg = new CppLogMessage(); + assertEquals("", msg.getLogger()); + assertTrue(msg.getTimestamp().toString(), msg.getTimestamp().getTime() > 0); + assertEquals("", msg.getLevel()); + assertEquals(0, msg.getPid()); + assertEquals("", msg.getThread()); + assertEquals("", msg.getMessage()); + assertEquals("", msg.getClazz()); + assertEquals("", msg.getMethod()); + assertEquals("", msg.getFile()); + assertEquals(0, msg.getLine()); + } + + @Override + protected CppLogMessage createTestInstance() { + CppLogMessage msg = new CppLogMessage(); + msg.setLogger("autodetect"); + msg.setLevel("INFO"); + msg.setPid(12345); + msg.setThread("0x123456789"); + msg.setMessage("Very informative"); + msg.setClazz("CAnomalyDetector"); + msg.setMethod("detectAnomalies"); + msg.setFile("CAnomalyDetector.cc"); + msg.setLine(123); + return msg; + } + + @Override + protected Reader instanceReader() { + return CppLogMessage::new; + } + + @Override + protected CppLogMessage parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return CppLogMessage.PARSER.apply(parser, () -> matcher); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logs/JobLogsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logs/JobLogsTests.java new file mode 100644 index 00000000000..1e05f68d898 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/logs/JobLogsTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.logs; + +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.messages.Messages; +import org.hamcrest.core.StringStartsWith; + +import java.io.IOException; +import java.nio.file.Path; + +public class JobLogsTests extends ESTestCase { + + public void testOperationsNotAllowedWithInvalidPath() throws IOException { + Path pathOutsideLogsDir = PathUtils.getDefaultFileSystem().getPath("..", "..", "..", "etc"); + + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Environment env = new Environment( + settings); + + // delete + try { + JobLogs jobLogs = new JobLogs(settings); + jobLogs.deleteLogs(env, pathOutsideLogsDir.toString()); + fail(); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), StringStartsWith.startsWith("Invalid log file path.")); + } + } + + public void testSanitizePath_GivenInvalid() { + + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Path filePath = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert", "../../etc"); + try { + Path rootDir = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert"); + new JobLogs(settings).sanitizePath(filePath, rootDir); + fail(); + } catch (IllegalArgumentException e) { + assertEquals(Messages.getMessage(Messages.LOGFILE_INVALID_PATH, filePath), e.getMessage()); + } + } + + public void testSanitizePath() { + + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Path filePath = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert", "logs", "logfile.log"); + Path rootDir = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert", "logs"); + Path normalized = new JobLogs(settings).sanitizePath(filePath, rootDir); + assertEquals(filePath, normalized); + + Path logDir = PathUtils.getDefaultFileSystem().getPath("./logs"); + Path filePathStartingDot = logDir.resolve("farequote").resolve("logfile.log"); + normalized = new JobLogs(settings).sanitizePath(filePathStartingDot, logDir); + assertEquals(filePathStartingDot.normalize(), normalized); + + Path filePathWithDotDot = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert", "logs", "../logs/logfile.log"); + rootDir = PathUtils.getDefaultFileSystem().getPath("/opt", "prelert", "logs"); + normalized = new JobLogs(settings).sanitizePath(filePathWithDotDot, rootDir); + + assertEquals(filePath, normalized); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManagerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManagerTests.java new file mode 100644 index 00000000000..0be789b5539 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/AutodetectProcessManagerTests.java @@ -0,0 +1,222 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.manager; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.prelert.job.AnalysisConfig; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.DataDescription; +import org.elasticsearch.xpack.prelert.job.Detector; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectCommunicator; +import org.elasticsearch.xpack.prelert.job.process.autodetect.AutodetectProcessFactory; +import org.elasticsearch.xpack.prelert.job.process.autodetect.output.parsing.AutodetectResultsParser; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.prelert.job.process.autodetect.params.TimeRange; +import org.junit.Before; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import static org.elasticsearch.mock.orig.Mockito.doThrow; +import static org.elasticsearch.mock.orig.Mockito.verify; +import static org.elasticsearch.mock.orig.Mockito.when; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +/** + * Calling the {@link AutodetectProcessManager#processData(String, InputStream, DataLoadParams)} + * method causes an AutodetectCommunicator to be created on demand. Most of these tests have to + * do that before they can assert other things + */ +public class AutodetectProcessManagerTests extends ESTestCase { + + private JobManager jobManager; + + @Before + public void initMocks() { + jobManager = Mockito.mock(JobManager.class); + givenAllocationWithStatus(JobStatus.CLOSED); + } + + public void testCreateProcessBySubmittingData() { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + assertEquals(0, manager.numberOfRunningJobs()); + + DataLoadParams params = new DataLoadParams(TimeRange.builder().build()); + manager.processData("foo", createInputStream(""), params); + assertEquals(1, manager.numberOfRunningJobs()); + } + + public void testProcessDataThrowsElasticsearchStatusException_onIoException() throws Exception { + AutodetectCommunicator communicator = Mockito.mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + + InputStream inputStream = createInputStream(""); + doThrow(new IOException("blah")).when(communicator).writeToJob(inputStream); + + ESTestCase.expectThrows(ElasticsearchException.class, + () -> manager.processData("foo", inputStream, mock(DataLoadParams.class))); + } + + public void testCloseJob() { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); + AutodetectProcessManager manager = createManager(communicator); + assertEquals(0, manager.numberOfRunningJobs()); + + manager.processData("foo", createInputStream(""), mock(DataLoadParams.class)); + + // job is created + assertEquals(1, manager.numberOfRunningJobs()); + manager.closeJob("foo"); + assertEquals(0, manager.numberOfRunningJobs()); + } + + public void testBucketResetMessageIsSent() throws IOException { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + + DataLoadParams params = new DataLoadParams(TimeRange.builder().startTime("1000").endTime("2000").build(), true); + InputStream inputStream = createInputStream(""); + manager.processData("foo", inputStream, params); + + verify(communicator).writeResetBucketsControlMessage(params); + verify(communicator).writeToJob(inputStream); + } + + public void testFlush() throws IOException { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(createJobDetails("foo")); + + InputStream inputStream = createInputStream(""); + manager.processData("foo", inputStream, mock(DataLoadParams.class)); + + InterimResultsParams params = InterimResultsParams.builder().build(); + manager.flushJob("foo", params); + + verify(communicator).flushJob(params); + } + + public void testFlushThrows() throws IOException { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManagerAndCallProcessData(communicator, "foo"); + + InterimResultsParams params = InterimResultsParams.builder().build(); + doThrow(new IOException("blah")).when(communicator).flushJob(params); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, () -> manager.flushJob("foo", params)); + assertEquals("Exception flushing process for job foo", e.getMessage()); + } + + public void testWriteUpdateConfigMessage() throws IOException { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManagerAndCallProcessData(communicator, "foo"); + manager.writeUpdateConfigMessage("foo", "go faster"); + verify(communicator).writeUpdateConfigMessage("go faster"); + } + + public void testJobHasActiveAutodetectProcess() throws IOException { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + assertFalse(manager.jobHasActiveAutodetectProcess("foo")); + + manager.processData("foo", createInputStream(""), mock(DataLoadParams.class)); + + assertTrue(manager.jobHasActiveAutodetectProcess("foo")); + assertFalse(manager.jobHasActiveAutodetectProcess("bar")); + } + + public void testProcessData_GivenPausingJob() { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + AutodetectProcessManager manager = createManager(communicator); + + Job job = createJobDetails("foo"); + + when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(job); + givenAllocationWithStatus(JobStatus.PAUSING); + + InputStream inputStream = createInputStream(""); + DataCounts dataCounts = manager.processData("foo", inputStream, mock(DataLoadParams.class)); + + assertThat(dataCounts, equalTo(new DataCounts("foo"))); + } + + public void testProcessData_GivenPausedJob() { + AutodetectCommunicator communicator = mock(AutodetectCommunicator.class); + Job job = createJobDetails("foo"); + when(jobManager.getJobOrThrowIfUnknown("foo")).thenReturn(job); + givenAllocationWithStatus(JobStatus.PAUSED); + AutodetectProcessManager manager = createManager(communicator); + InputStream inputStream = createInputStream(""); + DataCounts dataCounts = manager.processData("foo", inputStream, mock(DataLoadParams.class)); + + assertThat(dataCounts, equalTo(new DataCounts("foo"))); + } + + private void givenAllocationWithStatus(JobStatus status) { + Allocation.Builder allocation = new Allocation.Builder(); + allocation.setStatus(status); + when(jobManager.getJobAllocation("foo")).thenReturn(allocation.build()); + } + + private AutodetectProcessManager createManager(AutodetectCommunicator communicator) { + Client client = mock(Client.class); + Environment environment = mock(Environment.class); + ThreadPool threadPool = mock(ThreadPool.class); + AutodetectResultsParser parser = mock(AutodetectResultsParser.class); + AutodetectProcessFactory autodetectProcessFactory = mock(AutodetectProcessFactory.class); + AutodetectProcessManager manager = + new AutodetectProcessManager(Settings.EMPTY, client, environment, threadPool, jobManager, parser, autodetectProcessFactory); + manager = spy(manager); + doReturn(communicator).when(manager).create(any(), anyBoolean()); + return manager; + } + + private AutodetectProcessManager createManagerAndCallProcessData(AutodetectCommunicator communicator, String jobId) { + AutodetectProcessManager manager = createManager(communicator); + manager.processData(jobId, createInputStream(""), mock(DataLoadParams.class)); + return manager; + } + + private Job createJobDetails(String jobId) { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setFormat(DataDescription.DataFormat.DELIMITED); + dd.setFieldDelimiter(','); + + Detector d = new Detector.Builder("metric", "value").build(); + + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d)); + + Job.Builder builder = new Job.Builder(jobId); + builder.setDataDescription(dd); + builder.setAnalysisConfig(ac); + + return builder.build(); + } + + private static InputStream createInputStream(String input) { + return new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/JobManagerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/JobManagerTests.java new file mode 100644 index 00000000000..8ee3c867732 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/manager/JobManagerTests.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.manager; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.audit.Auditor; +import org.elasticsearch.xpack.prelert.job.metadata.PrelertMetadata; +import org.elasticsearch.xpack.prelert.job.persistence.JobProvider; +import org.elasticsearch.xpack.prelert.job.persistence.QueryPage; +import org.junit.Before; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class JobManagerTests extends ESTestCase { + + private ClusterService clusterService; + private JobProvider jobProvider; + private Auditor auditor; + + @Before + public void setupMocks() { + clusterService = mock(ClusterService.class); + jobProvider = mock(JobProvider.class); + auditor = mock(Auditor.class); + when(jobProvider.audit(anyString())).thenReturn(auditor); + } + + public void testGetJob() { + JobManager jobManager = createJobManager(); + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(); + builder.putJob(buildJobBuilder("foo").build(), false); + ClusterState clusterState = ClusterState.builder(new ClusterName("name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, builder.build())).build(); + Optional doc = jobManager.getJob("foo", clusterState); + assertTrue(doc.isPresent()); + } + + public void testFilter() { + Set running = new HashSet(Arrays.asList("henry", "dim", "dave")); + Set diff = new HashSet<>(Arrays.asList("dave", "tom")).stream().filter((s) -> !running.contains(s)) + .collect(Collectors.toCollection(HashSet::new)); + + assertTrue(diff.size() == 1); + assertTrue(diff.contains("tom")); + } + + public void testRemoveJobFromClusterState_GivenExistingMetadata() { + JobManager jobManager = createJobManager(); + ClusterState clusterState = createClusterState(); + Job job = buildJobBuilder("foo").build(); + clusterState = jobManager.innerPutJob(job, false, clusterState); + + clusterState = jobManager.removeJobFromClusterState("foo", clusterState); + + PrelertMetadata prelertMetadata = clusterState.metaData().custom(PrelertMetadata.TYPE); + assertThat(prelertMetadata.getJobs().containsKey("foo"), is(false)); + } + + public void testRemoveJobFromClusterState_jobMissing() { + JobManager jobManager = createJobManager(); + ClusterState clusterState = createClusterState(); + Job job = buildJobBuilder("foo").build(); + ClusterState clusterState2 = jobManager.innerPutJob(job, false, clusterState); + Exception e = expectThrows(ResourceNotFoundException.class, () -> jobManager.removeJobFromClusterState("bar", clusterState2)); + assertThat(e.getMessage(), equalTo("job [bar] does not exist")); + } + + public void testGetJobOrThrowIfUnknown_GivenUnknownJob() { + JobManager jobManager = createJobManager(); + ClusterState cs = createClusterState(); + ESTestCase.expectThrows(ResourceNotFoundException.class, () -> jobManager.getJobOrThrowIfUnknown(cs, "foo")); + } + + public void testGetJobOrThrowIfUnknown_GivenKnownJob() { + JobManager jobManager = createJobManager(); + Job job = buildJobBuilder("foo").build(); + PrelertMetadata prelertMetadata = new PrelertMetadata.Builder().putJob(job, false).build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, prelertMetadata)).build(); + + assertEquals(job, jobManager.getJobOrThrowIfUnknown(cs, "foo")); + } + + public void tesGetJobAllocation() { + JobManager jobManager = createJobManager(); + Job job = buildJobBuilder("foo").build(); + PrelertMetadata prelertMetadata = new PrelertMetadata.Builder() + .putJob(job, false) + .putAllocation("nodeId", "foo") + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, prelertMetadata)).build(); + when(clusterService.state()).thenReturn(cs); + + assertEquals("nodeId", jobManager.getJobAllocation("foo").getNodeId()); + expectThrows(ResourceNotFoundException.class, () -> jobManager.getJobAllocation("bar")); + } + + public void testGetJobs() { + PrelertMetadata.Builder prelertMetadata = new PrelertMetadata.Builder(); + for (int i = 0; i < 10; i++) { + prelertMetadata.putJob(buildJobBuilder(Integer.toString(i)).build(), false); + } + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, prelertMetadata.build())).build(); + + JobManager jobManager = createJobManager(); + QueryPage result = jobManager.getJobs(0, 10, clusterState); + assertThat(result.hitCount(), equalTo(10L)); + assertThat(result.hits().get(0).getId(), equalTo("0")); + assertThat(result.hits().get(1).getId(), equalTo("1")); + assertThat(result.hits().get(2).getId(), equalTo("2")); + assertThat(result.hits().get(3).getId(), equalTo("3")); + assertThat(result.hits().get(4).getId(), equalTo("4")); + assertThat(result.hits().get(5).getId(), equalTo("5")); + assertThat(result.hits().get(6).getId(), equalTo("6")); + assertThat(result.hits().get(7).getId(), equalTo("7")); + assertThat(result.hits().get(8).getId(), equalTo("8")); + assertThat(result.hits().get(9).getId(), equalTo("9")); + + result = jobManager.getJobs(0, 5, clusterState); + assertThat(result.hitCount(), equalTo(10L)); + assertThat(result.hits().get(0).getId(), equalTo("0")); + assertThat(result.hits().get(1).getId(), equalTo("1")); + assertThat(result.hits().get(2).getId(), equalTo("2")); + assertThat(result.hits().get(3).getId(), equalTo("3")); + assertThat(result.hits().get(4).getId(), equalTo("4")); + + result = jobManager.getJobs(5, 5, clusterState); + assertThat(result.hitCount(), equalTo(10L)); + assertThat(result.hits().get(0).getId(), equalTo("5")); + assertThat(result.hits().get(1).getId(), equalTo("6")); + assertThat(result.hits().get(2).getId(), equalTo("7")); + assertThat(result.hits().get(3).getId(), equalTo("8")); + assertThat(result.hits().get(4).getId(), equalTo("9")); + + result = jobManager.getJobs(9, 1, clusterState); + assertThat(result.hitCount(), equalTo(10L)); + assertThat(result.hits().get(0).getId(), equalTo("9")); + + result = jobManager.getJobs(9, 10, clusterState); + assertThat(result.hitCount(), equalTo(10L)); + assertThat(result.hits().get(0).getId(), equalTo("9")); + } + + public void testInnerPutJob() { + JobManager jobManager = createJobManager(); + ClusterState cs = createClusterState(); + + Job job1 = buildJobBuilder("_id").build(); + ClusterState result1 = jobManager.innerPutJob(job1, false, cs); + PrelertMetadata pm = result1.getMetaData().custom(PrelertMetadata.TYPE); + assertThat(pm.getJobs().get("_id"), sameInstance(job1)); + + Job job2 = buildJobBuilder("_id").build(); + expectThrows(ResourceAlreadyExistsException.class, () -> jobManager.innerPutJob(job2, false, result1)); + + ClusterState result2 = jobManager.innerPutJob(job2, true, result1); + pm = result2.getMetaData().custom(PrelertMetadata.TYPE); + assertThat(pm.getJobs().get("_id"), sameInstance(job2)); + } + + private JobManager createJobManager() { + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + Environment env = new Environment( + settings); + return new JobManager(env, settings, jobProvider, clusterService); + } + + private ClusterState createClusterState() { + ClusterState.Builder builder = ClusterState.builder(new ClusterName("_name")); + builder.metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, PrelertMetadata.PROTO)); + return builder.build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/messages/MessagesTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/messages/MessagesTests.java new file mode 100644 index 00000000000..42694ce15fb --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/messages/MessagesTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.messages; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.ResourceBundle; +import java.util.Set; + +@SuppressForbidden(reason = "Need to use reflection to make sure all constants are resolvable") +public class MessagesTests extends ESTestCase { + + public void testAllStringsAreInTheResourceBundle() throws IllegalArgumentException, IllegalAccessException { + ResourceBundle bundle = Messages.load(); + + // get all the public string constants + // excluding BUNDLE_NAME + List publicStrings = new ArrayList<>(); + Field[] allFields = Messages.class.getDeclaredFields(); + for (Field field : allFields) { + if (Modifier.isPublic(field.getModifiers())) { + if (field.getType() == String.class && field.getName() != "BUNDLE_NAME") { + publicStrings.add(field); + } + } + } + + assertTrue(bundle.keySet().size() > 0); + + // Make debugging easier- print any keys not defined in Messages + Set keys = bundle.keySet(); + for (Field field : publicStrings) { + String key = (String) field.get(new String()); + keys.remove(key); + } + + assertEquals(bundle.keySet().size(), publicStrings.size()); + + for (Field field : publicStrings) { + String key = (String) field.get(new String()); + + assertTrue("String constant " + field.getName() + " = " + key + " not in resource bundle", bundle.containsKey(key)); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/AllocationTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/AllocationTests.java new file mode 100644 index 00000000000..641672b4f2f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/AllocationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.prelert.job.JobSchedulerStatus; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.SchedulerState; +import org.elasticsearch.xpack.prelert.support.AbstractSerializingTestCase; + +public class AllocationTests extends AbstractSerializingTestCase { + + @Override + protected Allocation createTestInstance() { + String nodeId = randomAsciiOfLength(10); + String jobId = randomAsciiOfLength(10); + JobStatus jobStatus = randomFrom(JobStatus.values()); + SchedulerState schedulerState = new SchedulerState(JobSchedulerStatus.STARTING, randomPositiveLong(), randomPositiveLong()); + return new Allocation(nodeId, jobId, jobStatus, schedulerState); + } + + @Override + protected Writeable.Reader instanceReader() { + return Allocation::new; + } + + @Override + protected Allocation parseInstance(XContentParser parser, ParseFieldMatcher matcher) { + return Allocation.PARSER.apply(parser, () -> matcher).build(); + } + + @Override + protected boolean skipJacksonTest() { + return true; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocatorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocatorTests.java new file mode 100644 index 00000000000..af0ca645cc0 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobAllocatorTests.java @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.LocalTransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Before; + +import java.util.concurrent.ExecutorService; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.mockito.Matchers.any; + +public class JobAllocatorTests extends ESTestCase { + + private ClusterService clusterService; + private ThreadPool threadPool; + private JobAllocator jobAllocator; + + @Before + public void instantiateJobAllocator() { + clusterService = mock(ClusterService.class); + threadPool = mock(ThreadPool.class); + jobAllocator = new JobAllocator(Settings.EMPTY, clusterService, threadPool); + } + + public void testShouldAllocate() { + ClusterState cs = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, new PrelertMetadata.Builder().build())) + .build(); + assertFalse("No jobs, so nothing to allocate", jobAllocator.shouldAllocate(cs)); + + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(cs.metaData().custom(PrelertMetadata.TYPE)); + pmBuilder.putJob((buildJobBuilder("_job_id").build()), false); + cs = ClusterState.builder(cs).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .build(); + assertTrue("A unassigned job, so we should allocate", jobAllocator.shouldAllocate(cs)); + + pmBuilder = new PrelertMetadata.Builder(cs.metaData().custom(PrelertMetadata.TYPE)); + pmBuilder.putAllocation("_node_id", "_job_id"); + cs = ClusterState.builder(cs).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .build(); + assertFalse("Job is allocate, so nothing to allocate", jobAllocator.shouldAllocate(cs)); + } + + public void testAllocateJobs() { + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(); + pmBuilder.putJob(buildJobBuilder("_job_id").build(), false); + ClusterState cs1 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .masterNodeId("_node_id")) + .build(); + ClusterState result1 = jobAllocator.allocateJobs(cs1); + PrelertMetadata pm = result1.metaData().custom(PrelertMetadata.TYPE); + assertEquals("_job_id must be allocated to _node_id", pm.getAllocations().get("_job_id").getNodeId(), "_node_id"); + + ClusterState result2 = jobAllocator.allocateJobs(result1); + assertSame("job has been allocated, same instance must be returned", result1, result2); + + ClusterState cs2 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes( + DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id1", new LocalTransportAddress("_id1"), Version.CURRENT)) + .add(new DiscoveryNode("_node_id2", new LocalTransportAddress("_id2"), Version.CURRENT)) + .masterNodeId("_node_id1") + ) + .build(); + // should fail, prelert only support single node for now + expectThrows(IllegalStateException.class, () -> jobAllocator.allocateJobs(cs2)); + + ClusterState cs3 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .build(); + // we need to have at least one node + expectThrows(IllegalStateException.class, () -> jobAllocator.allocateJobs(cs3)); + + pmBuilder = new PrelertMetadata.Builder(result1.getMetaData().custom(PrelertMetadata.TYPE)); + pmBuilder.removeJob("_job_id"); + ClusterState cs4 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .masterNodeId("_node_id")) + .build(); + ClusterState result3 = jobAllocator.allocateJobs(cs4); + pm = result3.metaData().custom(PrelertMetadata.TYPE); + assertNull("_job_id must be unallocated, because job has been removed", pm.getAllocations().get("_job_id")); + } + + public void testClusterChanged_onlyAllocateIfMasterAndHaveUnAllocatedJobs() { + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, new PrelertMetadata.Builder().build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .localNodeId("_id") + ) + .build(); + jobAllocator.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + verify(threadPool, never()).executor(ThreadPool.Names.GENERIC); + verify(clusterService, never()).submitStateUpdateTask(any(), any()); + + // make node master + cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, new PrelertMetadata.Builder().build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .masterNodeId("_id") + .localNodeId("_id") + ) + .build(); + jobAllocator.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + verify(threadPool, never()).executor(ThreadPool.Names.GENERIC); + verify(clusterService, never()).submitStateUpdateTask(any(), any()); + + // add an allocated job + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(); + pmBuilder.putJob(buildJobBuilder("_id").build(), false); + pmBuilder.putAllocation("_id", "_id"); + cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .masterNodeId("_id") + .localNodeId("_id") + ) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .build(); + jobAllocator.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + verify(threadPool, never()).executor(ThreadPool.Names.GENERIC); + verify(clusterService, never()).submitStateUpdateTask(any(), any()); + + // make job not allocated + pmBuilder = new PrelertMetadata.Builder(); + pmBuilder.putJob(buildJobBuilder("_job_id").build(), false); + cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .masterNodeId("_id") + .localNodeId("_id") + ) + .metaData(MetaData.builder().putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .build(); + jobAllocator.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + verify(threadPool, times(1)).executor(ThreadPool.Names.GENERIC); + verify(clusterService, times(1)).submitStateUpdateTask(any(), any()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleServiceTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleServiceTests.java new file mode 100644 index 00000000000..c2a8b19a4db --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/JobLifeCycleServiceTests.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.LocalTransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.action.UpdateJobStatusAction; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobStatus; +import org.elasticsearch.xpack.prelert.job.data.DataProcessor; +import org.elasticsearch.xpack.prelert.job.scheduler.ScheduledJobService; +import org.junit.Before; +import org.mockito.Mockito; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.verify; + +public class JobLifeCycleServiceTests extends ESTestCase { + + private ClusterService clusterService; + private ScheduledJobService scheduledJobService; + private DataProcessor dataProcessor; + private Client client; + private JobLifeCycleService jobLifeCycleService; + + @Before + public void instantiateJobAllocator() { + clusterService = Mockito.mock(ClusterService.class); + scheduledJobService = Mockito.mock(ScheduledJobService.class); + dataProcessor = Mockito.mock(DataProcessor.class); + client = Mockito.mock(Client.class); + jobLifeCycleService = new JobLifeCycleService(Settings.EMPTY, client, clusterService, scheduledJobService, dataProcessor, + Runnable::run); + } + + public void testStartStop() { + jobLifeCycleService.startJob(buildJobBuilder("_job_id").build()); + assertTrue(jobLifeCycleService.localAllocatedJobs.contains("_job_id")); + jobLifeCycleService.stopJob("_job_id"); + assertTrue(jobLifeCycleService.localAllocatedJobs.isEmpty()); + } + + public void testClusterChanged() { + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(); + pmBuilder.putJob(buildJobBuilder("_job_id").build(), false); + pmBuilder.putAllocation("_node_id", "_job_id"); + ClusterState cs1 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .localNodeId("_node_id")) + .build(); + jobLifeCycleService.clusterChanged(new ClusterChangedEvent("_source", cs1, cs1)); + assertTrue("Expect allocation, because job allocation says _job_id should be allocated locally", + jobLifeCycleService.localAllocatedJobs.contains("_job_id")); + + pmBuilder.removeJob("_job_id"); + ClusterState cs2 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .localNodeId("_node_id")) + .build(); + jobLifeCycleService.clusterChanged(new ClusterChangedEvent("_source", cs2, cs1)); + assertFalse("Expect no allocation, because the job has been removed", jobLifeCycleService.localAllocatedJobs.contains("_job_id")); + } + + public void testClusterChanged_GivenJobIsPausing() { + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(); + Job.Builder job = buildJobBuilder("foo"); + pmBuilder.putJob(job.build(), false); + pmBuilder.putAllocation("_node_id", "foo"); + Allocation.Builder allocation = new Allocation.Builder(); + allocation.setJobId("foo"); + allocation.setNodeId("_node_id"); + allocation.setStatus(JobStatus.PAUSING); + pmBuilder.updateAllocation("foo", allocation.build()); + ClusterState cs1 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .localNodeId("_node_id")) + .build(); + + jobLifeCycleService.clusterChanged(new ClusterChangedEvent("_source", cs1, cs1)); + + verify(dataProcessor).closeJob("foo"); + UpdateJobStatusAction.Request expectedRequest = new UpdateJobStatusAction.Request("foo", JobStatus.PAUSED); + verify(client).execute(eq(UpdateJobStatusAction.INSTANCE), eq(expectedRequest), any()); + } + + public void testClusterChanged_GivenJobIsPausingAndCloseJobThrows() { + PrelertMetadata.Builder pmBuilder = new PrelertMetadata.Builder(); + Job.Builder job = buildJobBuilder("foo"); + pmBuilder.putJob(job.build(), false); + pmBuilder.putAllocation("_node_id", "foo"); + Allocation.Builder allocation = new Allocation.Builder(); + allocation.setJobId("foo"); + allocation.setNodeId("_node_id"); + allocation.setStatus(JobStatus.PAUSING); + pmBuilder.updateAllocation("foo", allocation.build()); + ClusterState cs1 = ClusterState.builder(new ClusterName("_cluster_name")).metaData(MetaData.builder() + .putCustom(PrelertMetadata.TYPE, pmBuilder.build())) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), Version.CURRENT)) + .localNodeId("_node_id")) + .build(); + doThrow(new ElasticsearchException("")).when(dataProcessor).closeJob("foo"); + + jobLifeCycleService.clusterChanged(new ClusterChangedEvent("_source", cs1, cs1)); + + verify(dataProcessor).closeJob("foo"); + UpdateJobStatusAction.Request expectedRequest = new UpdateJobStatusAction.Request("foo", JobStatus.FAILED); + verify(client).execute(eq(UpdateJobStatusAction.INSTANCE), eq(expectedRequest), any()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadataTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadataTests.java new file mode 100644 index 00000000000..bcf06c21be4 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/metadata/PrelertMetadataTests.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.metadata; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.OutputStreamStreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.JobTests; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +public class PrelertMetadataTests extends ESTestCase { + + public void testSerialization() throws Exception { + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(); + + Job job1 = JobTests.createRandomizedJob(); + Job job2 = JobTests.createRandomizedJob(); + Job job3 = JobTests.createRandomizedJob(); + + builder.putJob(job1, false); + builder.putJob(job2, false); + builder.putJob(job3, false); + + builder.putAllocation(job1.getId(), "node1"); + builder.putAllocation(job2.getId(), "node1"); + builder.putAllocation(job3.getId(), "node1"); + + PrelertMetadata expected = builder.build(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + expected.writeTo(new OutputStreamStreamOutput(out)); + + PrelertMetadata result = (PrelertMetadata) + PrelertMetadata.PROTO.readFrom(new InputStreamStreamInput(new ByteArrayInputStream(out.toByteArray()))); + assertThat(result, equalTo(expected)); + } + + public void testFromXContent() throws IOException { + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(); + + Job job1 = JobTests.createRandomizedJob(); + Job job2 = JobTests.createRandomizedJob(); + Job job3 = JobTests.createRandomizedJob(); + + builder.putJob(job1, false); + builder.putJob(job2, false); + builder.putJob(job3, false); + + builder.putAllocation(job1.getId(), "node1"); + builder.putAllocation(job2.getId(), "node1"); + builder.putAllocation(job3.getId(), "node1"); + + PrelertMetadata expected = builder.build(); + + XContentBuilder xBuilder = XContentFactory.contentBuilder(XContentType.SMILE); + xBuilder.prettyPrint(); + xBuilder.startObject(); + expected.toXContent(xBuilder, ToXContent.EMPTY_PARAMS); + xBuilder.endObject(); + XContentBuilder shuffled = shuffleXContent(xBuilder); + final XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes()); + MetaData.Custom custom = expected.fromXContent(parser); + assertTrue(custom instanceof PrelertMetadata); + PrelertMetadata result = (PrelertMetadata) custom; + assertThat(result, equalTo(expected)); + } + + public void testPutJob() { + PrelertMetadata.Builder builder = new PrelertMetadata.Builder(); + builder.putJob(buildJobBuilder("1").build(), false); + builder.putJob(buildJobBuilder("2").build(), false); + + ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, + () -> builder.putJob(buildJobBuilder("2").build(), false)); + assertEquals("The job cannot be created with the Id '2'. The Id is already used.", e.getMessage()); + + builder.putJob(buildJobBuilder("2").build(), true); + + PrelertMetadata result = builder.build(); + assertThat(result.getJobs().size(), equalTo(2)); + assertThat(result.getJobs().get("1"), notNullValue()); + assertThat(result.getJobs().get("2"), notNullValue()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilderTests.java new file mode 100644 index 00000000000..79797e61f2d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketQueryBuilderTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.test.ESTestCase; +import org.junit.Assert; + +public class BucketQueryBuilderTests extends ESTestCase { + + public void testDefaultBuild() throws Exception { + BucketQueryBuilder.BucketQuery query = new BucketQueryBuilder("1000").build(); + + Assert.assertEquals("1000", query.getTimestamp()); + assertEquals(false, query.isIncludeInterim()); + assertEquals(false, query.isExpand()); + assertEquals(null, query.getPartitionValue()); + } + + public void testDefaultAll() throws Exception { + BucketQueryBuilder.BucketQuery query = + new BucketQueryBuilder("1000") + .expand(true) + .includeInterim(true) + .partitionValue("p") + .build(); + + Assert.assertEquals("1000", query.getTimestamp()); + assertEquals(true, query.isIncludeInterim()); + assertEquals(true, query.isExpand()); + assertEquals("p", query.getPartitionValue()); + } + + public void testEqualsHash() throws Exception { + BucketQueryBuilder.BucketQuery query = + new BucketQueryBuilder("1000") + .expand(true) + .includeInterim(true) + .partitionValue("p") + .build(); + + BucketQueryBuilder.BucketQuery query2 = + new BucketQueryBuilder("1000") + .expand(true) + .includeInterim(true) + .partitionValue("p") + .build(); + + assertEquals(query2, query); + assertEquals(query2.hashCode(), query.hashCode()); + + query2 = + new BucketQueryBuilder("1000") + .expand(true) + .includeInterim(true) + .partitionValue("q") + .build(); + + assertFalse(query2.equals(query)); + assertFalse(query2.hashCode() == query.hashCode()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilderTests.java new file mode 100644 index 00000000000..4a033dc7ff6 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/BucketsQueryBuilderTests.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.test.ESTestCase; + +public class BucketsQueryBuilderTests extends ESTestCase { + + public void testDefaultBuild() throws Exception { + BucketsQueryBuilder.BucketsQuery query = new BucketsQueryBuilder().build(); + + assertEquals(0, query.getFrom()); + assertEquals(BucketsQueryBuilder.DEFAULT_SIZE, query.getSize()); + assertEquals(false, query.isIncludeInterim()); + assertEquals(false, query.isExpand()); + assertEquals(0.0, query.getAnomalyScoreFilter(), 0.0001); + assertEquals(0.0, query.getNormalizedProbability(), 0.0001); + assertNull(query.getEpochStart()); + assertNull(query.getEpochEnd()); + assertEquals("timestamp", query.getSortField()); + assertFalse(query.isSortDescending()); + } + + public void testAll() { + BucketsQueryBuilder.BucketsQuery query = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .epochStart("1000") + .epochEnd("2000") + .partitionValue("foo") + .sortField("anomalyScore") + .sortDescending(true) + .build(); + + assertEquals(20, query.getFrom()); + assertEquals(40, query.getSize()); + assertEquals(true, query.isIncludeInterim()); + assertEquals(true, query.isExpand()); + assertEquals(50.0d, query.getAnomalyScoreFilter(), 0.00001); + assertEquals(70.0d, query.getNormalizedProbability(), 0.00001); + assertEquals("1000", query.getEpochStart()); + assertEquals("2000", query.getEpochEnd()); + assertEquals("foo", query.getPartitionValue()); + assertEquals("anomalyScore", query.getSortField()); + assertTrue(query.isSortDescending()); + } + + public void testEqualsHash() { + BucketsQueryBuilder query = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .epochStart("1000") + .epochEnd("2000") + .partitionValue("foo"); + + BucketsQueryBuilder query2 = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .epochStart("1000") + .epochEnd("2000") + .partitionValue("foo"); + + assertEquals(query.build(), query2.build()); + assertEquals(query.build().hashCode(), query2.build().hashCode()); + query2.clear(); + assertFalse(query.build().equals(query2.build())); + + query2.from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .epochStart("1000") + .epochEnd("2000") + .partitionValue("foo"); + assertEquals(query.build(), query2.build()); + + query2.clear(); + query2.from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.1d) + .normalizedProbabilityThreshold(70.0d) + .epochStart("1000") + .epochEnd("2000") + .partitionValue("foo"); + assertFalse(query.build().equals(query2.build())); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditorTests.java new file mode 100644 index 00000000000..b19a88f7549 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchAuditorTests.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.elasticsearch.xpack.prelert.job.audit.AuditActivity; +import org.elasticsearch.xpack.prelert.job.audit.AuditMessage; +import org.elasticsearch.xpack.prelert.job.audit.Level; + +public class ElasticsearchAuditorTests extends ESTestCase { + private Client client; + private ListenableActionFuture indexResponse; + private ArgumentCaptor indexCaptor; + private ArgumentCaptor jsonCaptor; + + @SuppressWarnings("unchecked") + @Before + public void setUpMocks() { + client = Mockito.mock(Client.class); + indexResponse = Mockito.mock(ListenableActionFuture.class); + indexCaptor = ArgumentCaptor.forClass(String.class); + jsonCaptor = ArgumentCaptor.forClass(XContentBuilder.class); + } + + + public void testInfo() { + givenClientPersistsSuccessfully(); + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "prelert-int", "foo"); + + auditor.info("Here is my info"); + + assertEquals("prelert-int", indexCaptor.getValue()); + AuditMessage auditMessage = parseAuditMessage(); + assertEquals("foo", auditMessage.getJobId()); + assertEquals("Here is my info", auditMessage.getMessage()); + assertEquals(Level.INFO, auditMessage.getLevel()); + } + + + public void testWarning() { + givenClientPersistsSuccessfully(); + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "someIndex", "bar"); + + auditor.warning("Here is my warning"); + + assertEquals("someIndex", indexCaptor.getValue()); + AuditMessage auditMessage = parseAuditMessage(); + assertEquals("bar", auditMessage.getJobId()); + assertEquals("Here is my warning", auditMessage.getMessage()); + assertEquals(Level.WARNING, auditMessage.getLevel()); + } + + + public void testError() { + givenClientPersistsSuccessfully(); + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "someIndex", "foobar"); + + auditor.error("Here is my error"); + + assertEquals("someIndex", indexCaptor.getValue()); + AuditMessage auditMessage = parseAuditMessage(); + assertEquals("foobar", auditMessage.getJobId()); + assertEquals("Here is my error", auditMessage.getMessage()); + assertEquals(Level.ERROR, auditMessage.getLevel()); + } + + + public void testActivity_GivenString() { + givenClientPersistsSuccessfully(); + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "someIndex", ""); + + auditor.activity("Here is my activity"); + + assertEquals("someIndex", indexCaptor.getValue()); + AuditMessage auditMessage = parseAuditMessage(); + assertEquals("", auditMessage.getJobId()); + assertEquals("Here is my activity", auditMessage.getMessage()); + assertEquals(Level.ACTIVITY, auditMessage.getLevel()); + } + + + public void testActivity_GivenNumbers() { + givenClientPersistsSuccessfully(); + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "someIndex", ""); + + auditor.activity(10, 100, 5, 50); + + assertEquals("someIndex", indexCaptor.getValue()); + AuditActivity auditActivity = parseAuditActivity(); + assertEquals(10, auditActivity.getTotalJobs()); + assertEquals(100, auditActivity.getTotalDetectors()); + assertEquals(5, auditActivity.getRunningJobs()); + assertEquals(50, auditActivity.getRunningDetectors()); + } + + + public void testError_GivenNoSuchIndex() { + when(client.prepareIndex("someIndex", "auditMessage")) + .thenThrow(new IndexNotFoundException("someIndex")); + + ElasticsearchAuditor auditor = new ElasticsearchAuditor(client, "someIndex", "foobar"); + + auditor.error("Here is my error"); + } + + private void givenClientPersistsSuccessfully() { + IndexRequestBuilder indexRequestBuilder = Mockito.mock(IndexRequestBuilder.class); + when(indexRequestBuilder.setSource(jsonCaptor.capture())).thenReturn(indexRequestBuilder); + when(indexRequestBuilder.execute()).thenReturn(indexResponse); + when(client.prepareIndex(indexCaptor.capture(), eq("auditMessage"))) + .thenReturn(indexRequestBuilder); + when(client.prepareIndex(indexCaptor.capture(), eq("auditActivity"))) + .thenReturn(indexRequestBuilder); + } + + private AuditMessage parseAuditMessage() { + try { + String json = jsonCaptor.getValue().string(); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + return AuditMessage.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT); + } catch (IOException e) { + return new AuditMessage(); + } + } + + private AuditActivity parseAuditActivity() { + try { + String json = jsonCaptor.getValue().string(); + XContentParser parser = XContentFactory.xContent(json).createParser(json); + return AuditActivity.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT); + } catch (IOException e) { + return new AuditActivity(); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIteratorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIteratorTests.java new file mode 100644 index 00000000000..602f2f0b0ed --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchBatchedDocumentsIteratorTests.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.NoSuchElementException; + +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +public class ElasticsearchBatchedDocumentsIteratorTests extends ESTestCase { + private static final String INDEX_NAME = "prelertresults-foo"; + private static final String SCROLL_ID = "someScrollId"; + + private Client client; + private boolean wasScrollCleared; + + private TestIterator testIterator; + + @Before + public void setUpMocks() { + client = Mockito.mock(Client.class); + wasScrollCleared = false; + testIterator = new TestIterator(client, INDEX_NAME, ParseFieldMatcher.STRICT); + givenClearScrollRequest(); + } + + public void testQueryReturnsNoResults() { + new ScrollResponsesMocker().finishMock(); + + assertTrue(testIterator.hasNext()); + assertTrue(testIterator.next().isEmpty()); + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + public void testCallingNextWhenHasNextIsFalseThrows() { + new ScrollResponsesMocker().addBatch("a", "b", "c").finishMock(); + testIterator.next(); + assertFalse(testIterator.hasNext()); + + ESTestCase.expectThrows(NoSuchElementException.class, () -> testIterator.next()); + } + + public void testQueryReturnsSingleBatch() { + new ScrollResponsesMocker().addBatch("a", "b", "c").finishMock(); + + assertTrue(testIterator.hasNext()); + Deque batch = testIterator.next(); + assertEquals(3, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("a", "b", "c"))); + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + public void testQueryReturnsThreeBatches() { + new ScrollResponsesMocker() + .addBatch("a", "b", "c") + .addBatch("d", "e") + .addBatch("f") + .finishMock(); + + assertTrue(testIterator.hasNext()); + + Deque batch = testIterator.next(); + assertEquals(3, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("a", "b", "c"))); + + batch = testIterator.next(); + assertEquals(2, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("d", "e"))); + + batch = testIterator.next(); + assertEquals(1, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("f"))); + + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + private void givenClearScrollRequest() { + ClearScrollRequestBuilder requestBuilder = mock(ClearScrollRequestBuilder.class); + when(client.prepareClearScroll()).thenReturn(requestBuilder); + when(requestBuilder.setScrollIds(Arrays.asList(SCROLL_ID))).thenReturn(requestBuilder); + when(requestBuilder.get()).thenAnswer((invocation) -> { + wasScrollCleared = true; + return null; + }); + } + + private class ScrollResponsesMocker { + private List batches = new ArrayList<>(); + private long totalHits = 0; + private List nextRequestBuilders = new ArrayList<>(); + + ScrollResponsesMocker addBatch(String... hits) { + totalHits += hits.length; + batches.add(hits); + return this; + } + + void finishMock() { + if (batches.isEmpty()) { + givenInitialResponse(); + return; + } + givenInitialResponse(batches.get(0)); + for (int i = 1; i < batches.size(); ++i) { + givenNextResponse(batches.get(i)); + } + if (nextRequestBuilders.size() > 0) { + SearchScrollRequestBuilder first = nextRequestBuilders.get(0); + if (nextRequestBuilders.size() > 1) { + SearchScrollRequestBuilder[] rest = new SearchScrollRequestBuilder[batches.size() - 1]; + for (int i = 1; i < nextRequestBuilders.size(); ++i) { + rest[i - 1] = nextRequestBuilders.get(i); + } + when(client.prepareSearchScroll(SCROLL_ID)).thenReturn(first, rest); + } else { + when(client.prepareSearchScroll(SCROLL_ID)).thenReturn(first); + } + } + } + + private void givenInitialResponse(String... hits) { + SearchResponse searchResponse = createSearchResponseWithHits(hits); + SearchRequestBuilder requestBuilder = mock(SearchRequestBuilder.class); + when(client.prepareSearch(INDEX_NAME)).thenReturn(requestBuilder); + when(requestBuilder.setScroll("5m")).thenReturn(requestBuilder); + when(requestBuilder.setSize(10000)).thenReturn(requestBuilder); + when(requestBuilder.setTypes("String")).thenReturn(requestBuilder); + when(requestBuilder.setQuery(any(QueryBuilder.class))).thenReturn(requestBuilder); + when(requestBuilder.addSort(any(SortBuilder.class))).thenReturn(requestBuilder); + when(requestBuilder.get()).thenReturn(searchResponse); + } + + private void givenNextResponse(String... hits) { + SearchResponse searchResponse = createSearchResponseWithHits(hits); + SearchScrollRequestBuilder requestBuilder = mock(SearchScrollRequestBuilder.class); + when(requestBuilder.setScrollId(SCROLL_ID)).thenReturn(requestBuilder); + when(requestBuilder.setScroll("5m")).thenReturn(requestBuilder); + when(requestBuilder.get()).thenReturn(searchResponse); + nextRequestBuilders.add(requestBuilder); + } + + private SearchResponse createSearchResponseWithHits(String... hits) { + SearchHits searchHits = createHits(hits); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getScrollId()).thenReturn(SCROLL_ID); + when(searchResponse.getHits()).thenReturn(searchHits); + return searchResponse; + } + + private SearchHits createHits(String... values) { + SearchHits searchHits = mock(SearchHits.class); + List hits = new ArrayList<>(); + for (String value : values) { + SearchHit hit = mock(SearchHit.class); + when(hit.getSourceAsString()).thenReturn(value); + hits.add(hit); + } + when(searchHits.getTotalHits()).thenReturn(totalHits); + when(searchHits.getHits()).thenReturn(hits.toArray(new SearchHit[hits.size()])); + return searchHits; + } + } + + private static class TestIterator extends ElasticsearchBatchedDocumentsIterator { + public TestIterator(Client client, String jobId, ParseFieldMatcher parseFieldMatcher) { + super(client, jobId, parseFieldMatcher); + } + + @Override + protected String getType() { + return "String"; + } + + @Override + protected String map(SearchHit hit) { + return hit.getSourceAsString(); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverserTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverserTests.java new file mode 100644 index 00000000000..78d8eb86c30 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchDotNotationReverserTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; + + +public class ElasticsearchDotNotationReverserTests extends ESTestCase { + public void testResultsMap() throws Exception { + ElasticsearchDotNotationReverser reverser = createReverser(); + + String expected = "{\"complex\":{\"nested\":{\"structure\":{\"first\":\"x\"," + + "\"second\":\"y\"},\"value\":\"z\"}},\"cpu\":{\"system\":\"5\"," + + "\"user\":\"10\",\"wait\":\"1\"},\"simple\":\"simon\"}"; + + String actual = XContentFactory.jsonBuilder().map(reverser.getResultsMap()).string(); + assertEquals(expected, actual); + } + + public void testMappingsMap() throws Exception { + ElasticsearchDotNotationReverser reverser = createReverser(); + + String expected = "{\"complex\":{\"properties\":{\"nested\":{\"properties\":" + + "{\"structure\":{\"properties\":{\"first\":{\"type\":\"keyword\"}," + + "\"second\":{\"type\":\"keyword\"}},\"type\":\"object\"}," + + "\"value\":{\"type\":\"keyword\"}},\"type\":\"object\"}}," + + "\"type\":\"object\"},\"cpu\":{\"properties\":{\"system\":" + + "{\"type\":\"keyword\"},\"user\":{\"type\":\"keyword\"}," + + "\"wait\":{\"type\":\"keyword\"}},\"type\":\"object\"}," + + "\"simple\":{\"type\":\"keyword\"}}"; + + String actual = XContentFactory.jsonBuilder().map(reverser.getMappingsMap()).string(); + assertEquals(expected, actual); + } + + private ElasticsearchDotNotationReverser createReverser() { + ElasticsearchDotNotationReverser reverser = new ElasticsearchDotNotationReverser(); + // This should get ignored as it's a reserved field name + reverser.add("bucketSpan", "3600"); + reverser.add("simple", "simon"); + reverser.add("cpu.user", "10"); + reverser.add("cpu.system", "5"); + reverser.add("cpu.wait", "1"); + // This should get ignored as one of its segments is a reserved field name + reverser.add("foo.bucketSpan", "3600"); + reverser.add("complex.nested.structure.first", "x"); + reverser.add("complex.nested.structure.second", "y"); + reverser.add("complex.nested.value", "z"); + return reverser; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapperTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapperTests.java new file mode 100644 index 00000000000..ee43c2ca83f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobDetailsMapperTests.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ElasticsearchJobDetailsMapperTests extends ESTestCase { + private Client client; + + @Before + public void setUpMocks() { + client = Mockito.mock(Client.class); + } + + public void testMap_GivenJobSourceCannotBeParsed() { + BytesArray source = new BytesArray("{ \"invalidKey\": true }"); + + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(false); + GetRequestBuilder getRequestBuilder = mock(GetRequestBuilder.class); + when(getRequestBuilder.get()).thenReturn(getResponse); + when(client.prepareGet("prelertresults-foo", ModelSizeStats.TYPE.getPreferredName(), ModelSizeStats.TYPE.getPreferredName())) + .thenReturn(getRequestBuilder); + + ElasticsearchJobDetailsMapper mapper = new ElasticsearchJobDetailsMapper(client, ParseFieldMatcher.STRICT); + + ESTestCase.expectThrows(IllegalArgumentException.class, () -> mapper.map(source)); + } + + public void testMap_GivenModelSizeStatsExists() throws Exception { + ModelSizeStats.Builder modelSizeStats = new ModelSizeStats.Builder("foo"); + modelSizeStats.setModelBytes(42L); + Date now = new Date(); + modelSizeStats.setTimestamp(now); + + Job originalJob = buildJobBuilder("foo").build(); + + BytesReference source = originalJob.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes(); + BytesReference modelSizeStatsSource = modelSizeStats.build().toXContent( + XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes(); + + GetResponse getModelSizeResponse = mock(GetResponse.class); + when(getModelSizeResponse.isExists()).thenReturn(true); + when(getModelSizeResponse.getSourceAsBytesRef()).thenReturn(modelSizeStatsSource); + GetRequestBuilder getModelSizeRequestBuilder = mock(GetRequestBuilder.class); + when(getModelSizeRequestBuilder.get()).thenReturn(getModelSizeResponse); + when(client.prepareGet("prelertresults-foo", ModelSizeStats.TYPE.getPreferredName(), ModelSizeStats.TYPE.getPreferredName())) + .thenReturn(getModelSizeRequestBuilder); + + + Map procTimeSource = new HashMap<>(); + procTimeSource.put(ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS, 20.2); + + GetResponse getProcTimeResponse = mock(GetResponse.class); + when(getProcTimeResponse.isExists()).thenReturn(true); + when(getProcTimeResponse.getSource()).thenReturn(procTimeSource); + GetRequestBuilder getProcTimeRequestBuilder = mock(GetRequestBuilder.class); + when(getProcTimeRequestBuilder.get()).thenReturn(getProcTimeResponse); + when(client.prepareGet("prelertresults-foo", ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE, + ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS)) + .thenReturn(getProcTimeRequestBuilder); + + + ElasticsearchJobDetailsMapper mapper = new ElasticsearchJobDetailsMapper(client, ParseFieldMatcher.STRICT); + + Job mappedJob = mapper.map(source); + + assertEquals("foo", mappedJob.getId()); + assertEquals(42L, mappedJob.getModelSizeStats().getModelBytes()); + assertEquals(now, mappedJob.getModelSizeStats().getTimestamp()); + assertEquals(20.2, mappedJob.getAverageBucketProcessingTimeMs(), 0.0001); + } + + public void testMap_GivenModelSizeStatsDoesNotExist() throws Exception { + Job originalJob = buildJobBuilder("foo").build(); + + BytesReference source = originalJob.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes(); + + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(false); + GetRequestBuilder getRequestBuilder = mock(GetRequestBuilder.class); + when(getRequestBuilder.get()).thenReturn(getResponse); + when(client.prepareGet("prelertresults-foo", ModelSizeStats.TYPE.getPreferredName(), ModelSizeStats.TYPE.getPreferredName())) + .thenReturn(getRequestBuilder); + + + GetResponse getProcTimeResponse = mock(GetResponse.class); + when(getProcTimeResponse.isExists()).thenReturn(false); + GetRequestBuilder getProcTimeRequestBuilder = mock(GetRequestBuilder.class); + when(getProcTimeRequestBuilder.get()).thenReturn(getProcTimeResponse); + when(client.prepareGet("prelertresults-foo", ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE, + ReservedFieldNames.AVERAGE_PROCESSING_TIME_MS)) + .thenReturn(getProcTimeRequestBuilder); + + ElasticsearchJobDetailsMapper mapper = new ElasticsearchJobDetailsMapper(client, ParseFieldMatcher.STRICT); + + Job mappedJob = mapper.map(source); + + assertEquals("foo", mappedJob.getId()); + assertNull(mappedJob.getModelSizeStats()); + assertNull(mappedJob.getAverageBucketProcessingTimeMs()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProviderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProviderTests.java new file mode 100644 index 00000000000..d417fcabc89 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchJobProviderTests.java @@ -0,0 +1,1044 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.AnalysisLimits; +import org.elasticsearch.xpack.prelert.job.CategorizerState; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.ModelState; +import org.elasticsearch.xpack.prelert.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; + +import static org.elasticsearch.xpack.prelert.job.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ElasticsearchJobProviderTests extends ESTestCase { + private static final String CLUSTER_NAME = "myCluster"; + private static final String JOB_ID = "foo"; + private static final String INDEX_NAME = "prelertresults-foo"; + + @Captor + private ArgumentCaptor> mapCaptor; + + public void testGetQuantiles_GivenNoIndexForJob() throws InterruptedException, ExecutionException { + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .throwMissingIndexOnPrepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + ESTestCase.expectThrows(IndexNotFoundException.class, () -> provider.getQuantiles(JOB_ID)); + } + + public void testGetQuantiles_GivenNoQuantilesForJob() throws Exception { + GetResponse getResponse = createGetResponse(false, null); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + Optional quantiles = provider.getQuantiles(JOB_ID); + + assertFalse(quantiles.isPresent()); + } + + public void testGetQuantiles_GivenQuantilesHaveNonEmptyState() throws Exception { + Map source = new HashMap<>(); + source.put(Quantiles.JOB_ID.getPreferredName(), "foo"); + source.put(Quantiles.TIMESTAMP.getPreferredName(), 0L); + source.put(Quantiles.QUANTILE_STATE.getPreferredName(), "state"); + GetResponse getResponse = createGetResponse(true, source); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + Optional quantiles = provider.getQuantiles(JOB_ID); + + assertTrue(quantiles.isPresent()); + assertEquals("state", quantiles.get().getQuantileState()); + } + + public void testGetQuantiles_GivenQuantilesHaveEmptyState() throws Exception { + Map source = new HashMap<>(); + source.put(Quantiles.JOB_ID.getPreferredName(), "foo"); + source.put(Quantiles.TIMESTAMP.getPreferredName(), new Date(0L).getTime()); + source.put(Quantiles.QUANTILE_STATE.getPreferredName(), ""); + GetResponse getResponse = createGetResponse(true, source); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareGet(INDEX_NAME, Quantiles.TYPE.getPreferredName(), Quantiles.QUANTILES_ID, getResponse); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + Optional quantiles = provider.getQuantiles(JOB_ID); + + assertTrue(quantiles.isPresent()); + assertEquals("", quantiles.get().getQuantileState()); + } + + public void testCreateUsageMetering() throws InterruptedException, ExecutionException { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, false) + .prepareCreate(ElasticsearchJobProvider.PRELERT_USAGE_INDEX) + .addClusterStatusYellowResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX); + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + provider.initialize(); + clientBuilder.verifyIndexCreated(ElasticsearchJobProvider.PRELERT_USAGE_INDEX); + } + + public void testCreateJob() throws InterruptedException, ExecutionException { + Job.Builder job = buildJobBuilder("marscapone"); + job.setDescription("This is a very cheesy job"); + AnalysisLimits limits = new AnalysisLimits(9878695309134L, null); + job.setAnalysisLimits(limits); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).createIndexRequest("prelertresults-" + job.getId()); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + AtomicReference resultHolder = new AtomicReference<>(); + provider.createJobRelatedIndices(job.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + resultHolder.set(aBoolean); + } + + @Override + public void onFailure(Exception e) { + + } + }); + assertNotNull(resultHolder.get()); + assertTrue(resultHolder.get()); + } + + public void testDeleteJob() throws InterruptedException, ExecutionException, IOException { + @SuppressWarnings("unchecked") + ActionListener actionListener = mock(ActionListener.class); + String jobId = "ThisIsMyJob"; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true); + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + clientBuilder.resetIndices(); + clientBuilder.addIndicesExistsResponse("prelertresults-" + jobId, true).addIndicesDeleteResponse("prelertresults-" + jobId, true, + false, actionListener); + clientBuilder.build(); + + provider.deleteJobRelatedIndices(jobId, actionListener); + + verify(actionListener).onResponse(true); + } + + public void testDeleteJob_InvalidIndex() throws InterruptedException, ExecutionException, IOException { + @SuppressWarnings("unchecked") + ActionListener actionListener = mock(ActionListener.class); + String jobId = "ThisIsMyJob"; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true); + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + clientBuilder.resetIndices(); + clientBuilder.addIndicesExistsResponse("prelertresults-" + jobId, true).addIndicesDeleteResponse("prelertresults-" + jobId, true, + true, actionListener); + clientBuilder.build(); + + provider.deleteJobRelatedIndices(jobId, actionListener); + + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(exceptionCaptor.capture()); + assertThat(exceptionCaptor.getValue(), instanceOf(InterruptedException.class)); + } + + public void testBuckets_OneBucketNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("timestamp", now.getTime()); + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + int from = 0; + int size = 10; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(0.0) + .normalizedProbabilityThreshold(1.0); + + QueryPage buckets = provider.buckets(jobId, bq.build()); + assertEquals(1L, buckets.hitCount()); + QueryBuilder query = queryBuilder.getValue(); + String queryString = query.toString(); + assertTrue( + queryString.matches("(?s).*maxNormalizedProbability[^}]*from. : 1\\.0.*must_not[^}]*term[^}]*isInterim.*value. : .true.*")); + } + + public void testBuckets_OneBucketInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("timestamp", now.getTime()); + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + int from = 99; + int size = 17; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(5.1) + .normalizedProbabilityThreshold(10.9).includeInterim(true); + + QueryPage buckets = provider.buckets(jobId, bq.build()); + assertEquals(1L, buckets.hitCount()); + QueryBuilder query = queryBuilder.getValue(); + String queryString = query.toString(); + assertTrue(queryString.matches("(?s).*maxNormalizedProbability[^}]*from. : 10\\.9.*")); + assertTrue(queryString.matches("(?s).*anomalyScore[^}]*from. : 5\\.1.*")); + assertFalse(queryString.matches("(?s).*isInterim.*")); + } + + public void testBuckets_UsingBuilder() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("timestamp", now.getTime()); + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + int from = 99; + int size = 17; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder(); + bq.from(from); + bq.size(size); + bq.anomalyScoreThreshold(5.1); + bq.normalizedProbabilityThreshold(10.9); + bq.includeInterim(true); + + QueryPage buckets = provider.buckets(jobId, bq.build()); + assertEquals(1L, buckets.hitCount()); + QueryBuilder query = queryBuilder.getValue(); + String queryString = query.toString(); + assertTrue(queryString.matches("(?s).*maxNormalizedProbability[^}]*from. : 10\\.9.*")); + assertTrue(queryString.matches("(?s).*anomalyScore[^}]*from. : 5\\.1.*")); + assertFalse(queryString.matches("(?s).*isInterim.*")); + } + + public void testBucket_NoBucketNoExpandNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Long timestamp = 98765432123456789L; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("timestamp", now.getTime()); + // source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(false, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), 0, 0, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketQueryBuilder bq = new BucketQueryBuilder(Long.toString(timestamp)); + + QueryPage bucket = provider.bucket(jobId, bq.build()); + assertThat(bucket.hitCount(), equalTo(0L)); + assertThat(bucket.hits(), empty()); + } + + public void testBucket_OneBucketNoExpandNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("timestamp", now.getTime()); + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), 0, 0, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketQueryBuilder bq = new BucketQueryBuilder(Long.toString(now.getTime())); + + QueryPage bucketHolder = provider.bucket(jobId, bq.build()); + assertThat(bucketHolder.hitCount(), equalTo(1L)); + Bucket b = bucketHolder.hits().get(0); + assertEquals(now, b.getTimestamp()); + } + + public void testBucket_OneBucketNoExpandInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("timestamp", now.getTime()); + map.put("isInterim", true); + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Bucket.TYPE.getPreferredName(), 0, 0, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + BucketQueryBuilder bq = new BucketQueryBuilder(Long.toString(now.getTime())); + + QueryPage bucketHolder = provider.bucket(jobId, bq.build()); + assertThat(bucketHolder.hitCount(), equalTo(0L)); + assertThat(bucketHolder.hits(), empty()); + } + + public void testRecords() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucketSpan", 22); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucketSpan", 22); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, AnomalyRecord.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + RecordsQueryBuilder rqb = new RecordsQueryBuilder().from(from).size(size).epochStart(String.valueOf(now.getTime())) + .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield).anomalyScoreThreshold(11.1) + .normalizedProbability(2.2); + + QueryPage recordPage = provider.records(jobId, rqb.build()); + assertEquals(2L, recordPage.hitCount()); + List records = recordPage.hits(); + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testRecords_UsingBuilder() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucketSpan", 22); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucketSpan", 22); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, AnomalyRecord.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + RecordsQueryBuilder rqb = new RecordsQueryBuilder(); + rqb.from(from); + rqb.size(size); + rqb.epochStart(String.valueOf(now.getTime())); + rqb.epochEnd(String.valueOf(now.getTime())); + rqb.includeInterim(true); + rqb.sortField(sortfield); + rqb.anomalyScoreThreshold(11.1); + rqb.normalizedProbability(2.2); + + QueryPage recordPage = provider.records(jobId, rqb.build()); + assertEquals(2L, recordPage.hitCount()); + List records = recordPage.hits(); + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testBucketRecords() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = mock(Bucket.class); + when(bucket.getTimestamp()).thenReturn(now); + + List> source = new ArrayList<>(); + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucketSpan", 22); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucketSpan", 22); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, AnomalyRecord.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + QueryPage recordPage = provider.bucketRecords(jobId, bucket, from, size, true, sortfield, true, ""); + + assertEquals(2L, recordPage.hitCount()); + List records = recordPage.hits(); + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testexpandBucket() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = new Bucket("foo"); + bucket.setTimestamp(now); + + List> source = new ArrayList<>(); + for (int i = 0; i < 400; i++) { + Map recordMap = new HashMap<>(); + recordMap.put("jobId", "foo"); + recordMap.put("typical", 22.4 + i); + recordMap.put("actual", 33.3 + i); + recordMap.put("timestamp", now.getTime()); + recordMap.put("function", "irritable"); + recordMap.put("bucketSpan", 22); + source.add(recordMap); + } + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearchAnySize("prelertresults-" + jobId, AnomalyRecord.TYPE.getPreferredName(), response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + int records = provider.expandBucket(jobId, false, bucket); + assertEquals(400L, records); + } + + public void testexpandBucket_WithManyRecords() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = new Bucket("foo"); + bucket.setTimestamp(now); + + List> source = new ArrayList<>(); + for (int i = 0; i < 600; i++) { + Map recordMap = new HashMap<>(); + recordMap.put("jobId", "foo"); + recordMap.put("typical", 22.4 + i); + recordMap.put("actual", 33.3 + i); + recordMap.put("timestamp", now.getTime()); + recordMap.put("function", "irritable"); + recordMap.put("bucketSpan", 22); + source.add(recordMap); + } + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearchAnySize("prelertresults-" + jobId, AnomalyRecord.TYPE.getPreferredName(), response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + int records = provider.expandBucket(jobId, false, bucket); + // This is not realistic, but is an artifact of the fact that the mock + // query + // returns all the records, not a subset + assertEquals(1200L, records); + } + + public void testCategoryDefinitions() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + String terms = "the terms and conditions are not valid here"; + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("jobId", "foo"); + map.put("categoryId", String.valueOf(map.hashCode())); + map.put("terms", terms); + + source.add(map); + + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + int from = 0; + int size = 10; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, CategoryDefinition.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + QueryPage categoryDefinitions = provider.categoryDefinitions(jobId, from, size); + assertEquals(1L, categoryDefinitions.hitCount()); + assertEquals(terms, categoryDefinitions.hits().get(0).getTerms()); + } + + public void testCategoryDefinition() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + String terms = "the terms and conditions are not valid here"; + + Map source = new HashMap<>(); + String categoryId = String.valueOf(source.hashCode()); + source.put("jobId", "foo"); + source.put("categoryId", categoryId); + source.put("terms", terms); + + GetResponse getResponse = createGetResponse(true, source); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareGet("prelertresults-" + jobId, CategoryDefinition.TYPE.getPreferredName(), categoryId, getResponse); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + QueryPage categoryDefinitions = provider.categoryDefinition(jobId, categoryId); + assertEquals(1L, categoryDefinitions.hitCount()); + assertEquals(terms, categoryDefinitions.hits().get(0).getTerms()); + } + + public void testInfluencers_NoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("probability", 0.555); + recordMap1.put("influencerFieldName", "Builder"); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("influencerFieldValue", "Bob"); + recordMap1.put("initialAnomalyScore", 22.2); + recordMap1.put("anomalyScore", 22.6); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("probability", 0.99); + recordMap2.put("influencerFieldName", "Builder"); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("influencerFieldValue", "James"); + recordMap2.put("initialAnomalyScore", 5.0); + recordMap2.put("anomalyScore", 5.0); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Influencer.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).includeInterim(false).build(); + QueryPage page = provider.influencers(jobId, query); + assertEquals(2L, page.hitCount()); + + String queryString = queryBuilder.getValue().toString(); + assertTrue(queryString.matches("(?s).*must_not[^}]*term[^}]*isInterim.*value. : .true.*")); + + List records = page.hits(); + assertEquals("foo", records.get(0).getJobId()); + assertEquals("Bob", records.get(0).getInfluencerFieldValue()); + assertEquals("Builder", records.get(0).getInfluencerFieldName()); + assertEquals(now, records.get(0).getTimestamp()); + assertEquals(0.555, records.get(0).getProbability(), 0.00001); + assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + + assertEquals("James", records.get(1).getInfluencerFieldValue()); + assertEquals("Builder", records.get(1).getInfluencerFieldName()); + assertEquals(now, records.get(1).getTimestamp()); + assertEquals(0.99, records.get(1).getProbability(), 0.00001); + assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + } + + public void testInfluencers_WithInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("probability", 0.555); + recordMap1.put("influencerFieldName", "Builder"); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("influencerFieldValue", "Bob"); + recordMap1.put("initialAnomalyScore", 22.2); + recordMap1.put("anomalyScore", 22.6); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("probability", 0.99); + recordMap2.put("influencerFieldName", "Builder"); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("influencerFieldValue", "James"); + recordMap2.put("initialAnomalyScore", 5.0); + recordMap2.put("anomalyScore", 5.0); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, Influencer.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).epochStart("0").epochEnd("0").sortField("sort") + .sortDescending(true).anomalyScoreThreshold(0.0).includeInterim(true).build(); + QueryPage page = provider.influencers(jobId, query); + assertEquals(2L, page.hitCount()); + + String queryString = queryBuilder.getValue().toString(); + assertFalse(queryString.matches("(?s).*isInterim.*")); + + List records = page.hits(); + assertEquals("Bob", records.get(0).getInfluencerFieldValue()); + assertEquals("Builder", records.get(0).getInfluencerFieldName()); + assertEquals(now, records.get(0).getTimestamp()); + assertEquals(0.555, records.get(0).getProbability(), 0.00001); + assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + + assertEquals("James", records.get(1).getInfluencerFieldValue()); + assertEquals("Builder", records.get(1).getInfluencerFieldName()); + assertEquals(now, records.get(1).getTimestamp()); + assertEquals(0.99, records.get(1).getProbability(), 0.00001); + assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + } + + public void testInfluencer() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + String influencerId = "ThisIsAnInfluencerId"; + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + try { + provider.influencer(jobId, influencerId); + assertTrue(false); + } catch (IllegalStateException e) { + } + } + + public void testModelSnapshots() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("description", "snapshot1"); + recordMap1.put("restorePriority", 1); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("snapshotDocCount", 5); + recordMap1.put("latestRecordTimeStamp", now.getTime()); + recordMap1.put("latestResultTimeStamp", now.getTime()); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("description", "snapshot2"); + recordMap2.put("restorePriority", 999); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("snapshotDocCount", 6); + recordMap2.put("latestRecordTimeStamp", now.getTime()); + recordMap2.put("latestResultTimeStamp", now.getTime()); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, ModelSnapshot.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + QueryPage page = provider.modelSnapshots(jobId, from, size); + assertEquals(2L, page.hitCount()); + List snapshots = page.hits(); + + assertEquals("foo", snapshots.get(0).getJobId()); + assertEquals(now, snapshots.get(0).getTimestamp()); + assertEquals(now, snapshots.get(0).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(0).getLatestResultTimeStamp()); + assertEquals("snapshot1", snapshots.get(0).getDescription()); + assertEquals(1L, snapshots.get(0).getRestorePriority()); + assertEquals(5, snapshots.get(0).getSnapshotDocCount()); + + assertEquals(now, snapshots.get(1).getTimestamp()); + assertEquals(now, snapshots.get(1).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(1).getLatestResultTimeStamp()); + assertEquals("snapshot2", snapshots.get(1).getDescription()); + assertEquals(999L, snapshots.get(1).getRestorePriority()); + assertEquals(6, snapshots.get(1).getSnapshotDocCount()); + } + + public void testModelSnapshots_WithDescription() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("jobId", "foo"); + recordMap1.put("description", "snapshot1"); + recordMap1.put("restorePriority", 1); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("snapshotDocCount", 5); + recordMap1.put("latestRecordTimeStamp", now.getTime()); + recordMap1.put("latestResultTimeStamp", now.getTime()); + Map recordMap2 = new HashMap<>(); + recordMap2.put("jobId", "foo"); + recordMap2.put("description", "snapshot2"); + recordMap2.put("restorePriority", 999); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("snapshotDocCount", 6); + recordMap2.put("latestRecordTimeStamp", now.getTime()); + recordMap2.put("latestResultTimeStamp", now.getTime()); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + ArgumentCaptor queryBuilder = ArgumentCaptor.forClass(QueryBuilder.class); + SearchResponse response = createSearchResponse(true, source); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareSearch("prelertresults-" + jobId, ModelSnapshot.TYPE.getPreferredName(), from, size, response, queryBuilder); + + Client client = clientBuilder.build(); + ElasticsearchJobProvider provider = createProvider(client); + + QueryPage page = provider.modelSnapshots(jobId, from, size, null, null, "sortfield", true, "snappyId", + "description1"); + assertEquals(2L, page.hitCount()); + List snapshots = page.hits(); + + assertEquals(now, snapshots.get(0).getTimestamp()); + assertEquals(now, snapshots.get(0).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(0).getLatestResultTimeStamp()); + assertEquals("snapshot1", snapshots.get(0).getDescription()); + assertEquals(1L, snapshots.get(0).getRestorePriority()); + assertEquals(5, snapshots.get(0).getSnapshotDocCount()); + + assertEquals(now, snapshots.get(1).getTimestamp()); + assertEquals(now, snapshots.get(1).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(1).getLatestResultTimeStamp()); + assertEquals("snapshot2", snapshots.get(1).getDescription()); + assertEquals(999L, snapshots.get(1).getRestorePriority()); + assertEquals(6, snapshots.get(1).getSnapshotDocCount()); + + String queryString = queryBuilder.getValue().toString(); + assertTrue(queryString.matches("(?s).*snapshotId.*value. : .snappyId.*description.*value. : .description1.*")); + } + + public void testMergePartitionScoresIntoBucket() throws InterruptedException, ExecutionException { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME) + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true).addClusterStatusYellowResponse(); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + List scores = new ArrayList<>(); + scores.add(provider.new ScoreTimestamp(new Date(2), 1.0)); + scores.add(provider.new ScoreTimestamp(new Date(3), 2.0)); + scores.add(provider.new ScoreTimestamp(new Date(5), 3.0)); + + List buckets = new ArrayList<>(); + buckets.add(createBucketAtEpochTime(1)); + buckets.add(createBucketAtEpochTime(2)); + buckets.add(createBucketAtEpochTime(3)); + buckets.add(createBucketAtEpochTime(4)); + buckets.add(createBucketAtEpochTime(5)); + buckets.add(createBucketAtEpochTime(6)); + + provider.mergePartitionScoresIntoBucket(scores, buckets); + assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); + assertEquals(1.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); + assertEquals(2.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); + assertEquals(3.0, buckets.get(4).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(5).getMaxNormalizedProbability(), 0.001); + } + + public void testMergePartitionScoresIntoBucket_WithEmptyScoresList() throws InterruptedException, ExecutionException { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME) + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true).addClusterStatusYellowResponse(); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + List scores = new ArrayList<>(); + + List buckets = new ArrayList<>(); + buckets.add(createBucketAtEpochTime(1)); + buckets.add(createBucketAtEpochTime(2)); + buckets.add(createBucketAtEpochTime(3)); + buckets.add(createBucketAtEpochTime(4)); + + provider.mergePartitionScoresIntoBucket(scores, buckets); + assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); + } + + public void testRestoreStateToStream() throws Exception { + Map categorizerState = new HashMap<>(); + categorizerState.put("catName", "catVal"); + GetResponse categorizerStateGetResponse1 = createGetResponse(true, categorizerState); + GetResponse categorizerStateGetResponse2 = createGetResponse(false, null); + Map modelState = new HashMap<>(); + modelState.put("modName", "modVal1"); + GetResponse modelStateGetResponse1 = createGetResponse(true, modelState); + modelState.put("modName", "modVal2"); + GetResponse modelStateGetResponse2 = createGetResponse(true, modelState); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .addIndicesExistsResponse(ElasticsearchJobProvider.PRELERT_USAGE_INDEX, true) + .prepareGet(INDEX_NAME, CategorizerState.TYPE, "1", categorizerStateGetResponse1) + .prepareGet(INDEX_NAME, CategorizerState.TYPE, "2", categorizerStateGetResponse2) + .prepareGet(INDEX_NAME, ModelState.TYPE, "123_1", modelStateGetResponse1) + .prepareGet(INDEX_NAME, ModelState.TYPE, "123_2", modelStateGetResponse2); + + ElasticsearchJobProvider provider = createProvider(clientBuilder.build()); + + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setSnapshotId("123"); + modelSnapshot.setSnapshotDocCount(2); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + provider.restoreStateToStream(JOB_ID, modelSnapshot, stream); + + String[] restoreData = stream.toString(StandardCharsets.UTF_8.name()).split("\0"); + assertEquals(3, restoreData.length); + assertEquals("{\"catName\":\"catVal\"}", restoreData[0]); + assertEquals("{\"modName\":\"modVal1\"}", restoreData[1]); + assertEquals("{\"modName\":\"modVal2\"}", restoreData[2]); + } + + private Bucket createBucketAtEpochTime(long epoch) { + Bucket b = new Bucket("foo"); + b.setTimestamp(new Date(epoch)); + b.setMaxNormalizedProbability(10.0); + return b; + } + + private ElasticsearchJobProvider createProvider(Client client) { + return new ElasticsearchJobProvider(client, 0, ParseFieldMatcher.STRICT); + } + + private static GetResponse createGetResponse(boolean exists, Map source) throws IOException { + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(exists); + when(getResponse.getSourceAsBytesRef()).thenReturn(XContentFactory.jsonBuilder().map(source).bytes()); + return getResponse; + } + + private static SearchResponse createSearchResponse(boolean exists, List> source) throws IOException { + SearchResponse response = mock(SearchResponse.class); + SearchHits hits = mock(SearchHits.class); + List list = new ArrayList<>(); + + for (Map map : source) { + SearchHit hit = mock(SearchHit.class); + // remove the _parent from the field we use for _source + Map _source = new HashMap<>(map); + when(hit.getSourceRef()).thenReturn(XContentFactory.jsonBuilder().map(_source).bytes()); + when(hit.getId()).thenReturn(String.valueOf(map.hashCode())); + doAnswer(invocation -> { + String field = (String) invocation.getArguments()[0]; + SearchHitField shf = mock(SearchHitField.class); + when(shf.getValue()).thenReturn(map.get(field)); + return shf; + }).when(hit).field(any(String.class)); + list.add(hit); + } + when(response.getHits()).thenReturn(hits); + when(hits.getHits()).thenReturn(list.toArray(new SearchHit[0])); + when(hits.getTotalHits()).thenReturn((long) source.size()); + + doAnswer(invocation -> { + Integer idx = (Integer) invocation.getArguments()[0]; + return list.get(idx); + }).when(hits).getAt(any(Integer.class)); + + return response; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappingsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappingsTests.java new file mode 100644 index 00000000000..152bc5ab731 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchMappingsTests.java @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.prelert.job.CategorizerState; +import org.elasticsearch.xpack.prelert.job.DataCounts; +import org.elasticsearch.xpack.prelert.job.Job; +import org.elasticsearch.xpack.prelert.job.ModelSizeStats; +import org.elasticsearch.xpack.prelert.job.ModelSnapshot; +import org.elasticsearch.xpack.prelert.job.ModelState; +import org.elasticsearch.xpack.prelert.job.SchedulerConfig; +import org.elasticsearch.xpack.prelert.job.audit.AuditActivity; +import org.elasticsearch.xpack.prelert.job.audit.AuditMessage; +import org.elasticsearch.xpack.prelert.job.metadata.Allocation; +import org.elasticsearch.xpack.prelert.job.quantiles.Quantiles; +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.CategoryDefinition; +import org.elasticsearch.xpack.prelert.job.results.Influencer; +import org.elasticsearch.xpack.prelert.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.prelert.job.results.ReservedFieldNames; +import org.elasticsearch.xpack.prelert.job.usage.Usage; +import org.elasticsearch.xpack.prelert.lists.ListDocument; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + + +public class ElasticsearchMappingsTests extends ESTestCase { + private void parseJson(JsonParser parser, Set expected) throws IOException { + try { + JsonToken token = parser.nextToken(); + while (token != null && token != JsonToken.END_OBJECT) { + switch (token) { + case START_OBJECT: + parseJson(parser, expected); + break; + case FIELD_NAME: + String fieldName = parser.getCurrentName(); + expected.add(fieldName); + break; + default: + break; + } + token = parser.nextToken(); + } + } catch (JsonParseException e) { + fail("Cannot parse JSON: " + e); + } + } + + public void testReservedFields() + throws IOException, ClassNotFoundException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + Set overridden = new HashSet<>(); + + // These are not reserved because they're Elasticsearch keywords, not + // field names + overridden.add(ElasticsearchMappings.ALL); + overridden.add(ElasticsearchMappings.ANALYZER); + overridden.add(ElasticsearchMappings.COPY_TO); + overridden.add(ElasticsearchMappings.DYNAMIC); + overridden.add(ElasticsearchMappings.ENABLED); + overridden.add(ElasticsearchMappings.INCLUDE_IN_ALL); + overridden.add(ElasticsearchMappings.INDEX); + overridden.add(ElasticsearchMappings.NESTED); + overridden.add(ElasticsearchMappings.NO); + overridden.add(ElasticsearchMappings.PARENT); + overridden.add(ElasticsearchMappings.PROPERTIES); + overridden.add(ElasticsearchMappings.TYPE); + overridden.add(ElasticsearchMappings.WHITESPACE); + + // These are not reserved because they're data types, not field names + overridden.add(AnomalyRecord.TYPE.getPreferredName()); + overridden.add(AuditActivity.TYPE.getPreferredName()); + overridden.add(AuditMessage.TYPE.getPreferredName()); + overridden.add(Bucket.TYPE.getPreferredName()); + overridden.add(DataCounts.TYPE.getPreferredName()); + overridden.add(ReservedFieldNames.BUCKET_PROCESSING_TIME_TYPE); + overridden.add(BucketInfluencer.TYPE.getPreferredName()); + overridden.add(CategorizerState.TYPE); + overridden.add(CategoryDefinition.TYPE.getPreferredName()); + overridden.add(Influencer.TYPE.getPreferredName()); + overridden.add(Job.TYPE); + overridden.add(ListDocument.TYPE.getPreferredName()); + overridden.add(ModelDebugOutput.TYPE.getPreferredName()); + overridden.add(ModelState.TYPE); + overridden.add(ModelSnapshot.TYPE.getPreferredName()); + overridden.add(ModelSizeStats.TYPE.getPreferredName()); + overridden.add(Quantiles.TYPE.getPreferredName()); + overridden.add(Usage.TYPE); + + // These are not reserved because they're in the prelert-int index, not + // prelertresults-* + overridden.add(ListDocument.ID.getPreferredName()); + overridden.add(ListDocument.ITEMS.getPreferredName()); + + // These are not reserved because they're analyzed strings, i.e. the + // same type as user-specified fields + overridden.add(Job.DESCRIPTION.getPreferredName()); + overridden.add(Allocation.STATUS.getPreferredName()); + overridden.add(ModelSnapshot.DESCRIPTION.getPreferredName()); + overridden.add(SchedulerConfig.USERNAME.getPreferredName()); + + Set expected = new HashSet<>(); + + XContentBuilder builder = ElasticsearchMappings.auditActivityMapping(); + BufferedInputStream inputStream = new BufferedInputStream( + new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + JsonParser parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.auditMessageMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.bucketInfluencerMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.bucketMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.bucketPartitionMaxNormalizedScores(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.categorizerStateMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.categoryDefinitionMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.dataCountsMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.influencerMapping(null); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.modelDebugOutputMapping(null); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.modelSizeStatsMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.modelSnapshotMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.modelStateMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.processingTimeMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.quantilesMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.recordMapping(null); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.usageMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + expected.removeAll(overridden); + for (String s : expected) { + // By comparing like this the failure messages say which string is + // missing + String reserved = ReservedFieldNames.RESERVED_FIELD_NAMES.contains(s) ? s : null; + assertEquals(s, reserved); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersisterTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersisterTests.java new file mode 100644 index 00000000000..f67f8e8e641 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchPersisterTests.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.test.ESTestCase; +import org.mockito.ArgumentCaptor; + +import org.elasticsearch.xpack.prelert.job.results.AnomalyRecord; +import org.elasticsearch.xpack.prelert.job.results.Bucket; +import org.elasticsearch.xpack.prelert.job.results.BucketInfluencer; +import org.elasticsearch.xpack.prelert.job.results.Influencer; + + +public class ElasticsearchPersisterTests extends ESTestCase { + + private static final String CLUSTER_NAME = "myCluster"; + private static final String JOB_ID = "testJobId"; + + public void testPersistBucket_NoRecords() { + Client client = mock(Client.class); + Bucket bucket = mock(Bucket.class); + when(bucket.getRecords()).thenReturn(null); + ElasticsearchPersister persister = new ElasticsearchPersister(JOB_ID, client); + persister.persistBucket(bucket); + verifyNoMoreInteractions(client); + } + + public void testPersistBucket_OneRecord() throws IOException { + ArgumentCaptor captor = ArgumentCaptor.forClass(XContentBuilder.class); + BulkResponse response = mock(BulkResponse.class); + String responseId = "abcXZY54321"; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME) + .prepareIndex("prelertresults-" + JOB_ID, Bucket.TYPE.getPreferredName(), responseId, captor) + .prepareIndex("prelertresults-" + JOB_ID, AnomalyRecord.TYPE.getPreferredName(), "", captor) + .prepareIndex("prelertresults-" + JOB_ID, BucketInfluencer.TYPE.getPreferredName(), "", captor) + .prepareIndex("prelertresults-" + JOB_ID, Influencer.TYPE.getPreferredName(), "", captor) + .prepareBulk(response); + + Client client = clientBuilder.build(); + Bucket bucket = getBucket(1, 0); + bucket.setId(responseId); + bucket.setAnomalyScore(99.9); + bucket.setBucketSpan(123456); + bucket.setEventCount(57); + bucket.setInitialAnomalyScore(88.8); + bucket.setMaxNormalizedProbability(42.0); + bucket.setProcessingTimeMs(8888); + bucket.setRecordCount(1); + + BucketInfluencer bi = new BucketInfluencer("foo"); + bi.setAnomalyScore(14.15); + bi.setInfluencerFieldName("biOne"); + bi.setInitialAnomalyScore(18.12); + bi.setProbability(0.0054); + bi.setRawAnomalyScore(19.19); + bucket.addBucketInfluencer(bi); + + Influencer inf = new Influencer("jobname", "infName1", "infValue1"); + inf.setAnomalyScore(16); + inf.setId("infID"); + inf.setInitialAnomalyScore(55.5); + inf.setProbability(0.4); + inf.setTimestamp(bucket.getTimestamp()); + bucket.setInfluencers(Arrays.asList(inf)); + + AnomalyRecord record = bucket.getRecords().get(0); + List actuals = new ArrayList<>(); + actuals.add(5.0); + actuals.add(5.1); + record.setActual(actuals); + record.setAnomalyScore(99.8); + record.setBucketSpan(42); + record.setByFieldName("byName"); + record.setByFieldValue("byValue"); + record.setCorrelatedByFieldValue("testCorrelations"); + record.setDetectorIndex(3); + record.setFieldName("testFieldName"); + record.setFunction("testFunction"); + record.setFunctionDescription("testDescription"); + record.setInitialNormalizedProbability(23.4); + record.setNormalizedProbability(0.005); + record.setOverFieldName("overName"); + record.setOverFieldValue("overValue"); + record.setPartitionFieldName("partName"); + record.setPartitionFieldValue("partValue"); + record.setProbability(0.1); + List typicals = new ArrayList<>(); + typicals.add(0.44); + typicals.add(998765.3); + record.setTypical(typicals); + + ElasticsearchPersister persister = new ElasticsearchPersister(JOB_ID, client); + persister.persistBucket(bucket); + List list = captor.getAllValues(); + assertEquals(4, list.size()); + + String s = list.get(0).string(); + assertTrue(s.matches(".*anomalyScore.:99\\.9.*")); + assertTrue(s.matches(".*initialAnomalyScore.:88\\.8.*")); + assertTrue(s.matches(".*maxNormalizedProbability.:42\\.0.*")); + assertTrue(s.matches(".*recordCount.:1.*")); + assertTrue(s.matches(".*eventCount.:57.*")); + assertTrue(s.matches(".*bucketSpan.:123456.*")); + assertTrue(s.matches(".*processingTimeMs.:8888.*")); + + s = list.get(1).string(); + assertTrue(s.matches(".*probability.:0\\.0054.*")); + assertTrue(s.matches(".*influencerFieldName.:.biOne.*")); + assertTrue(s.matches(".*initialAnomalyScore.:18\\.12.*")); + assertTrue(s.matches(".*anomalyScore.:14\\.15.*")); + assertTrue(s.matches(".*rawAnomalyScore.:19\\.19.*")); + + s = list.get(2).string(); + assertTrue(s.matches(".*probability.:0\\.4.*")); + assertTrue(s.matches(".*influencerFieldName.:.infName1.*")); + assertTrue(s.matches(".*influencerFieldValue.:.infValue1.*")); + assertTrue(s.matches(".*initialAnomalyScore.:55\\.5.*")); + assertTrue(s.matches(".*anomalyScore.:16\\.0.*")); + + s = list.get(3).string(); + assertTrue(s.matches(".*detectorIndex.:3.*")); + assertTrue(s.matches(".*\"probability\":0\\.1.*")); + assertTrue(s.matches(".*\"anomalyScore\":99\\.8.*")); + assertTrue(s.matches(".*\"normalizedProbability\":0\\.005.*")); + assertTrue(s.matches(".*initialNormalizedProbability.:23.4.*")); + assertTrue(s.matches(".*bucketSpan.:42.*")); + assertTrue(s.matches(".*byFieldName.:.byName.*")); + assertTrue(s.matches(".*byFieldValue.:.byValue.*")); + assertTrue(s.matches(".*correlatedByFieldValue.:.testCorrelations.*")); + assertTrue(s.matches(".*typical.:.0\\.44,998765\\.3.*")); + assertTrue(s.matches(".*actual.:.5\\.0,5\\.1.*")); + assertTrue(s.matches(".*fieldName.:.testFieldName.*")); + assertTrue(s.matches(".*function.:.testFunction.*")); + assertTrue(s.matches(".*functionDescription.:.testDescription.*")); + assertTrue(s.matches(".*partitionFieldName.:.partName.*")); + assertTrue(s.matches(".*partitionFieldValue.:.partValue.*")); + assertTrue(s.matches(".*overFieldName.:.overName.*")); + assertTrue(s.matches(".*overFieldValue.:.overValue.*")); + } + + private Bucket getBucket(int numRecords, int numInfluencers) { + Bucket b = new Bucket("foo"); + b.setId("1"); + b.setTimestamp(new Date()); + List records = new ArrayList<>(); + for (int i = 0; i < numRecords; ++i) { + AnomalyRecord r = new AnomalyRecord("foo"); + records.add(r); + } + b.setRecords(records); + return b; + } + + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScriptsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScriptsTests.java new file mode 100644 index 00000000000..aeef35b7d71 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/prelert/job/persistence/ElasticsearchScriptsTests.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.prelert.job.persistence; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.MockitoAnnotations; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.hamcrest.Matchers.containsString; + +public class ElasticsearchScriptsTests extends ESTestCase { + @Captor + private ArgumentCaptor> mapCaptor; + + @Before + public void setUpMocks() throws InterruptedException, ExecutionException { + MockitoAnnotations.initMocks(this); + } + + public void testNewUpdateBucketCount() { + Script script = ElasticsearchScripts.newUpdateBucketCount(42L); + assertEquals("ctx._source.counts.bucketCount += params.count", script.getIdOrCode()); + assertEquals(1, script.getParams().size()); + assertEquals(42L, script.getParams().get("count")); + } + + public void testNewUpdateUsage() { + Script script = ElasticsearchScripts.newUpdateUsage(1L, 2L, 3L); + assertEquals( + "ctx._source.inputBytes += params.bytes;ctx._source.inputFieldCount += params.fieldCount;ctx._source.inputRecordCount" + + " += params.recordCount;", + script.getIdOrCode()); + assertEquals(3, script.getParams().size()); + assertEquals(1L, script.getParams().get("bytes")); + assertEquals(2L, script.getParams().get("fieldCount")); + assertEquals(3L, script.getParams().get("recordCount")); + } + + public void testUpdateProcessingTime() { + Long time = 135790L; + Script script = ElasticsearchScripts.updateProcessingTime(time); + assertEquals("ctx._source.averageProcessingTimeMs = ctx._source.averageProcessingTimeMs * 0.9 + params.timeMs * 0.1", + script.getIdOrCode()); + assertEquals(time, script.getParams().get("timeMs")); + } + + public void testUpdateUpsertViaScript() { + String index = "idx"; + String docId = "docId"; + String type = "type"; + Map map = new HashMap<>(); + map.put("testKey", "testValue"); + + Script script = new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test-script-here", map); + ArgumentCaptor