From 90e98766ac717ecbce59e1d9cb872026361240be Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 16 Jul 2015 13:59:09 -0700 Subject: [PATCH 01/32] Adding shard_allocation vis Original commit: elastic/x-pack-elasticsearch@081ba8aa5d65df98d5d3a3d20bcbb143fb8afc5e --- .jscsrc | 58 +++++++++++++++++++++++++++++++++++++++++++++++ .jshintrc | 27 ++++++++++++++++++++++ .jshintrc.browser | 14 ++++++++++++ .jshintrc.node | 9 ++++++++ 4 files changed, 108 insertions(+) create mode 100644 .jscsrc create mode 100644 .jshintrc create mode 100644 .jshintrc.browser create mode 100644 .jshintrc.node diff --git a/.jscsrc b/.jscsrc new file mode 100644 index 00000000000..285ada95c3e --- /dev/null +++ b/.jscsrc @@ -0,0 +1,58 @@ +{ + "maximumLineLength": { + "value": 140, + "allowComments": true + }, + "requireCurlyBraces": [ + "for", + "do", + "try", + "catch" + ], + "requireSpaceAfterKeywords": [ + "if", + "else", + "for", + "while", + "do", + "switch", + "case", + "return", + "try", + "catch", + "function", + "typeof" + ], + "requireSpaceBeforeBlockStatements": true, + "requireParenthesesAroundIIFE": true, + "requireSpacesInConditionalExpression": true, + "disallowSpacesInNamedFunctionExpression": { + "beforeOpeningRoundBrace": true + }, + "disallowSpacesInFunctionDeclaration": { + "beforeOpeningRoundBrace": true + }, + "requireSpaceBetweenArguments": true, + "disallowMultipleVarDecl": "exceptUndefined", + "disallowEmptyBlocks": true, + "requireCommaBeforeLineBreak": true, + "disallowSpaceAfterPrefixUnaryOperators": true, + "disallowSpaceBeforePostfixUnaryOperators": true, + "disallowSpaceBeforeBinaryOperators": [ + "," + ], + "requireSpacesInForStatement": true, + "requireSpaceBeforeBinaryOperators": true, + "requireSpaceAfterBinaryOperators": true, + "disallowKeywords": [ + "with" + ], + "validateIndentation": 2, + "validateQuoteMarks": { "mark": "'", "escape": true }, + "disallowMixedSpacesAndTabs": true, + "disallowTrailingWhitespace": true, + "requireCapitalizedConstructors": true, + "requireDotNotation": true, + "disallowNewlineBeforeBlockStatements": true, + "disallowMultipleLineStrings": true +} diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 00000000000..8e5b2febb1b --- /dev/null +++ b/.jshintrc @@ -0,0 +1,27 @@ +{ + // for files at project root + "node": true, + + // shared with .jshintrc files for browser and node + "unused": false, + "bitwise": false, + "eqnull": true, + "eqeqeq": true, + "forin": true, + "immed": true, + "expr": true, + "latedef": "nofunc", + "noarg": true, + "noempty": true, + "undef": true, + "quotmark": "single", + "plusplus": false, + "boss": true, + "laxbreak": true, + "laxcomma": true, + "validthis": true, + "sub": true, + "-W084": true, + "scripturl": true, + "evil": true +} diff --git a/.jshintrc.browser b/.jshintrc.browser new file mode 100644 index 00000000000..09cdd36321a --- /dev/null +++ b/.jshintrc.browser @@ -0,0 +1,14 @@ +{ + "extends": "./.jshintrc", + "node": false, + "browser": true, + + "predef": { + // require.js + "define": true, + "require": true, + "console": false, + "-event": true, + "-name": true + } +} diff --git a/.jshintrc.node b/.jshintrc.node new file mode 100644 index 00000000000..a77f81af35e --- /dev/null +++ b/.jshintrc.node @@ -0,0 +1,9 @@ +{ + "extends": "./.jshintrc", + "node": true, + + "globals": { + "Promise": true, + "status": true + } +} From 8bba75c275334a424974ce381be2fcf784797614 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 17 Jul 2015 14:40:48 -0700 Subject: [PATCH 02/32] tesing build Original commit: elastic/x-pack-elasticsearch@7697fc9dffc661a0e7146769c2b813a25f1de5aa --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 884a5b6516b..8f749d1d0ce 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# WARNING THIS IS ALL IN FLUX + # Installing the last release of Marvel The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): From 4dd92cf2e8985d616f575b599dda1f29199f3f5e Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 17 Jul 2015 16:32:40 -0700 Subject: [PATCH 03/32] Moving the source to app; setup jsx parsing Original commit: elastic/x-pack-elasticsearch@46cb23d58382a170764c86b95f3bf89a125d22af --- .jscsrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.jscsrc b/.jscsrc index 285ada95c3e..d8ea99cdc6c 100644 --- a/.jscsrc +++ b/.jscsrc @@ -54,5 +54,6 @@ "requireCapitalizedConstructors": true, "requireDotNotation": true, "disallowNewlineBeforeBlockStatements": true, - "disallowMultipleLineStrings": true + "disallowMultipleLineStrings": true, + "esprima" : "esprima-fb" } From e71843ecb8e66b3f568b7531ee00a9a54ac917f1 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 20 Jul 2015 16:30:20 -0700 Subject: [PATCH 04/32] Fixing gulpfile Original commit: elastic/x-pack-elasticsearch@cda37f6d85edfe9ebd429209f6907ebddca0e384 --- .agignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .agignore diff --git a/.agignore b/.agignore new file mode 100644 index 00000000000..0847b65d054 --- /dev/null +++ b/.agignore @@ -0,0 +1,4 @@ +public +agent +node_modules +bower_components From 953597e1c084da157b14289dcbc4552bea4e83fd Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 27 Jul 2015 17:45:09 -0700 Subject: [PATCH 05/32] Triggering an update Original commit: elastic/x-pack-elasticsearch@3b03b5ec1cb5e436a7e868713a8c34bc5f0a9100 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8f749d1d0ce..840ffd4ea8b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # WARNING THIS IS ALL IN FLUX + # Installing the last release of Marvel The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): From a779ac8100caaf5858652e3deff4977229656ff9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 30 Jul 2015 12:49:04 -0700 Subject: [PATCH 06/32] Adding indexer and optimizing shard allocations Original commit: elastic/x-pack-elasticsearch@146fbec5750e7d35356ce97eb2474f143cea6fee --- marvel_index_template.json | 129 +++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 marvel_index_template.json diff --git a/marvel_index_template.json b/marvel_index_template.json new file mode 100644 index 00000000000..2834e940481 --- /dev/null +++ b/marvel_index_template.json @@ -0,0 +1,129 @@ +{ + "template": ".marvel*", + "settings": { + "marvel.index_format": 7, + "marvel.version": "2.0.0", + "index.number_of_shards": 1, + "index.number_of_replicas": 1, + "index.codec": "best_compression", + "index.mapper.dynamic": false + }, + "mappings": { + "_default_": { + "_all": { + "enabled": false + }, + "date_detection": false, + "properties": { + "cluster_name": { + "type": "string", + "index": "not_analyzed" + }, + "timestamp": { + "type": "date", + "format": "date_time" + } + } + }, + "marvel_index_stats": { + "properties": { + "index_stats": { + "properties": { + "index": { + "type": "string", + "index": "not_analyzed" + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "throttle_time_in_millis": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "marvel_cluster_stats": { + "properties": { + "cluster_stats": { + "properties": { + "nodes": { + "type": "object" + }, + "indices": { + "type": "object" + } + } + } + } + }, + "marvel_cluster_state": { + "properties": { + "cluster_state": { + "properties": { + "version": { "type": "long" }, + "nodes": { "type": "object" }, + "master_node": { "type": "string", "index": "not_analyzed" }, + "shards": { "type": "object" } + } + } + } + }, + "marvel_node_stats": { + "properties": { + "node_id": { + "type": "string", + "index": "not_analyzed" + }, + "node_master": { + "type": "boolean" + }, + "mlockall": { + "type": "boolean" + }, + "disk_threshold_enabled": { + "type": "boolean" + }, + "disk_threshold_watermark_high": { + "type": "short" + }, + "node_stats": { + "type": "object" + } + } + }, + "marvel_index_recovery": { + "properties": { + "index_name": { + "type": "string", + "index": "not_analyzed" + }, + "shards": { + "type": "object" + } + } + } + } +} From 86ebd14587d0539818d31eb9e81d12fecacccc9d Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 4 Aug 2015 10:41:44 -0700 Subject: [PATCH 07/32] Movign to the new Kibana Original commit: elastic/x-pack-elasticsearch@a943185e94c8cfac5de6100ec140683115b7be32 --- .bowerrc | 4 ---- watcher/lib/inject_stats.js | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 .bowerrc diff --git a/.bowerrc b/.bowerrc deleted file mode 100644 index 1d69e778685..00000000000 --- a/.bowerrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "directory": "./public/bower_components" -} - diff --git a/watcher/lib/inject_stats.js b/watcher/lib/inject_stats.js index e14ec726915..c81e85c7270 100644 --- a/watcher/lib/inject_stats.js +++ b/watcher/lib/inject_stats.js @@ -1,5 +1,4 @@ var _ = require('lodash'); -_.mixin(require('lodash-deep')); var moment = require('moment'); var client = require('./client'); @@ -21,7 +20,7 @@ module.exports = function (fixture) { var index = workingDate.format(indexPattern); var entry = { '@timestamp': workingDate.toISOString() }; row.forEach(function (val, index) { - _.deepSet(entry, fields[index], val); + _.set(entry, fields[index], val); }); indices.push(index); @@ -40,7 +39,7 @@ module.exports = function (fixture) { var index = moment.utc(row[dateField]).format(indexPattern); var entry = {}; _.each(row, function (val, key) { - _.deepSet(entry, key, val); + _.set(entry, key, val); }); indices.push(index); body.push({ From c21ca69acd81b6454d7a1b1b563f7155f14e49f9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 6 Aug 2015 15:30:05 -0700 Subject: [PATCH 08/32] Moving everythign to the new data format Original commit: elastic/x-pack-elasticsearch@b35c192536bc793d79083a30fde7ae8f30e29b6a --- .agignore | 1 - watcher/lib/inject_stats.js | 2 +- watcher/lib/test_no_execute.js | 2 +- watcher/test/cluster_status.js | 20 ++++++++++---------- watcher/watches/cluster_status.json | 8 ++++---- watcher/watches/cpu_usage.json | 4 ++-- watcher/watches/fielddata.json | 4 ++-- watcher/watches/file_descriptors.json | 4 ++-- watcher/watches/heap_used.json | 4 ++-- 9 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.agignore b/.agignore index 0847b65d054..10fb4038cbd 100644 --- a/.agignore +++ b/.agignore @@ -1,4 +1,3 @@ -public agent node_modules bower_components diff --git a/watcher/lib/inject_stats.js b/watcher/lib/inject_stats.js index c81e85c7270..9a2b1cb6b96 100644 --- a/watcher/lib/inject_stats.js +++ b/watcher/lib/inject_stats.js @@ -18,7 +18,7 @@ module.exports = function (fixture) { function createEntries(row) { var index = workingDate.format(indexPattern); - var entry = { '@timestamp': workingDate.toISOString() }; + var entry = { 'timestamp': workingDate.toISOString() }; row.forEach(function (val, index) { _.set(entry, fields[index], val); }); diff --git a/watcher/lib/test_no_execute.js b/watcher/lib/test_no_execute.js index 7c94d531487..c90fe915a67 100644 --- a/watcher/lib/test_no_execute.js +++ b/watcher/lib/test_no_execute.js @@ -10,7 +10,7 @@ module.exports = function testNoExecute(options, message, generateRawData) { var fixture = { indexPattern: options.indexPattern, type: options.type, - dateField: '@timestamp', + dateField: 'timestamp', rawData: rawData }; return executeWatcher(options.watcher, fixture).then(function (resp) { diff --git a/watcher/test/cluster_status.js b/watcher/test/cluster_status.js index 91eeb774c92..17e97701144 100644 --- a/watcher/test/cluster_status.js +++ b/watcher/test/cluster_status.js @@ -22,12 +22,12 @@ describe('Marvel Watchers', function () { this.timeout(5000); var workingDate = moment.utc(); var rawData = _.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); var fixture = { indexPattern: '[.marvel-]YYYY.MM.DD', type: 'cluster_stats', - dateField: '@timestamp', + dateField: 'timestamp', rawData: rawData }; return executeWatcher('cluster_status', fixture).then(function (resp) { @@ -54,10 +54,10 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 55 then Yellow for 60 seconds', function () { var workingDate = moment.utc(); var rawData = _.times(11, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); return rawData; }); @@ -65,10 +65,10 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 30 then Yellow for 60 seconds', function () { var workingDate = moment.utc(); var rawData = _.times(6, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); return rawData; }); @@ -76,16 +76,16 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 5 Yellow for 10 Red for 10 Green for 60', function () { var workingDate = moment.utc(); var rawData = _.times(1, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(2, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); rawData.concat(_.times(2, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; })); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'green' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'green' }; })); return rawData; }); diff --git a/watcher/watches/cluster_status.json b/watcher/watches/cluster_status.json index 52556996e73..74ec231e69a 100644 --- a/watcher/watches/cluster_status.json +++ b/watcher/watches/cluster_status.json @@ -17,7 +17,7 @@ "must": [ { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -45,10 +45,10 @@ } } }, - "fields": ["@timestamp","status"], + "fields": ["timestamp","status"], "sort": [ { - "@timestamp": { + "timestamp": { "order": "desc" } } @@ -57,7 +57,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/cpu_usage.json b/watcher/watches/cpu_usage.json index edda8624f64..f2a7422f948 100644 --- a/watcher/watches/cpu_usage.json +++ b/watcher/watches/cpu_usage.json @@ -16,7 +16,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -27,7 +27,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "minute" }, "aggs": { diff --git a/watcher/watches/fielddata.json b/watcher/watches/fielddata.json index 08072f35ebf..854be35cbfc 100644 --- a/watcher/watches/fielddata.json +++ b/watcher/watches/fielddata.json @@ -21,7 +21,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-1m", "lte": "now" } @@ -32,7 +32,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/file_descriptors.json b/watcher/watches/file_descriptors.json index 83fbe122aca..bfd5fb48ccc 100644 --- a/watcher/watches/file_descriptors.json +++ b/watcher/watches/file_descriptors.json @@ -21,7 +21,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-1m", "lte": "now" } @@ -32,7 +32,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/heap_used.json b/watcher/watches/heap_used.json index 3f33719a9e8..ba92437475b 100644 --- a/watcher/watches/heap_used.json +++ b/watcher/watches/heap_used.json @@ -16,7 +16,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -27,7 +27,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "minute" }, "aggs": { From b9f0574b0d5fcc5a450eea7f164bca57c3fc8d22 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 10 Aug 2015 12:50:09 -0700 Subject: [PATCH 09/32] adding license doc to sample data Original commit: elastic/x-pack-elasticsearch@c3232fed072dd1408e861192b4254d2b86990588 --- marvel_index_template.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/marvel_index_template.json b/marvel_index_template.json index 2834e940481..84b54dbbfa3 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -25,6 +25,13 @@ } } }, + "marvel_licenses": { + "properties": { + "licenses": { + "type": "object" + } + } + }, "marvel_index_stats": { "properties": { "index_stats": { From 0eb0e701e4f415b7e38eed081e6eca6d0eda91a9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 17 Aug 2015 11:20:32 -0700 Subject: [PATCH 10/32] Adding cluster home screen; cleaning up cluster_state; removing cluster_state service Original commit: elastic/x-pack-elasticsearch@0dc010a7b7901e8e077004f6f99dae93f22d429d --- .eslintrc | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .eslintrc diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 00000000000..136ac914a87 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,69 @@ +--- +parser: babel-eslint + +env: + es6: true + amd: true + node: true + mocha: true + browser: true + + +rules: + block-scoped-var: 2 + camelcase: [ 2, { properties: never } ] + comma-dangle: 0 + comma-style: [ 2, last ] + consistent-return: 0 + curly: [ 2, multi-line ] + dot-location: [ 2, property ] + dot-notation: [ 2, { allowKeywords: true } ] + eqeqeq: [ 2, allow-null ] + guard-for-in: 2 + indent: [ 2, 2, { SwitchCase: 1 } ] + key-spacing: [ 0, { align: value } ] + max-len: [ 2, 140, 2, { ignoreComments: true, ignoreUrls: true } ] + new-cap: [ 2, { capIsNewExceptions: [ Private ] } ] + no-bitwise: 0 + no-caller: 2 + no-cond-assign: 0 + no-debugger: 2 + no-empty: 2 + no-eval: 2 + no-extend-native: 2 + no-extra-parens: 0 + no-irregular-whitespace: 2 + no-iterator: 2 + no-loop-func: 2 + no-multi-spaces: 0 + no-multi-str: 2 + no-nested-ternary: 2 + no-new: 0 + no-path-concat: 0 + no-proto: 2 + no-return-assign: 0 + no-script-url: 2 + no-sequences: 2 + no-shadow: 0 + no-trailing-spaces: 2 + no-undef: 2 + no-underscore-dangle: 0 + no-unused-expressions: 0 + no-unused-vars: 0 + no-use-before-define: [ 2, nofunc ] + no-with: 2 + one-var: [ 2, never ] + quotes: [ 2, single ] + semi-spacing: [ 2, { before: false, after: true } ] + semi: [ 2, always ] + space-after-keywords: [ 2, always ] + space-before-blocks: [ 2, always ] + space-before-function-paren: [ 2, { anonymous: always, named: never } ] + space-in-parens: [ 2, never ] + space-infix-ops: [ 2, { int32Hint: false } ] + space-return-throw-case: [ 2 ] + space-unary-ops: [ 2 ] + strict: [ 2, never ] + valid-typeof: 2 + wrap-iife: [ 2, outside ] + yoda: 0 From e8f13f50b8300cf824081898c3aacafa1c9c7fe2 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 18 Aug 2015 12:33:17 -0700 Subject: [PATCH 11/32] Changing cluster_name to cluster_uuid Original commit: elastic/x-pack-elasticsearch@7b5227a0a7c1c8cff3313cc3801e7238d59f4249 --- marvel_index_template.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index 84b54dbbfa3..b9d1f9e7f2c 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -15,7 +15,11 @@ }, "date_detection": false, "properties": { - "cluster_name": { + "cluster_uuid": { + "type": "string", + "index": "not_analyzed" + }, + "cluster_uuid": { "type": "string", "index": "not_analyzed" }, From e7761d75ea751826b1e14faecf27f23b92b130c4 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 24 Aug 2015 16:48:57 -0700 Subject: [PATCH 12/32] Fixing schema Original commit: elastic/x-pack-elasticsearch@844479ccd3b36778f54bc24ee8e0edb34b90037a --- marvel_index_template.json | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index b9d1f9e7f2c..a889ebbbc35 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -104,24 +104,16 @@ }, "marvel_node_stats": { "properties": { - "node_id": { - "type": "string", - "index": "not_analyzed" - }, - "node_master": { - "type": "boolean" - }, - "mlockall": { - "type": "boolean" - }, - "disk_threshold_enabled": { - "type": "boolean" - }, - "disk_threshold_watermark_high": { - "type": "short" - }, "node_stats": { - "type": "object" + "properties": { + "node_id": { + "type": "string", + "index": "not_analyzed" + }, + "node_master": { + "type": "boolean" + } + } } } }, From d4836ed7a11787778783d46d468fb9db1022dce2 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 26 Aug 2015 15:25:28 -0700 Subject: [PATCH 13/32] Adding phone home details Original commit: elastic/x-pack-elasticsearch@23a305c196096ac2e8f552137dae003dab7ac989 --- marvel_index_template.json | 117 ++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 20 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index a889ebbbc35..933610879e1 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -1,8 +1,9 @@ + { "template": ".marvel*", "settings": { "marvel.index_format": 7, - "marvel.version": "2.0.0", + "marvel.version": "${project.version}", "index.number_of_shards": 1, "index.number_of_replicas": 1, "index.codec": "best_compression", @@ -15,10 +16,6 @@ }, "date_detection": false, "properties": { - "cluster_uuid": { - "type": "string", - "index": "not_analyzed" - }, "cluster_uuid": { "type": "string", "index": "not_analyzed" @@ -29,13 +26,6 @@ } } }, - "marvel_licenses": { - "properties": { - "licenses": { - "type": "object" - } - } - }, "marvel_index_stats": { "properties": { "index_stats": { @@ -44,6 +34,17 @@ "type": "string", "index": "not_analyzed" }, + "primaries": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, "total": { "properties": { "docs": { @@ -65,10 +66,47 @@ }, "indexing": { "properties": { + "index_total": { + "type": "long" + }, + "index_time_in_millis": { + "type": "long" + }, "throttle_time_in_millis": { "type": "long" } } + }, + "merges": { + "properties": { + "total_size_in_bytes": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_total": { + "type": "long" + }, + "query_time_in_millis": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "memory_in_bytes": { + "type": "long" + } + } + }, + "refresh": { + "properties": { + "total_time_in_millis": { + "type": "long" + } + } } } } @@ -94,14 +132,33 @@ "properties": { "cluster_state": { "properties": { - "version": { "type": "long" }, - "nodes": { "type": "object" }, - "master_node": { "type": "string", "index": "not_analyzed" }, - "shards": { "type": "object" } + "version": { + "type": "long" + }, + "master_node": { + "type": "string", + "index": "not_analyzed" + }, + "status": { + "type": "string", + "index": "not_analyzed" + }, + "nodes": { + "type": "object" + }, + "shards": { + "type": "object" + } } } } }, + "cluster_info": { + "enabled": false + }, + "phone_home": { + "enabled": false + }, "marvel_node_stats": { "properties": { "node_stats": { @@ -112,6 +169,30 @@ }, "node_master": { "type": "boolean" + }, + "mlockall": { + "type": "boolean" + }, + "disk_threshold_enabled": { + "type": "boolean" + }, + "disk_threshold_watermark_high": { + "type": "short" + }, + "indices": { + "type": "object" + }, + "fs": { + "type": "object" + }, + "process": { + "type": "object" + }, + "jvm": { + "type": "object" + }, + "thread_pool": { + "type": "object" } } } @@ -119,10 +200,6 @@ }, "marvel_index_recovery": { "properties": { - "index_name": { - "type": "string", - "index": "not_analyzed" - }, "shards": { "type": "object" } From 2b7c366939aa00495fc3ec6c11b4acc250d148d9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Fri, 28 Aug 2015 14:39:50 -0700 Subject: [PATCH 14/32] Adding faker Original commit: elastic/x-pack-elasticsearch@b800772adfcca6460d0d96d6dfe0f45f4fae9375 --- marvel_index_template.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/marvel_index_template.json b/marvel_index_template.json index 933610879e1..6fb62b8e990 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -198,6 +198,13 @@ } } }, + "marvel_indices_stats": { + "properties": { + "indices_stats": { + "type": "object" + } + } + }, "marvel_index_recovery": { "properties": { "shards": { From e95956bf22b769dfdf946ff4ae70876c2e06bcbd Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 1 Sep 2015 09:44:17 -0700 Subject: [PATCH 15/32] Adding shards and setting defaults for time Original commit: elastic/x-pack-elasticsearch@a7b9241b35ba1a6c24f6892c6b740604ec665486 --- marvel_index_template.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/marvel_index_template.json b/marvel_index_template.json index 6fb62b8e990..4167b0ba111 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -205,6 +205,13 @@ } } }, + "marvel_shards": { + "properties": { + "shard": { + "type": "object" + } + } + }, "marvel_index_recovery": { "properties": { "shards": { From c5f7ff429b05efb3f61dc646eb4d401cc57616fc Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 3 Sep 2015 13:59:51 -0700 Subject: [PATCH 16/32] Moving shards to marvel_shards documents - Closes elastic/elasticsearch#498 - Remove redundant column on shard visualization - Fix colspan on index view with unassigned shards - Add /marvel/api/v1/clusters/{cluster_uuid}/state/{state_uuid}/shards - Change shard vis to use id and type to request data from new shards api - Change indexer to use new shard schema Original commit: elastic/x-pack-elasticsearch@416e1c7fd4bde1a8aab466d816839fad71eb88e9 --- marvel_index_template.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index 4167b0ba111..cfd3f3021a6 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -207,8 +207,19 @@ }, "marvel_shards": { "properties": { + "state_uuid": { + "type": "string", + "index": "not_analyzed" + }, "shard": { - "type": "object" + "properties": { + "state": { "type": "string", "index": "not_analyzed" }, + "primary": { "type": "boolean" }, + "index": { "type": "string", "index": "not_analyzed" }, + "relocating_node": { "type": "string", "index": "not_analyzed" }, + "shard": { "type": "long" }, + "node": { "type": "string", "index": "not_analyzed" } + } } } }, From c1d05b8ee588c943fa01a7e752c1efa7ab82d56b Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Wed, 16 Sep 2015 08:42:03 -0700 Subject: [PATCH 17/32] Changing cluster state and fixing the indexPattern for Kibana Original commit: elastic/x-pack-elasticsearch@9838dd5bd7ad0811af0e428ce5acca929145efca --- marvel_index_template.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index cfd3f3021a6..4f668024d0f 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -144,10 +144,7 @@ "index": "not_analyzed" }, "nodes": { - "type": "object" - }, - "shards": { - "type": "object" + "enabled": false } } } From d28e337689ab2665889e52700cc96134b314e7b5 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 22 Sep 2015 10:48:29 -0700 Subject: [PATCH 18/32] Closes elastic/elasticsearch#534 - Remove marvel_ prefix for all doc types Original commit: elastic/x-pack-elasticsearch@bdea3f915fc0c4406d0b3ea84cc49163f94f75be --- marvel_index_template.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index 4f668024d0f..7237838ac83 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -26,7 +26,7 @@ } } }, - "marvel_index_stats": { + "index_stats": { "properties": { "index_stats": { "properties": { @@ -114,7 +114,7 @@ } } }, - "marvel_cluster_stats": { + "cluster_stats": { "properties": { "cluster_stats": { "properties": { @@ -128,7 +128,7 @@ } } }, - "marvel_cluster_state": { + "cluster_state": { "properties": { "cluster_state": { "properties": { @@ -156,7 +156,7 @@ "phone_home": { "enabled": false }, - "marvel_node_stats": { + "node_stats": { "properties": { "node_stats": { "properties": { @@ -195,14 +195,14 @@ } } }, - "marvel_indices_stats": { + "indices_stats": { "properties": { "indices_stats": { "type": "object" } } }, - "marvel_shards": { + "shards": { "properties": { "state_uuid": { "type": "string", @@ -220,7 +220,7 @@ } } }, - "marvel_index_recovery": { + "index_recovery": { "properties": { "shards": { "type": "object" From 0feb382737102a930b1c25c91056b4885fe49337 Mon Sep 17 00:00:00 2001 From: Khalah Jones-Golden Date: Wed, 23 Sep 2015 17:58:01 -0400 Subject: [PATCH 19/32] [Contributing] Started a contributing doc Original commit: elastic/x-pack-elasticsearch@3a29d1b360def1026046ed5792f74c5da7282a86 --- CONTRIBUTING.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..e5984c984a1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,21 @@ +# Development Environment Setup + +- You must have first setup [Kibana](https://github.com/elastic/kibana) by following their [contributing](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md) docs. + +- Next esvm is strongly suggested simply `npm install -g esvm` + +- Clone and go into this project +``` +git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearch-marvel +``` +- install dependencies +``` +npm install +``` +- Start it up + - Start Elasticsearch, with esvm `esvm --branch 2.0` or with traditionally `bin/elasticsearch`. + - Start up the mock Marvel agent with `gulp index` + - Start up the watching and sync of marvel and Kibana with `gulp dev` + - Finally startup Kibana from that directory run `bin/kibana` + +- Check it out, navigate to your [Kibana App](http://localhost:5601) From ccbaae20d8ebd62972939e65631fcdf1356c4d20 Mon Sep 17 00:00:00 2001 From: Khalah Jones-Golden Date: Thu, 24 Sep 2015 09:21:26 -0400 Subject: [PATCH 20/32] [cleanup] used less variables, cleaned up and made more clean Contributing.md Original commit: elastic/x-pack-elasticsearch@724bb509ec91c6fc5c808083bdd78a6238390943 --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5984c984a1..09e40e646dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,14 +8,14 @@ ``` git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearch-marvel ``` -- install dependencies +- Install dependencies ``` npm install ``` - Start it up - Start Elasticsearch, with esvm `esvm --branch 2.0` or with traditionally `bin/elasticsearch`. - Start up the mock Marvel agent with `gulp index` - - Start up the watching and sync of marvel and Kibana with `gulp dev` + - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev` - Finally startup Kibana from that directory run `bin/kibana` - Check it out, navigate to your [Kibana App](http://localhost:5601) From 53cad5f3c82865620506431e102adae920f6bf5d Mon Sep 17 00:00:00 2001 From: Court Ewing Date: Thu, 8 Oct 2015 11:16:55 -0400 Subject: [PATCH 21/32] Fix incorrect install command in readme Original commit: elastic/x-pack-elasticsearch@c0e292ed3ec667d627299d8232c2705a17346309 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 840ffd4ea8b..ef158c582c6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): ``` -./bin/plugin -i elasticsearch/marvel/latest +./bin/kibana plugin -i elasticsearch/marvel/latest ``` Once done, open up the following url (assuming standard ES config): http://localhost:9200/_plugin/marvel . This will take you to the Overview Dashboard. Use Kibana's Load dashboard menu to navigate to the Cluster Pulse dashboard From 274d217fa545e604b6350cec327fddd589308cb0 Mon Sep 17 00:00:00 2001 From: Khalah Jones-Golden Date: Thu, 22 Oct 2015 14:51:53 -0400 Subject: [PATCH 22/32] [Readme] Closes elastic/elasticsearch#575. Updated the readme to accomodate for new design Original commit: elastic/x-pack-elasticsearch@df76fcc08226c41655248e717a9004d1e9ea22ae --- README.md | 64 +++++++++---------------------------------------------- 1 file changed, 10 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index ef158c582c6..1dde0b485de 100644 --- a/README.md +++ b/README.md @@ -1,71 +1,27 @@ # WARNING THIS IS ALL IN FLUX +# Testing/Installing Marvel 2.0.0 -# Installing the last release of Marvel +The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.0.0 release version of it. -The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): - -``` -./bin/kibana plugin -i elasticsearch/marvel/latest -``` - -Once done, open up the following url (assuming standard ES config): http://localhost:9200/_plugin/marvel . This will take you to the Overview Dashboard. Use Kibana's Load dashboard menu to navigate to the Cluster Pulse dashboard +- Install the marvel plugin on kibana ` ./bin/kibana plugin -i elasticsearch/marvel/2.0.0 ` +- Install the License plugin on your cluster `./bin/plugin install license` +- Install the Marvel agent on your cluster ` ./bin/plugin install marvel-agent ` -## I just want to run Sense +Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel) -To run Sense, checkout a copy of this repo. You can now double click `sense/index.html` and it will open in your default browser. - -For a cleaner experience, we recommend using something like http://anvilformac.com and point it at the repo checkout. Once done you can access Sense via `http://webserver/sense/`, for example http://elasticsearch-marvel.dev/sense if using Anvil. - -Note: to run the Kibana side of Marvel, you'd need to install grunt as described bellow. - -## Grunt build system (for running the UI from a code checkout) -This grunt-based build system handles Kibana development environment setup for Marvel as well as building, packaging and distribution of the Marvel plugin. Note that you **must** run *grunt setup* before any other tasks as this build system reuses parts of the Kibana build system that must be fetched - -### Installing -You will need node.js+npm and grunt. Node is available via brew, install grunt with the command below. Once grunt is installed you may run grunt tasks to setup your environment and build Marvel - -```npm install -g grunt-cli``` - -```npm install``` - -### Tasks - -```grunt setup``` - -**Run this first.** It will download the right Kibana version to ./vendor/kibana, copies the appropriate config.js to the right place and make any symlinks needed for a proper marvel/kibana environment - -```grunt server``` - -Starts a web server on http://127.0.0.1:5601 pointing at the kibana directory, while also serving custom marvel panels. - -You can use `grunt server --port=5601 --es_host=9200` to control the ports used for kibana and the elasticsearch port used. - -```grunt jshint``` - -Lints code without building - -```grunt build``` - -Merges kibana and marvel code, builds Kibana and the plugin (via mvn) and puts them in ./build. - -```grunt package``` - -Zips and tar+gzips the build in build/packages. Includes grunt build - -```grunt release``` - -Uploads created archives to download.elasticsearch.org/elasticsearch/marvel/marvel-VERSION.extention. You will need S3 credentials in .aws-config.json. Format as so: +# Deploying Marvel +The `release task` creates archives and uploads them to download.elasticsearch.org/elasticsearch/marvel/VERSION. You will need S3 credentials in .aws-config.json. Format as so: ``` { "key":"MY_KEY_HERE", "secret":"your/long/secret/string" } - ``` To upload the current archive as the "latest" release, use: -```grunt release --latest``` +`gulp release` + From 13ee16c1d5c7333f2dba73cb2c92f2aa9ceab058 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 3 Nov 2015 13:44:00 -0700 Subject: [PATCH 23/32] Fixing recovery indexer Original commit: elastic/x-pack-elasticsearch@d045a4a805db76f63d99019d24381971ddde0fa7 --- marvel_index_template.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index 7237838ac83..97464c7f064 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -222,8 +222,12 @@ }, "index_recovery": { "properties": { - "shards": { - "type": "object" + "index_recovery": { + "properties": { + "shards": { + "type": "object" + } + } } } } From c0edaca67b0d48d6b6bcbe88f6dfd5ee64e0058c Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 3 Nov 2015 13:44:00 -0700 Subject: [PATCH 24/32] Fixing recovery indexer Original commit: elastic/x-pack-elasticsearch@cd39dd8ac0684c5d5f86b688e23436e401264d1c --- marvel_index_template.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/marvel_index_template.json b/marvel_index_template.json index 7237838ac83..97464c7f064 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -222,8 +222,12 @@ }, "index_recovery": { "properties": { - "shards": { - "type": "object" + "index_recovery": { + "properties": { + "shards": { + "type": "object" + } + } } } } From f84d2481f359a926b6780b52aa16902d81571ec2 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 5 Nov 2015 14:09:19 -0700 Subject: [PATCH 25/32] Initial indexing for nodes Original commit: elastic/x-pack-elasticsearch@45988280b7ed25774352168261cf9a40580db49b --- marvel_index_template.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/marvel_index_template.json b/marvel_index_template.json index 97464c7f064..c3fd8b6ff14 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -202,6 +202,23 @@ } } }, + "nodes": { + "properties": { + "state_uuid": { + "type": "string", + "index": "not_analyzed" + }, + "node": { + "properties": { + "id": { "type": "string", "index": "not_analyzed" }, + "name": { "type": "string", "index": "not_analyzed" }, + "transport_address": { "type": "string", "index": "not_analyzed" }, + "master": { "type": "boolean" }, + "attributes": { "type": "object" } + } + } + } + }, "shards": { "properties": { "state_uuid": { From 932ed158de9594a505b21004ad403ef35d0ab3d5 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 19 Nov 2015 17:27:42 -0600 Subject: [PATCH 26/32] Fixing issue with nodes Original commit: elastic/x-pack-elasticsearch@d8ef1d6023ee2d68c3971067aae7851be30f04bb --- marvel_index_template.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/marvel_index_template.json b/marvel_index_template.json index c3fd8b6ff14..03e663b96ea 100644 --- a/marvel_index_template.json +++ b/marvel_index_template.json @@ -202,6 +202,23 @@ } } }, + "node": { + "properties": { + "state_uuid": { + "type": "string", + "index": "not_analyzed" + }, + "node": { + "properties": { + "id": { "type": "string", "index": "not_analyzed" }, + "name": { "type": "string", "index": "not_analyzed" }, + "transport_address": { "type": "string", "index": "not_analyzed" }, + "master": { "type": "boolean" }, + "attributes": { "type": "object" } + } + } + } + }, "nodes": { "properties": { "state_uuid": { From e4b7f6af9491978542a4949bf4a8fe142ab61633 Mon Sep 17 00:00:00 2001 From: debadair Date: Wed, 18 Nov 2015 17:19:50 -0800 Subject: [PATCH 27/32] Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. Original commit: elastic/x-pack-elasticsearch@dc99f258996e20aa74da0b3ca46549d2eecd2ef5 --- CONTRIBUTING.md | 2 +- README.md | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09e40e646dd..ebc2dfa31f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearc npm install ``` - Start it up - - Start Elasticsearch, with esvm `esvm --branch 2.0` or with traditionally `bin/elasticsearch`. + - Start Elasticsearch, with esvm `esvm --branch 2.1` or with traditionally `bin/elasticsearch`. - Start up the mock Marvel agent with `gulp index` - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev` - Finally startup Kibana from that directory run `bin/kibana` diff --git a/README.md b/README.md index ef158c582c6..4bbb6d35dfe 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,19 @@ # WARNING THIS IS ALL IN FLUX +<<<<<<< HEAD # Installing the last release of Marvel The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): +======= +# Testing/Installing Marvel 2.1.0 + +The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.1.0 release version of it. + +- Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0` +- Install the License plugin on your cluster `./bin/plugin install license` +- Install the Marvel agent on your cluster `./bin/plugin install marvel-agent` +>>>>>>> 44fbb83... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. ``` ./bin/kibana plugin -i elasticsearch/marvel/latest From 3aa3376ca63bed91f5c0cf6656760e25a45d5f5c Mon Sep 17 00:00:00 2001 From: debadair Date: Wed, 18 Nov 2015 17:19:50 -0800 Subject: [PATCH 28/32] Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. Original commit: elastic/x-pack-elasticsearch@e39cf3e654582eed19a64694775ee3aa646c62c2 --- CONTRIBUTING.md | 2 +- README.md | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09e40e646dd..ebc2dfa31f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearc npm install ``` - Start it up - - Start Elasticsearch, with esvm `esvm --branch 2.0` or with traditionally `bin/elasticsearch`. + - Start Elasticsearch, with esvm `esvm --branch 2.1` or with traditionally `bin/elasticsearch`. - Start up the mock Marvel agent with `gulp index` - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev` - Finally startup Kibana from that directory run `bin/kibana` diff --git a/README.md b/README.md index 1dde0b485de..4ff5302a1b7 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,29 @@ # WARNING THIS IS ALL IN FLUX +<<<<<<< HEAD # Testing/Installing Marvel 2.0.0 +======= +<<<<<<< HEAD +>>>>>>> 46ac02d... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.0.0 release version of it. +<<<<<<< HEAD - Install the marvel plugin on kibana ` ./bin/kibana plugin -i elasticsearch/marvel/2.0.0 ` - Install the License plugin on your cluster `./bin/plugin install license` - Install the Marvel agent on your cluster ` ./bin/plugin install marvel-agent ` +======= +The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): +======= +# Testing/Installing Marvel 2.1.0 + +The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.1.0 release version of it. + +- Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0` +- Install the License plugin on your cluster `./bin/plugin install license` +- Install the Marvel agent on your cluster `./bin/plugin install marvel-agent` +>>>>>>> 44fbb83... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. +>>>>>>> 46ac02d... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel) From bfbf8a252d06859a37b74575e48271d59591724e Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 13:48:01 -0800 Subject: [PATCH 29/32] Fixed conflict in intro. Original commit: elastic/x-pack-elasticsearch@eee009f4c1e6977eb23c3c02acdee57d5ae136c3 --- README.md | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4ff5302a1b7..8be65c45693 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,14 @@ # WARNING THIS IS ALL IN FLUX -<<<<<<< HEAD -# Testing/Installing Marvel 2.0.0 -======= -<<<<<<< HEAD ->>>>>>> 46ac02d... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. -The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.0.0 release version of it. - -<<<<<<< HEAD -- Install the marvel plugin on kibana ` ./bin/kibana plugin -i elasticsearch/marvel/2.0.0 ` -- Install the License plugin on your cluster `./bin/plugin install license` -- Install the Marvel agent on your cluster ` ./bin/plugin install marvel-agent ` -======= -The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): -======= # Testing/Installing Marvel 2.1.0 -The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.1.0 release version of it. +The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc). +The second easiest way is to just install it. - Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0` - Install the License plugin on your cluster `./bin/plugin install license` - Install the Marvel agent on your cluster `./bin/plugin install marvel-agent` ->>>>>>> 44fbb83... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. ->>>>>>> 46ac02d... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. - Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel) From 611db9ba17f525b892b00b2cbceed1d81ab3513a Mon Sep 17 00:00:00 2001 From: debadair Date: Mon, 23 Nov 2015 14:00:48 -0800 Subject: [PATCH 30/32] Fixing conflicts with Master Original commit: elastic/x-pack-elasticsearch@63396e9588781156f0433a5a44e5341f2285177d --- README.md | 67 ++++++------------------------------------------------- 1 file changed, 7 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 4bbb6d35dfe..8be65c45693 100644 --- a/README.md +++ b/README.md @@ -1,81 +1,28 @@ # WARNING THIS IS ALL IN FLUX -<<<<<<< HEAD -# Installing the last release of Marvel - -The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): -======= # Testing/Installing Marvel 2.1.0 -The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc), the second easiest way, however, is probably to install the 2.1.0 release version of it. +The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc). +The second easiest way is to just install it. - Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0` - Install the License plugin on your cluster `./bin/plugin install license` - Install the Marvel agent on your cluster `./bin/plugin install marvel-agent` ->>>>>>> 44fbb83... Docs: Bumped version number to 2.1.0 and added 2.1.0 release notes. -``` -./bin/kibana plugin -i elasticsearch/marvel/latest -``` +Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel) -Once done, open up the following url (assuming standard ES config): http://localhost:9200/_plugin/marvel . This will take you to the Overview Dashboard. Use Kibana's Load dashboard menu to navigate to the Cluster Pulse dashboard - - -## I just want to run Sense - -To run Sense, checkout a copy of this repo. You can now double click `sense/index.html` and it will open in your default browser. - -For a cleaner experience, we recommend using something like http://anvilformac.com and point it at the repo checkout. Once done you can access Sense via `http://webserver/sense/`, for example http://elasticsearch-marvel.dev/sense if using Anvil. - -Note: to run the Kibana side of Marvel, you'd need to install grunt as described bellow. - -## Grunt build system (for running the UI from a code checkout) -This grunt-based build system handles Kibana development environment setup for Marvel as well as building, packaging and distribution of the Marvel plugin. Note that you **must** run *grunt setup* before any other tasks as this build system reuses parts of the Kibana build system that must be fetched - -### Installing -You will need node.js+npm and grunt. Node is available via brew, install grunt with the command below. Once grunt is installed you may run grunt tasks to setup your environment and build Marvel - -```npm install -g grunt-cli``` - -```npm install``` - -### Tasks - -```grunt setup``` - -**Run this first.** It will download the right Kibana version to ./vendor/kibana, copies the appropriate config.js to the right place and make any symlinks needed for a proper marvel/kibana environment - -```grunt server``` - -Starts a web server on http://127.0.0.1:5601 pointing at the kibana directory, while also serving custom marvel panels. - -You can use `grunt server --port=5601 --es_host=9200` to control the ports used for kibana and the elasticsearch port used. - -```grunt jshint``` - -Lints code without building - -```grunt build``` - -Merges kibana and marvel code, builds Kibana and the plugin (via mvn) and puts them in ./build. - -```grunt package``` - -Zips and tar+gzips the build in build/packages. Includes grunt build - -```grunt release``` - -Uploads created archives to download.elasticsearch.org/elasticsearch/marvel/marvel-VERSION.extention. You will need S3 credentials in .aws-config.json. Format as so: +# Deploying Marvel +The `release task` creates archives and uploads them to download.elasticsearch.org/elasticsearch/marvel/VERSION. You will need S3 credentials in .aws-config.json. Format as so: ``` { "key":"MY_KEY_HERE", "secret":"your/long/secret/string" } - ``` To upload the current archive as the "latest" release, use: -```grunt release --latest``` +`gulp release` + From 3982c12964ac5ca39df09e18398e46e254600061 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Thu, 12 Nov 2015 13:49:12 -0700 Subject: [PATCH 31/32] add the first server-side unit test Original commit: elastic/x-pack-elasticsearch@d6680ecb57a22461aff20e39a1e5387affebc2fe --- .eslintrc | 6 ++++-- CONTRIBUTING.md | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.eslintrc b/.eslintrc index 136ac914a87..891bdfbe209 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,13 +1,13 @@ --- parser: babel-eslint +plugins: [ mocha ] + env: es6: true amd: true node: true mocha: true - browser: true - rules: block-scoped-var: 2 @@ -67,3 +67,5 @@ rules: valid-typeof: 2 wrap-iife: [ 2, outside ] yoda: 0 + mocha/no-exclusive-tests: 2 + mocha/handle-done-callback: 2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ebc2dfa31f1..90143a845cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,9 +13,19 @@ git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearc npm install ``` - Start it up - - Start Elasticsearch, with esvm `esvm --branch 2.1` or with traditionally `bin/elasticsearch`. - - Start up the mock Marvel agent with `gulp index` - - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev` - - Finally startup Kibana from that directory run `bin/kibana` - + - Start Elasticsearch, with esvm `esvm --branch ` or traditionally with `bin/elasticsearch`. + - Start up the mock Marvel agent with `gulp index`. + - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev`. + - Finally, startup Kibana by running `bin/kibana --dev` from the root of the Kibana project. - Check it out, navigate to your [Kibana App](http://localhost:5601) + +- Run tests +``` +gulp test +``` + +- Debug tests +Add a `debugger` line to create a breakpoint, and then: +``` +gulp sync && mocha debug --compilers js:babel/register /pathto/kibana/installedPlugins/marvel/pathto/__test__/testfile.js +``` From d3fb668346d5b27499857a8783f4a393e724c327 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Sun, 6 Dec 2015 00:51:20 -0700 Subject: [PATCH 32/32] move public code to kibana/marvel Original commit: elastic/x-pack-elasticsearch@13d409da5d89ffd8bca88ea3e48c376a8cbe2f14 --- .agignore | 3 - .eslintrc | 71 ---- .jscsrc | 59 ---- .jshintignore | 2 - .jshintrc | 27 -- .jshintrc.browser | 14 - .jshintrc.node | 9 - CONTRIBUTING.md | 31 -- NOTICE.txt | 5 - README.md | 28 -- marvel_index_template.json | 269 ---------------- watcher/README.md | 11 - watcher/lib/client.js | 4 - watcher/lib/create_template.js | 11 - watcher/lib/execute_watcher.js | 40 --- watcher/lib/find_port.js | 13 - watcher/lib/inject_stats.js | 73 ----- watcher/lib/load_watcher.js | 26 -- watcher/lib/setup_es.js | 71 ---- watcher/lib/setup_smtp_server.js | 56 ---- watcher/lib/template.json | 447 -------------------------- watcher/lib/test_no_execute.js | 28 -- watcher/package.json | 23 -- watcher/test/cluster_status.js | 95 ------ watcher/test/cpu_usage.js | 81 ----- watcher/test/fielddata.js | 82 ----- watcher/test/file_descriptors.js | 82 ----- watcher/test/heap_used.js | 81 ----- watcher/test/mocha.opts | 2 - watcher/watches/cluster_status.json | 92 ------ watcher/watches/cpu_usage.json | 73 ----- watcher/watches/fielddata.json | 79 ----- watcher/watches/file_descriptors.json | 79 ----- watcher/watches/heap_used.json | 73 ----- 34 files changed, 2140 deletions(-) delete mode 100644 .agignore delete mode 100644 .eslintrc delete mode 100644 .jscsrc delete mode 100644 .jshintignore delete mode 100644 .jshintrc delete mode 100644 .jshintrc.browser delete mode 100644 .jshintrc.node delete mode 100644 CONTRIBUTING.md delete mode 100644 NOTICE.txt delete mode 100644 README.md delete mode 100644 marvel_index_template.json delete mode 100644 watcher/README.md delete mode 100644 watcher/lib/client.js delete mode 100644 watcher/lib/create_template.js delete mode 100644 watcher/lib/execute_watcher.js delete mode 100644 watcher/lib/find_port.js delete mode 100644 watcher/lib/inject_stats.js delete mode 100644 watcher/lib/load_watcher.js delete mode 100644 watcher/lib/setup_es.js delete mode 100644 watcher/lib/setup_smtp_server.js delete mode 100644 watcher/lib/template.json delete mode 100644 watcher/lib/test_no_execute.js delete mode 100644 watcher/package.json delete mode 100644 watcher/test/cluster_status.js delete mode 100644 watcher/test/cpu_usage.js delete mode 100644 watcher/test/fielddata.js delete mode 100644 watcher/test/file_descriptors.js delete mode 100644 watcher/test/heap_used.js delete mode 100644 watcher/test/mocha.opts delete mode 100644 watcher/watches/cluster_status.json delete mode 100644 watcher/watches/cpu_usage.json delete mode 100644 watcher/watches/fielddata.json delete mode 100644 watcher/watches/file_descriptors.json delete mode 100644 watcher/watches/heap_used.json diff --git a/.agignore b/.agignore deleted file mode 100644 index 10fb4038cbd..00000000000 --- a/.agignore +++ /dev/null @@ -1,3 +0,0 @@ -agent -node_modules -bower_components diff --git a/.eslintrc b/.eslintrc deleted file mode 100644 index 891bdfbe209..00000000000 --- a/.eslintrc +++ /dev/null @@ -1,71 +0,0 @@ ---- -parser: babel-eslint - -plugins: [ mocha ] - -env: - es6: true - amd: true - node: true - mocha: true - -rules: - block-scoped-var: 2 - camelcase: [ 2, { properties: never } ] - comma-dangle: 0 - comma-style: [ 2, last ] - consistent-return: 0 - curly: [ 2, multi-line ] - dot-location: [ 2, property ] - dot-notation: [ 2, { allowKeywords: true } ] - eqeqeq: [ 2, allow-null ] - guard-for-in: 2 - indent: [ 2, 2, { SwitchCase: 1 } ] - key-spacing: [ 0, { align: value } ] - max-len: [ 2, 140, 2, { ignoreComments: true, ignoreUrls: true } ] - new-cap: [ 2, { capIsNewExceptions: [ Private ] } ] - no-bitwise: 0 - no-caller: 2 - no-cond-assign: 0 - no-debugger: 2 - no-empty: 2 - no-eval: 2 - no-extend-native: 2 - no-extra-parens: 0 - no-irregular-whitespace: 2 - no-iterator: 2 - no-loop-func: 2 - no-multi-spaces: 0 - no-multi-str: 2 - no-nested-ternary: 2 - no-new: 0 - no-path-concat: 0 - no-proto: 2 - no-return-assign: 0 - no-script-url: 2 - no-sequences: 2 - no-shadow: 0 - no-trailing-spaces: 2 - no-undef: 2 - no-underscore-dangle: 0 - no-unused-expressions: 0 - no-unused-vars: 0 - no-use-before-define: [ 2, nofunc ] - no-with: 2 - one-var: [ 2, never ] - quotes: [ 2, single ] - semi-spacing: [ 2, { before: false, after: true } ] - semi: [ 2, always ] - space-after-keywords: [ 2, always ] - space-before-blocks: [ 2, always ] - space-before-function-paren: [ 2, { anonymous: always, named: never } ] - space-in-parens: [ 2, never ] - space-infix-ops: [ 2, { int32Hint: false } ] - space-return-throw-case: [ 2 ] - space-unary-ops: [ 2 ] - strict: [ 2, never ] - valid-typeof: 2 - wrap-iife: [ 2, outside ] - yoda: 0 - mocha/no-exclusive-tests: 2 - mocha/handle-done-callback: 2 diff --git a/.jscsrc b/.jscsrc deleted file mode 100644 index d8ea99cdc6c..00000000000 --- a/.jscsrc +++ /dev/null @@ -1,59 +0,0 @@ -{ - "maximumLineLength": { - "value": 140, - "allowComments": true - }, - "requireCurlyBraces": [ - "for", - "do", - "try", - "catch" - ], - "requireSpaceAfterKeywords": [ - "if", - "else", - "for", - "while", - "do", - "switch", - "case", - "return", - "try", - "catch", - "function", - "typeof" - ], - "requireSpaceBeforeBlockStatements": true, - "requireParenthesesAroundIIFE": true, - "requireSpacesInConditionalExpression": true, - "disallowSpacesInNamedFunctionExpression": { - "beforeOpeningRoundBrace": true - }, - "disallowSpacesInFunctionDeclaration": { - "beforeOpeningRoundBrace": true - }, - "requireSpaceBetweenArguments": true, - "disallowMultipleVarDecl": "exceptUndefined", - "disallowEmptyBlocks": true, - "requireCommaBeforeLineBreak": true, - "disallowSpaceAfterPrefixUnaryOperators": true, - "disallowSpaceBeforePostfixUnaryOperators": true, - "disallowSpaceBeforeBinaryOperators": [ - "," - ], - "requireSpacesInForStatement": true, - "requireSpaceBeforeBinaryOperators": true, - "requireSpaceAfterBinaryOperators": true, - "disallowKeywords": [ - "with" - ], - "validateIndentation": 2, - "validateQuoteMarks": { "mark": "'", "escape": true }, - "disallowMixedSpacesAndTabs": true, - "disallowTrailingWhitespace": true, - "requireCapitalizedConstructors": true, - "requireDotNotation": true, - "disallowNewlineBeforeBlockStatements": true, - "disallowMultipleLineStrings": true, - "esprima" : "esprima-fb" -} diff --git a/.jshintignore b/.jshintignore deleted file mode 100644 index a7d5694758a..00000000000 --- a/.jshintignore +++ /dev/null @@ -1,2 +0,0 @@ -vendor -Gruntfile.js diff --git a/.jshintrc b/.jshintrc deleted file mode 100644 index 8e5b2febb1b..00000000000 --- a/.jshintrc +++ /dev/null @@ -1,27 +0,0 @@ -{ - // for files at project root - "node": true, - - // shared with .jshintrc files for browser and node - "unused": false, - "bitwise": false, - "eqnull": true, - "eqeqeq": true, - "forin": true, - "immed": true, - "expr": true, - "latedef": "nofunc", - "noarg": true, - "noempty": true, - "undef": true, - "quotmark": "single", - "plusplus": false, - "boss": true, - "laxbreak": true, - "laxcomma": true, - "validthis": true, - "sub": true, - "-W084": true, - "scripturl": true, - "evil": true -} diff --git a/.jshintrc.browser b/.jshintrc.browser deleted file mode 100644 index 09cdd36321a..00000000000 --- a/.jshintrc.browser +++ /dev/null @@ -1,14 +0,0 @@ -{ - "extends": "./.jshintrc", - "node": false, - "browser": true, - - "predef": { - // require.js - "define": true, - "require": true, - "console": false, - "-event": true, - "-name": true - } -} diff --git a/.jshintrc.node b/.jshintrc.node deleted file mode 100644 index a77f81af35e..00000000000 --- a/.jshintrc.node +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "./.jshintrc", - "node": true, - - "globals": { - "Promise": true, - "status": true - } -} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 90143a845cd..00000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,31 +0,0 @@ -# Development Environment Setup - -- You must have first setup [Kibana](https://github.com/elastic/kibana) by following their [contributing](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md) docs. - -- Next esvm is strongly suggested simply `npm install -g esvm` - -- Clone and go into this project -``` -git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearch-marvel -``` -- Install dependencies -``` -npm install -``` -- Start it up - - Start Elasticsearch, with esvm `esvm --branch ` or traditionally with `bin/elasticsearch`. - - Start up the mock Marvel agent with `gulp index`. - - Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev`. - - Finally, startup Kibana by running `bin/kibana --dev` from the root of the Kibana project. -- Check it out, navigate to your [Kibana App](http://localhost:5601) - -- Run tests -``` -gulp test -``` - -- Debug tests -Add a `debugger` line to create a breakpoint, and then: -``` -gulp sync && mocha debug --compilers js:babel/register /pathto/kibana/installedPlugins/marvel/pathto/__test__/testfile.js -``` diff --git a/NOTICE.txt b/NOTICE.txt deleted file mode 100644 index 86016475acd..00000000000 --- a/NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -ElasticSearch -Copyright 2009-2013 ElasticSearch - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). diff --git a/README.md b/README.md deleted file mode 100644 index 8be65c45693..00000000000 --- a/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# WARNING THIS IS ALL IN FLUX - - -# Testing/Installing Marvel 2.1.0 - -The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc). -The second easiest way is to just install it. - -- Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0` -- Install the License plugin on your cluster `./bin/plugin install license` -- Install the Marvel agent on your cluster `./bin/plugin install marvel-agent` - -Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel) - -# Deploying Marvel -The `release task` creates archives and uploads them to download.elasticsearch.org/elasticsearch/marvel/VERSION. You will need S3 credentials in .aws-config.json. Format as so: - -``` -{ - "key":"MY_KEY_HERE", - "secret":"your/long/secret/string" -} -``` - -To upload the current archive as the "latest" release, use: - -`gulp release` - diff --git a/marvel_index_template.json b/marvel_index_template.json deleted file mode 100644 index 03e663b96ea..00000000000 --- a/marvel_index_template.json +++ /dev/null @@ -1,269 +0,0 @@ - -{ - "template": ".marvel*", - "settings": { - "marvel.index_format": 7, - "marvel.version": "${project.version}", - "index.number_of_shards": 1, - "index.number_of_replicas": 1, - "index.codec": "best_compression", - "index.mapper.dynamic": false - }, - "mappings": { - "_default_": { - "_all": { - "enabled": false - }, - "date_detection": false, - "properties": { - "cluster_uuid": { - "type": "string", - "index": "not_analyzed" - }, - "timestamp": { - "type": "date", - "format": "date_time" - } - } - }, - "index_stats": { - "properties": { - "index_stats": { - "properties": { - "index": { - "type": "string", - "index": "not_analyzed" - }, - "primaries": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - } - } - }, - "total": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - }, - "store": { - "properties": { - "size_in_bytes": { - "type": "long" - }, - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "indexing": { - "properties": { - "index_total": { - "type": "long" - }, - "index_time_in_millis": { - "type": "long" - }, - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "merges": { - "properties": { - "total_size_in_bytes": { - "type": "long" - } - } - }, - "search": { - "properties": { - "query_total": { - "type": "long" - }, - "query_time_in_millis": { - "type": "long" - } - } - }, - "segments": { - "properties": { - "memory_in_bytes": { - "type": "long" - } - } - }, - "refresh": { - "properties": { - "total_time_in_millis": { - "type": "long" - } - } - } - } - } - } - } - } - }, - "cluster_stats": { - "properties": { - "cluster_stats": { - "properties": { - "nodes": { - "type": "object" - }, - "indices": { - "type": "object" - } - } - } - } - }, - "cluster_state": { - "properties": { - "cluster_state": { - "properties": { - "version": { - "type": "long" - }, - "master_node": { - "type": "string", - "index": "not_analyzed" - }, - "status": { - "type": "string", - "index": "not_analyzed" - }, - "nodes": { - "enabled": false - } - } - } - } - }, - "cluster_info": { - "enabled": false - }, - "phone_home": { - "enabled": false - }, - "node_stats": { - "properties": { - "node_stats": { - "properties": { - "node_id": { - "type": "string", - "index": "not_analyzed" - }, - "node_master": { - "type": "boolean" - }, - "mlockall": { - "type": "boolean" - }, - "disk_threshold_enabled": { - "type": "boolean" - }, - "disk_threshold_watermark_high": { - "type": "short" - }, - "indices": { - "type": "object" - }, - "fs": { - "type": "object" - }, - "process": { - "type": "object" - }, - "jvm": { - "type": "object" - }, - "thread_pool": { - "type": "object" - } - } - } - } - }, - "indices_stats": { - "properties": { - "indices_stats": { - "type": "object" - } - } - }, - "node": { - "properties": { - "state_uuid": { - "type": "string", - "index": "not_analyzed" - }, - "node": { - "properties": { - "id": { "type": "string", "index": "not_analyzed" }, - "name": { "type": "string", "index": "not_analyzed" }, - "transport_address": { "type": "string", "index": "not_analyzed" }, - "master": { "type": "boolean" }, - "attributes": { "type": "object" } - } - } - } - }, - "nodes": { - "properties": { - "state_uuid": { - "type": "string", - "index": "not_analyzed" - }, - "node": { - "properties": { - "id": { "type": "string", "index": "not_analyzed" }, - "name": { "type": "string", "index": "not_analyzed" }, - "transport_address": { "type": "string", "index": "not_analyzed" }, - "master": { "type": "boolean" }, - "attributes": { "type": "object" } - } - } - } - }, - "shards": { - "properties": { - "state_uuid": { - "type": "string", - "index": "not_analyzed" - }, - "shard": { - "properties": { - "state": { "type": "string", "index": "not_analyzed" }, - "primary": { "type": "boolean" }, - "index": { "type": "string", "index": "not_analyzed" }, - "relocating_node": { "type": "string", "index": "not_analyzed" }, - "shard": { "type": "long" }, - "node": { "type": "string", "index": "not_analyzed" } - } - } - } - }, - "index_recovery": { - "properties": { - "index_recovery": { - "properties": { - "shards": { - "type": "object" - } - } - } - } - } - } -} diff --git a/watcher/README.md b/watcher/README.md deleted file mode 100644 index 0d29c7e0225..00000000000 --- a/watcher/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Marvel Watchers - -In this directory you will find example Watchers for Marvel that were distributed with the Watcher Beta documentation. The `watches` directory contains the example watches and the `test` directory contains integration test for each watcher. - -The testing framework is written in Node.js and use ESVM to create an Elasticsearch instance for running the tests against - -### Setup and Running Tests - -- Run `npm install` in this directory -- Ensure you have `mocha` installed globally -- Run `mocha` in this directory diff --git a/watcher/lib/client.js b/watcher/lib/client.js deleted file mode 100644 index 7118f52e4fc..00000000000 --- a/watcher/lib/client.js +++ /dev/null @@ -1,4 +0,0 @@ -var Client = require('elasticsearch').Client; -module.exports = new Client({ - host: 'http://localhost:9800' -}); diff --git a/watcher/lib/create_template.js b/watcher/lib/create_template.js deleted file mode 100644 index bf0506c3daf..00000000000 --- a/watcher/lib/create_template.js +++ /dev/null @@ -1,11 +0,0 @@ -var client = require('./client'); -var template = require('./template.json'); - -module.exports = function () { - return client.indices.putTemplate({ - body: template, - name: 'marvel' - }); -}; - - diff --git a/watcher/lib/execute_watcher.js b/watcher/lib/execute_watcher.js deleted file mode 100644 index 2247ff83112..00000000000 --- a/watcher/lib/execute_watcher.js +++ /dev/null @@ -1,40 +0,0 @@ -var Promise = require('bluebird'); -var request = require('request'); -var injectStats = require('./inject_stats'); -var loadWatcher = require('./load_watcher'); - -module.exports = function (watcher, fixture) { - - return injectStats(fixture).then(function () { - return loadWatcher(watcher); - }).then(function () { - return new Promise(function (resolve, reject) { - var options = { - method: 'POST', - url: 'http://localhost:9800/_watcher/watch/' + watcher + '/_execute', - json: true, - body: { - trigger_event: { - schedule: { - scheduled_time: 'now', - triggered_time: 'now' - } - } - } - }; - - request(options, function (err, resp, body) { - if (err) return reject(err); - if (resp.statusCode === 200) return resolve(body); - var message = options.url + ' responed with ' + resp.statusCode; - var error = new Error(body.error || message); - error.body = body; - error.resp = resp; - error.code = resp.statusCode; - error.options = options; - reject(error); - }); - }); - }); - -}; diff --git a/watcher/lib/find_port.js b/watcher/lib/find_port.js deleted file mode 100644 index 93313bb095e..00000000000 --- a/watcher/lib/find_port.js +++ /dev/null @@ -1,13 +0,0 @@ -var Promise = require('bluebird'); -var portscanner = require('portscanner'); -module.exports = function findPort(start, end, host) { - host = host || 'localhost'; - return new Promise(function (resolve, reject) { - portscanner.findAPortNotInUse(start, end, host, function (err, port) { - if (err) return reject(err); - resolve(port); - }); - }); -}; - - diff --git a/watcher/lib/inject_stats.js b/watcher/lib/inject_stats.js deleted file mode 100644 index 9a2b1cb6b96..00000000000 --- a/watcher/lib/inject_stats.js +++ /dev/null @@ -1,73 +0,0 @@ -var _ = require('lodash'); -var moment = require('moment'); -var client = require('./client'); - -module.exports = function (fixture) { - var indexPattern = fixture.indexPattern; - var type = fixture.type; - var data = fixture.data; - var rawData = fixture.rawData; - var startDate = fixture.startDate; - var duration = fixture.duration; - var dateField = fixture.dateField; - var workingDate = moment.utc(); - var indices = []; - var body = []; - var fields; - - - function createEntries(row) { - var index = workingDate.format(indexPattern); - var entry = { 'timestamp': workingDate.toISOString() }; - row.forEach(function (val, index) { - _.set(entry, fields[index], val); - }); - - indices.push(index); - - body.push({ - index: { - _index: index, - _type: type - } - }); - body.push(entry); - } - - if (rawData) { - rawData.forEach(function (row) { - var index = moment.utc(row[dateField]).format(indexPattern); - var entry = {}; - _.each(row, function (val, key) { - _.set(entry, key, val); - }); - indices.push(index); - body.push({ - index: { - _index: index, - _type: type - } - }); - body.push(entry); - }); - } else { - fields = data.shift(); - while(startDate <= workingDate) { - data.forEach(createEntries); - workingDate.subtract(duration); - } - } - - return client.deleteByQuery({ - index: _.unique(indices), - ignoreUnavailable: true, - allowNoIndices: true, - q: '*' - }) - .then(function (arg) { - return client.bulk({ - body: body, - refresh: true - }); - }); -}; diff --git a/watcher/lib/load_watcher.js b/watcher/lib/load_watcher.js deleted file mode 100644 index 6fc03a7fa08..00000000000 --- a/watcher/lib/load_watcher.js +++ /dev/null @@ -1,26 +0,0 @@ -var Promise = require('bluebird'); -var request = require('request'); -var path = require('path'); - -module.exports = function (name) { - var watch = require(path.join(__dirname, '..', 'watches', name + '.json')); - var options = { - method: 'PUT', - url: 'http://localhost:9800/_watcher/watch/' + name, - json: true, - body: watch - }; - return new Promise(function (resolve, reject) { - request(options, function (err, resp, body) { - if (err) return reject(err); - if (resp.statusCode <= 201) resolve({ resp: resp, body: body }); - var message = options.url + ' responded with ' + resp.statusCode; - var error = new Error(body.error || message); - error.body = body; - error.resp = resp; - error.code = resp.statusCode; - error.options = options; - reject(error); - }); - }); -}; diff --git a/watcher/lib/setup_es.js b/watcher/lib/setup_es.js deleted file mode 100644 index 436f37df02c..00000000000 --- a/watcher/lib/setup_es.js +++ /dev/null @@ -1,71 +0,0 @@ -var path = require('path'); -var Promise = require('bluebird'); -var libesvm = require('libesvm'); -var createTemplate = require('./create_template'); - -function startEs() { - var options = { - version: '1.5.2', - directory: path.join(__dirname, '..', 'esvm'), - purge: true, - plugins: [ - 'elasticsearch/watcher/1.0.0-Beta1-5', - 'elasticsearch/license/latest' - ], - config: { - 'script.groovy.sandbox.enabled': true, - 'cluster.name': 'test', - 'network.host': '127.0.0.1', - 'http.port': 9800, - 'watcher.actions.email.service.account': { - 'local': { - 'email_defaults.from': 'admin@example.com', - 'smtp': { - 'host': 'localhost', - 'user': 'test', - 'password': 'test', - 'port': 5555 - } - } - } - - } - }; - var cluster = libesvm.createCluster(options); - cluster.on('log', function (log) { - if (log.type === 'progress') return; - if (process.env.DEBUG) console.log('%s %s %s %s', log.level, log.node, log.type, log.message); - }); - return cluster.install() - .then(function () { - return cluster.installPlugins(); - }) - .then(function () { - return cluster.start(); - }) - .then(function () { - after(function () { - this.timeout(60000); - return cluster.shutdown(); - }); - return cluster; - }); -} - -before(function () { - var self = this; - this.timeout(60000); - return new Promise(function (resolve, reject) { - startEs().then(function (cluster) { - self.cluster = cluster; - cluster.on('log', function (log) { - if (/watch service has started/.test(log.message)) { - createTemplate().then(function () { - resolve(cluster); - }); - } - }); - }).catch(reject); - }); -}); - diff --git a/watcher/lib/setup_smtp_server.js b/watcher/lib/setup_smtp_server.js deleted file mode 100644 index 12e4f7ec922..00000000000 --- a/watcher/lib/setup_smtp_server.js +++ /dev/null @@ -1,56 +0,0 @@ -var Promise = require('bluebird'); -var SMTPServer = require('smtp-server').SMTPServer; -var MailParser = require('mailparser').MailParser; -var acceptAny = function (address, session, done) { - return done(); -}; - -var mailbox = []; - -function startSMTP() { - return new Promise(function (resolve, reject) { - var server = new SMTPServer({ - logger: false, - disabledCommands: ['STARTTLS'], - onAuth: function (auth, session, done) { - done(null, { user: 1 }); - }, - onMailFrom: acceptAny, - onRcptTo: acceptAny, - onData: function (stream, session, done) { - var mailparser = new MailParser(); - mailparser.on('end', function (mailObj) { - mailbox.push(mailObj); - done(); - }); - stream.pipe(mailparser); - } - }); - - server.listen(5555, function (err) { - if (err) return reject(err); - after(function (done) { - server.close(done); - }); - resolve(server); - }); - }); -} - -before(function () { - var self = this; - return startSMTP().then(function (server) { - this.smtp = server; - return server; - }); -}); - -beforeEach(function () { - this.mailbox = mailbox; -}); - -afterEach(function () { - mailbox = []; -}); - -module.exports = mailbox; diff --git a/watcher/lib/template.json b/watcher/lib/template.json deleted file mode 100644 index 362fc0d2c38..00000000000 --- a/watcher/lib/template.json +++ /dev/null @@ -1,447 +0,0 @@ -{ - "template": ".marvel*", - "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "analysis": { - "analyzer": { - "default": { - "type": "standard", - "stopwords": "_none_" - } - } - }, - "mapper.dynamic": true, - "marvel.index_format": 6 - }, - "mappings": { - "_default_": { - "dynamic_templates": [ - { - "string_fields": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "multi_field", - "fields": { - "{name}": { - "type": "string", - "index": "analyzed", - "omit_norms": true - }, - "raw": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 256 - } - } - } - } - } - ] - }, - "node_stats": { - "properties": { - "breakers": { - "properties": { - "fielddata": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - }, - "request": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - }, - "parent": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - } - } - }, - "fs": { - "properties": { - "total": { - "properties": { - "disk_io_op": { - "type": "long" - }, - "disk_reads": { - "type": "long" - }, - "disk_writes": { - "type": "long" - }, - "disk_io_size_in_bytes": { - "type": "long" - }, - "disk_read_size_in_bytes": { - "type": "long" - }, - "disk_write_size_in_bytes": { - "type": "long" - } - } - } - } - }, - "jvm": { - "properties": { - "buffer_pools": { - "properties": { - "direct": { - "properties": { - "used_in_bytes": { - "type": "long" - } - } - }, - "mapped": { - "properties": { - "used_in_bytes": { - "type": "long" - } - } - } - } - }, - "gc": { - "properties": { - "collectors": { - "properties": { - "young": { - "properties": { - "collection_count": { - "type": "long" - }, - "collection_time_in_millis": { - "type": "long" - } - } - }, - "old": { - "properties": { - "collection_count": { - "type": "long" - }, - "collection_time_in_millis": { - "type": "long" - } - } - } - } - } - } - } - } - }, - "indices": { - "properties": { - "indexing": { - "properties": { - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "percolate": { - "properties": { - "total": { - "type": "long" - }, - "time_in_millis": { - "type": "long" - }, - "queries": { - "type": "long" - }, - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "segments": { - "properties": { - "index_writer_memory_in_bytes": { - "type": "long" - }, - "version_map_memory_in_bytes": { - "type": "long" - }, - "index_writer_max_memory_in_bytes": { - "type": "long" - } - } - }, - "query_cache": { - "properties": { - "memory_size_in_bytes": { - "type": "long" - }, - "evictions": { - "type": "long" - }, - "hit_count": { - "type": "long" - }, - "miss_count": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "load_average": { - "properties": { - "1m": { - "type": "float" - }, - "5m": { - "type": "float" - }, - "15m": { - "type": "float" - } - } - } - } - }, - "thread_pool": { - "properties": { - "listener": { - "properties": { - "threads": { - "type": "long" - }, - "rejected": { - "type": "long" - }, - "completed": { - "type": "long" - }, - "queue": { - "type": "long" - }, - "largest": { - "type": "long" - } - } - } - } - } - } - }, - "index_stats": { - "properties": { - "index": { - "type": "multi_field", - "fields": { - "index": { - "type": "string", - "norms": { - "enabled": false - } - }, - "raw": { - "type": "string", - "index": "not_analyzed", - "norms": { - "enabled": false - }, - "index_options": "docs", - "include_in_all": false, - "ignore_above": 256 - } - } - }, - "total": { - "properties": { - "fielddata": { - "properties": { - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "indexing": { - "properties": { - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "merges": { - "properties": { - "total_size_in_bytes": { - "type": "long" - } - } - }, - "percolate": { - "properties": { - "total": { - "type": "long" - }, - "time_in_millis": { - "type": "long" - }, - "queries": { - "type": "long" - }, - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "search": { - "properties": { - "query": { - "properties": { - "query_total": { - "type": "long" - } - } - } - } - }, - "segments": { - "properties": { - "index_writer_memory_in_bytes": { - "type": "long" - }, - "version_map_memory_in_bytes": { - "type": "long" - }, - "index_writer_max_memory_in_bytes": { - "type": "long" - } - } - }, - "query_cache": { - "properties": { - "memory_size_in_bytes": { - "type": "long" - }, - "evictions": { - "type": "long" - }, - "hit_count": { - "type": "long" - }, - "miss_count": { - "type": "long" - } - } - } - } - }, - "primaries": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - }, - "indexing": { - "properties": { - "index_total": { - "type": "long" - } - } - } - } - } - } - }, - "cluster_event": {}, - "shard_event": {}, - "indices_stats": { - "properties": { - "primaries": { - "properties": { - "indexing": { - "properties": { - "index_total": { - "type": "long" - } - } - }, - "docs": { - "properties": { - "count": { - "type": "long" - } - } - } - } - }, - "total": { - "properties": { - "search": { - "properties": { - "query_total": { - "type": "long" - } - } - } - } - } - } - }, - "cluster_stats": {}, - "index_event": {}, - "node_event": {}, - "routing_event": {}, - "cluster_state": { - "properties": { - "blocks": { - "type": "object", - "enabled": false - }, - "nodes": { - "type": "object", - "enabled": false - }, - "routing_nodes": { - "type": "object", - "enabled": false - }, - "routing_table": { - "type": "object", - "enabled": false - } - } - } - } -} - diff --git a/watcher/lib/test_no_execute.js b/watcher/lib/test_no_execute.js deleted file mode 100644 index c90fe915a67..00000000000 --- a/watcher/lib/test_no_execute.js +++ /dev/null @@ -1,28 +0,0 @@ -var lib = require('requirefrom')('lib'); -var executeWatcher = lib('execute_watcher'); -var expect = require('expect.js'); -module.exports = function testNoExecute(options, message, generateRawData) { - describe(message, function () { - var response; - beforeEach(function () { - this.timeout(5000); - var rawData = generateRawData(); - var fixture = { - indexPattern: options.indexPattern, - type: options.type, - dateField: 'timestamp', - rawData: rawData - }; - return executeWatcher(options.watcher, fixture).then(function (resp) { - response = resp; - if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' ')); - return resp; - }); - }); - it('should not meet the script condition', function () { - expect(response.state).to.be('execution_not_needed'); - expect(response.execution_result.condition.script.met).to.be(false); - }); - }); -}; - diff --git a/watcher/package.json b/watcher/package.json deleted file mode 100644 index ab38bb94d83..00000000000 --- a/watcher/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "name": "marvel-watchers", - "version": "1.0.0", - "description": "", - "main": "index.js", - "directories": { - "test": "test" - }, - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "author": "", - "license": "ISC", - "dependencies": { - "libesvm": "0.0.12", - "lodash": "^3.8.0", - "lodash-deep": "^1.6.0", - "mailparser": "^0.5.1", - "moment": "^2.10.3", - "portscanner": "^1.0.0", - "requirefrom": "^0.2.0" - } -} diff --git a/watcher/test/cluster_status.js b/watcher/test/cluster_status.js deleted file mode 100644 index 17e97701144..00000000000 --- a/watcher/test/cluster_status.js +++ /dev/null @@ -1,95 +0,0 @@ -var _ = require('lodash'); -var lib = require('requirefrom')('lib'); -var expect = require('expect.js'); -var moment = require('moment'); -var executeWatcher = lib('execute_watcher'); -var options = { - indexPattern: '[.marvel-]YYYY.MM.DD', - type: 'cluster_stats', - watcher: 'cluster_status' -}; -var testNoExecute = lib('test_no_execute').bind(null, options); -var client = lib('client'); -lib('setup_es'); -lib('setup_smtp_server'); - -describe('Marvel Watchers', function () { - describe('Cluster Status', function () { - - describe('Red for 60 seconds', function () { - var response; - beforeEach(function () { - this.timeout(5000); - var workingDate = moment.utc(); - var rawData = _.times(12, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; - }); - var fixture = { - indexPattern: '[.marvel-]YYYY.MM.DD', - type: 'cluster_stats', - dateField: 'timestamp', - rawData: rawData - }; - return executeWatcher('cluster_status', fixture).then(function (resp) { - response = resp; - if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' ')); - return resp; - }); - }); - - it('should meet the script condition', function () { - expect(response.state).to.be('executed'); - expect(response.execution_result.condition.script.met).to.be(true); - }); - - it('should send an email', function () { - expect(this.mailbox).to.have.length(1); - var message = this.mailbox[0]; - expect(message.subject).to.contain('Watcher Notification - Cluster has been RED for the last 60 seconds'); - expect(message.text).to.contain('Your cluster has been red for the last 60 seconds.'); - }); - - }); - - testNoExecute('Red for 55 then Yellow for 60 seconds', function () { - var workingDate = moment.utc(); - var rawData = _.times(11, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; - }); - rawData.concat(_.times(12, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; - })); - return rawData; - }); - - testNoExecute('Red for 30 then Yellow for 60 seconds', function () { - var workingDate = moment.utc(); - var rawData = _.times(6, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; - }); - rawData.concat(_.times(12, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; - })); - return rawData; - }); - - testNoExecute('Red for 5 Yellow for 10 Red for 10 Green for 60', function () { - var workingDate = moment.utc(); - var rawData = _.times(1, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; - }); - rawData.concat(_.times(2, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; - })); - rawData.concat(_.times(2, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; - })); - rawData.concat(_.times(12, function () { - return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'green' }; - })); - return rawData; - }); - - }); -}); - diff --git a/watcher/test/cpu_usage.js b/watcher/test/cpu_usage.js deleted file mode 100644 index 2920d7698af..00000000000 --- a/watcher/test/cpu_usage.js +++ /dev/null @@ -1,81 +0,0 @@ -var lib = require('requirefrom')('lib'); -var expect = require('expect.js'); -var moment = require('moment'); -var executeWatcher = lib('execute_watcher'); -var client = lib('client'); -var indexPattern = '[.marvel-]YYYY.MM.DD'; -lib('setup_es'); -lib('setup_smtp_server'); - -describe('Marvel Watchers', function () { - describe('CPU Usage', function () { - - describe('above 75%', function () { - var response; - beforeEach(function () { - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'os.cpu.user'], - ['node-01', 75], - ['node-02', 85], - ['node-03', 60] - ] - }; - return executeWatcher('cpu_usage', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should meet the script condition', function () { - expect(response.state).to.be('executed'); - expect(response.execution_result.condition.script.met).to.be(true); - }); - - it('should send an email with multiple hosts', function () { - expect(this.mailbox).to.have.length(1); - var message = this.mailbox[0]; - expect(message.text).to.contain('"node-01" - CPU Usage is at 75.0%'); - expect(message.text).to.contain('"node-02" - CPU Usage is at 85.0%'); - }); - - }); - - describe('below 75%', function () { - var response; - beforeEach(function () { - var self = this; - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'os.cpu.user'], - ['node-01', 35], - ['node-02', 25], - ['node-03', 10] - ] - }; - return executeWatcher('cpu_usage', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should not send an email', function () { - expect(response.state).to.be('execution_not_needed'); - expect(response.execution_result.condition.script.met).to.be(false); - expect(this.mailbox).to.have.length(0); - }); - - }); - - }); -}); diff --git a/watcher/test/fielddata.js b/watcher/test/fielddata.js deleted file mode 100644 index 0601afdf38e..00000000000 --- a/watcher/test/fielddata.js +++ /dev/null @@ -1,82 +0,0 @@ -var lib = require('requirefrom')('lib'); -var expect = require('expect.js'); -var moment = require('moment'); -var executeWatcher = lib('execute_watcher'); -var client = lib('client'); -var indexPattern = '[.marvel-]YYYY.MM.DD'; -lib('setup_es'); -lib('setup_smtp_server'); - -describe('Marvel Watchers', function () { - describe('File Descriptors', function () { - - describe('above 80%', function () { - var response; - beforeEach(function () { - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'indices.fielddata.memory_size_in_bytes'], - ['node-01', 81000], - ['node-02', 70000], - ['node-03', 90000] - ] - }; - return executeWatcher('fielddata', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should meet the script condition', function () { - expect(response.state).to.be('executed'); - expect(response.execution_result.condition.script.met).to.be(true); - }); - - it('should send an email with multiple hosts', function () { - expect(this.mailbox).to.have.length(1); - var message = this.mailbox[0]; - expect(message.text).to.contain('"node-01" - Fielddata utilization is at 81000.0 bytes (81%)'); - expect(message.text).to.contain('"node-03" - Fielddata utilization is at 90000.0 bytes (90%)'); - }); - - }); - - describe('below 80%', function () { - var response; - beforeEach(function () { - var self = this; - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'indices.fielddata.memory_size_in_bytes'], - ['node-01', 12039], - ['node-02', 54393], - ['node-03', 20302] - ] - }; - return executeWatcher('fielddata', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should not send an email', function () { - expect(response.state).to.be('execution_not_needed'); - expect(response.execution_result.condition.script.met).to.be(false); - expect(this.mailbox).to.have.length(0); - }); - - }); - - }); -}); - diff --git a/watcher/test/file_descriptors.js b/watcher/test/file_descriptors.js deleted file mode 100644 index 6de8a188f80..00000000000 --- a/watcher/test/file_descriptors.js +++ /dev/null @@ -1,82 +0,0 @@ -var lib = require('requirefrom')('lib'); -var expect = require('expect.js'); -var moment = require('moment'); -var executeWatcher = lib('execute_watcher'); -var client = lib('client'); -var indexPattern = '[.marvel-]YYYY.MM.DD'; -lib('setup_es'); -lib('setup_smtp_server'); - -describe('Marvel Watchers', function () { - describe('File Descriptors', function () { - - describe('above 80%', function () { - var response; - beforeEach(function () { - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'process.open_file_descriptors'], - ['node-01', Math.round(65535*0.75)], - ['node-02', Math.round(65535*0.81)], - ['node-03', Math.round(65535*0.93)] - ] - }; - return executeWatcher('file_descriptors', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should meet the script condition', function () { - expect(response.state).to.be('executed'); - expect(response.execution_result.condition.script.met).to.be(true); - }); - - it('should send an email with multiple hosts', function () { - expect(this.mailbox).to.have.length(1); - var message = this.mailbox[0]; - expect(message.text).to.contain('"node-02" - File Descriptors is at ' + Math.round(65535*0.81) + '.0 ('+ Math.round(((65535*0.81)/65535)*100) + '%)'); - expect(message.text).to.contain('"node-03" - File Descriptors is at ' + Math.round(65535*0.93) + '.0 ('+ Math.round(((65535*0.93)/65535)*100) + '%)'); - }); - - }); - - describe('below 80%', function () { - var response; - beforeEach(function () { - var self = this; - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'process.open_file_descriptors'], - ['node-01', Math.round(65535*0.05)], - ['node-02', Math.round(65535*0.30)], - ['node-03', Math.round(65535*0.23)] - ] - }; - return executeWatcher('file_descriptors', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should not send an email', function () { - expect(response.state).to.be('execution_not_needed'); - expect(response.execution_result.condition.script.met).to.be(false); - expect(this.mailbox).to.have.length(0); - }); - - }); - - }); -}); - diff --git a/watcher/test/heap_used.js b/watcher/test/heap_used.js deleted file mode 100644 index ee5b444088d..00000000000 --- a/watcher/test/heap_used.js +++ /dev/null @@ -1,81 +0,0 @@ -var lib = require('requirefrom')('lib'); -var expect = require('expect.js'); -var moment = require('moment'); -var executeWatcher = lib('execute_watcher'); -var client = lib('client'); -var indexPattern = '[.marvel-]YYYY.MM.DD'; -lib('setup_es'); -lib('setup_smtp_server'); - -describe('Marvel Watchers', function () { - describe('Memory Usage', function () { - - describe('above 75%', function () { - var response; - beforeEach(function () { - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'jvm.mem.heap_used_percent'], - ['node-01', 75], - ['node-02', 85], - ['node-03', 60] - ] - }; - return executeWatcher('heap_used', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should meet the script condition', function () { - expect(response.state).to.be('executed'); - expect(response.execution_result.condition.script.met).to.be(true); - }); - - it('should send an email with multiple hosts', function () { - expect(this.mailbox).to.have.length(1); - var message = this.mailbox[0]; - expect(message.text).to.contain('"node-01" - Memory Usage is at 75.0%'); - expect(message.text).to.contain('"node-02" - Memory Usage is at 85.0%'); - }); - - }); - - describe('below 75%', function () { - var response; - beforeEach(function () { - var self = this; - this.timeout(5000); - var fixture = { - indexPattern: indexPattern, - type: 'node_stats', - duration: moment.duration(5, 's'), - startDate: moment.utc().subtract(5, 'm'), - data: [ - ['node.name', 'jvm.mem.heap_used_percent'], - ['node-01', 35], - ['node-02', 25], - ['node-03', 10] - ] - }; - return executeWatcher('heap_used', fixture).then(function (resp) { - response = resp; - return resp; - }); - }); - - it('should not send an email', function () { - expect(response.state).to.be('execution_not_needed'); - expect(response.execution_result.condition.script.met).to.be(false); - expect(this.mailbox).to.have.length(0); - }); - - }); - - }); -}); diff --git a/watcher/test/mocha.opts b/watcher/test/mocha.opts deleted file mode 100644 index ec71a5114b0..00000000000 --- a/watcher/test/mocha.opts +++ /dev/null @@ -1,2 +0,0 @@ ---require expect.js ---reporter spec diff --git a/watcher/watches/cluster_status.json b/watcher/watches/cluster_status.json deleted file mode 100644 index 74ec231e69a..00000000000 --- a/watcher/watches/cluster_status.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "trigger": { - "schedule": { - "interval": "1m" - } - }, - "input": { - "search": { - "request": { - "indices": ".marvel-*", - "types": "cluster_stats", - "body": { - "query": { - "filtered": { - "filter": { - "bool": { - "must": [ - { - "range": { - "timestamp": { - "gte": "now-2m", - "lte": "now" - } - } - } - ], - "should": [ - { - "term": { - "status.raw": "red" - } - }, - { - "term": { - "status.raw": "green" - } - }, - { - "term": { - "status.raw": "yellow" - } - } - ] - } - } - } - }, - "fields": ["timestamp","status"], - "sort": [ - { - "timestamp": { - "order": "desc" - } - } - ], - "size": 1, - "aggs": { - "minutes": { - "date_histogram": { - "field": "timestamp", - "interval": "5s" - }, - "aggs": { - "status": { - "terms": { - "field": "status.raw", - "size": 3 - } - } - } - } - } - } - } - } - }, - "throttle_period": "30m", - "condition": { - "script": { - "inline": "if (ctx.payload.hits.total < 1) return false; def rows = ctx.payload.hits.hits; if (rows[0].fields.status[0] != 'red') return false; if (ctx.payload.aggregations.minutes.buckets.size() < 12) return false; def last60Seconds = ctx.payload.aggregations.minutes.buckets[-12..-1]; return last60Seconds.every { it.status.buckets.every { s -> s.key == 'red' } }" - } - }, - "actions": { - "send_email": { - "email": { - "to": "user@example.com", - "subject": "Watcher Notification - Cluster has been RED for the last 60 seconds", - "body": "Your cluster has been red for the last 60 seconds." - } - } - } -} diff --git a/watcher/watches/cpu_usage.json b/watcher/watches/cpu_usage.json deleted file mode 100644 index f2a7422f948..00000000000 --- a/watcher/watches/cpu_usage.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "trigger": { - "schedule": { - "interval": "1m" - } - }, - "input": { - "search": { - "request": { - "indices": [ - ".marvel-*" - ], - "search_type": "count", - "body": { - "query": { - "filtered": { - "filter": { - "range": { - "timestamp": { - "gte": "now-2m", - "lte": "now" - } - } - } - } - }, - "aggs": { - "minutes": { - "date_histogram": { - "field": "timestamp", - "interval": "minute" - }, - "aggs": { - "nodes": { - "terms": { - "field": "node.name.raw", - "size": 10, - "order": { - "cpu": "desc" - } - }, - "aggs": { - "cpu": { - "avg": { - "field": "os.cpu.user" - } - } - } - } - } - } - } - } - } - } - }, - "throttle_period": "30m", - "condition": { - "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.cpu && node.cpu.value >= 75;" - }, - "actions": { - "send_email": { - "transform": { - "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.cpu && it.cpu.value >= 75 };" - }, - "email": { - "to": "user@example.com", - "subject": "Watcher Notification - HIGH CPU USAGE", - "body": "Nodes with HIGH CPU Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - CPU Usage is at {{cpu.value}}%\n{{/ctx.payload._value}}" - } - } - } -} diff --git a/watcher/watches/fielddata.json b/watcher/watches/fielddata.json deleted file mode 100644 index 854be35cbfc..00000000000 --- a/watcher/watches/fielddata.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "metadata": { - "fielddata_cache_size": 100000, - "threshold": 0.8 - }, - "trigger": { - "schedule": { - "interval": "1m" - } - }, - "input": { - "search": { - "request": { - "indices": [ - ".marvel-*" - ], - "types": "node_stats", - "search_type": "count", - "body": { - "query": { - "filtered": { - "filter": { - "range": { - "timestamp": { - "gte": "now-1m", - "lte": "now" - } - } - } - } - }, - "aggs": { - "minutes": { - "date_histogram": { - "field": "timestamp", - "interval": "5s" - }, - "aggs": { - "nodes": { - "terms": { - "field": "node.name.raw", - "size": 10, - "order": { - "fielddata": "desc" - } - }, - "aggs": { - "fielddata": { - "avg": { - "field": "indices.fielddata.memory_size_in_bytes" - } - } - } - } - } - } - } - } - } - } - }, - "throttle_period": "30m", - "condition": { - "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fielddata && node.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold);" - }, - "actions": { - "send_email": { - "transform": { - "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fielddata && it.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold) }).collect({ it.fielddata.percent = Math.round((it.fielddata.value/ctx.metadata.fielddata_cache_size)*100); it });" - }, - "email": { - "to": "user@example.com", - "subject": "Watcher Notification - NODES WITH 80% FIELDDATA UTILIZATION", - "body": "Nodes with 80% FIELDDATA UTILIZATION (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Fielddata utilization is at {{fielddata.value}} bytes ({{fielddata.percent}}%)\n{{/ctx.payload._value}}" - } - } - } -} - diff --git a/watcher/watches/file_descriptors.json b/watcher/watches/file_descriptors.json deleted file mode 100644 index bfd5fb48ccc..00000000000 --- a/watcher/watches/file_descriptors.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "metadata": { - "system_fd": 65535, - "threshold": 0.8 - }, - "trigger": { - "schedule": { - "interval": "1m" - } - }, - "input": { - "search": { - "request": { - "indices": [ - ".marvel-*" - ], - "types": "node_stats", - "search_type": "count", - "body": { - "query": { - "filtered": { - "filter": { - "range": { - "timestamp": { - "gte": "now-1m", - "lte": "now" - } - } - } - } - }, - "aggs": { - "minutes": { - "date_histogram": { - "field": "timestamp", - "interval": "5s" - }, - "aggs": { - "nodes": { - "terms": { - "field": "node.name.raw", - "size": 10, - "order": { - "fd": "desc" - } - }, - "aggs": { - "fd": { - "avg": { - "field": "process.open_file_descriptors" - } - } - } - } - } - } - } - } - } - } - }, - "throttle_period": "30m", - "condition": { - "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fd && node.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold);" - }, - "actions": { - "send_email": { - "transform": { - "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fd && it.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold) }).collect({ it.fd.percent = Math.round((it.fd.value/ctx.metadata.system_fd)*100); it });" - }, - "email": { - "to": "user@example.com", - "subject": "Watcher Notification - NODES WITH 80% FILE DESCRIPTORS USED", - "body": "Nodes with 80% FILE DESCRIPTORS USED (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - File Descriptors is at {{fd.value}} ({{fd.percent}}%)\n{{/ctx.payload._value}}" - } - } - } -} - diff --git a/watcher/watches/heap_used.json b/watcher/watches/heap_used.json deleted file mode 100644 index ba92437475b..00000000000 --- a/watcher/watches/heap_used.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "trigger": { - "schedule": { - "interval": "1m" - } - }, - "input": { - "search": { - "request": { - "indices": [ - ".marvel-*" - ], - "search_type": "count", - "body": { - "query": { - "filtered": { - "filter": { - "range": { - "timestamp": { - "gte": "now-2m", - "lte": "now" - } - } - } - } - }, - "aggs": { - "minutes": { - "date_histogram": { - "field": "timestamp", - "interval": "minute" - }, - "aggs": { - "nodes": { - "terms": { - "field": "node.name.raw", - "size": 10, - "order": { - "memory": "desc" - } - }, - "aggs": { - "memory": { - "avg": { - "field": "jvm.mem.heap_used_percent" - } - } - } - } - } - } - } - } - } - } - }, - "throttle_period": "30m", - "condition": { - "script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.memory && node.memory.value >= 75;" - }, - "actions": { - "send_email": { - "transform": { - "script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.memory && it.memory.value >= 75 };" - }, - "email": { - "to": "user@example.com", - "subject": "Watcher Notification - HIGH MEMORY USAGE", - "body": "Nodes with HIGH MEMORY Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Memory Usage is at {{memory.value}}%\n{{/ctx.payload._value}}" - } - } - } -}