From 953597e1c084da157b14289dcbc4552bea4e83fd Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Mon, 27 Jul 2015 17:45:09 -0700 Subject: [PATCH 1/4] Triggering an update Original commit: elastic/x-pack-elasticsearch@3b03b5ec1cb5e436a7e868713a8c34bc5f0a9100 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8f749d1d0ce..840ffd4ea8b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # WARNING THIS IS ALL IN FLUX + # Installing the last release of Marvel The easiest way to play/get to know Marvel is to install the latest release version of it. To do so, just run the following command on every node on your cluster (restart node for it to have effect): From a779ac8100caaf5858652e3deff4977229656ff9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 30 Jul 2015 12:49:04 -0700 Subject: [PATCH 2/4] Adding indexer and optimizing shard allocations Original commit: elastic/x-pack-elasticsearch@146fbec5750e7d35356ce97eb2474f143cea6fee --- marvel_index_template.json | 129 +++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 marvel_index_template.json diff --git a/marvel_index_template.json b/marvel_index_template.json new file mode 100644 index 00000000000..2834e940481 --- /dev/null +++ b/marvel_index_template.json @@ -0,0 +1,129 @@ +{ + "template": ".marvel*", + "settings": { + "marvel.index_format": 7, + "marvel.version": "2.0.0", + "index.number_of_shards": 1, + "index.number_of_replicas": 1, + "index.codec": "best_compression", + "index.mapper.dynamic": false + }, + "mappings": { + "_default_": { + "_all": { + "enabled": false + }, + "date_detection": false, + "properties": { + "cluster_name": { + "type": "string", + "index": "not_analyzed" + }, + "timestamp": { + "type": "date", + "format": "date_time" + } + } + }, + "marvel_index_stats": { + "properties": { + "index_stats": { + "properties": { + "index": { + "type": "string", + "index": "not_analyzed" + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "throttle_time_in_millis": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "marvel_cluster_stats": { + "properties": { + "cluster_stats": { + "properties": { + "nodes": { + "type": "object" + }, + "indices": { + "type": "object" + } + } + } + } + }, + "marvel_cluster_state": { + "properties": { + "cluster_state": { + "properties": { + "version": { "type": "long" }, + "nodes": { "type": "object" }, + "master_node": { "type": "string", "index": "not_analyzed" }, + "shards": { "type": "object" } + } + } + } + }, + "marvel_node_stats": { + "properties": { + "node_id": { + "type": "string", + "index": "not_analyzed" + }, + "node_master": { + "type": "boolean" + }, + "mlockall": { + "type": "boolean" + }, + "disk_threshold_enabled": { + "type": "boolean" + }, + "disk_threshold_watermark_high": { + "type": "short" + }, + "node_stats": { + "type": "object" + } + } + }, + "marvel_index_recovery": { + "properties": { + "index_name": { + "type": "string", + "index": "not_analyzed" + }, + "shards": { + "type": "object" + } + } + } + } +} From 86ebd14587d0539818d31eb9e81d12fecacccc9d Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Tue, 4 Aug 2015 10:41:44 -0700 Subject: [PATCH 3/4] Movign to the new Kibana Original commit: elastic/x-pack-elasticsearch@a943185e94c8cfac5de6100ec140683115b7be32 --- .bowerrc | 4 ---- watcher/lib/inject_stats.js | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 .bowerrc diff --git a/.bowerrc b/.bowerrc deleted file mode 100644 index 1d69e778685..00000000000 --- a/.bowerrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "directory": "./public/bower_components" -} - diff --git a/watcher/lib/inject_stats.js b/watcher/lib/inject_stats.js index e14ec726915..c81e85c7270 100644 --- a/watcher/lib/inject_stats.js +++ b/watcher/lib/inject_stats.js @@ -1,5 +1,4 @@ var _ = require('lodash'); -_.mixin(require('lodash-deep')); var moment = require('moment'); var client = require('./client'); @@ -21,7 +20,7 @@ module.exports = function (fixture) { var index = workingDate.format(indexPattern); var entry = { '@timestamp': workingDate.toISOString() }; row.forEach(function (val, index) { - _.deepSet(entry, fields[index], val); + _.set(entry, fields[index], val); }); indices.push(index); @@ -40,7 +39,7 @@ module.exports = function (fixture) { var index = moment.utc(row[dateField]).format(indexPattern); var entry = {}; _.each(row, function (val, key) { - _.deepSet(entry, key, val); + _.set(entry, key, val); }); indices.push(index); body.push({ From c21ca69acd81b6454d7a1b1b563f7155f14e49f9 Mon Sep 17 00:00:00 2001 From: Chris Cowan Date: Thu, 6 Aug 2015 15:30:05 -0700 Subject: [PATCH 4/4] Moving everythign to the new data format Original commit: elastic/x-pack-elasticsearch@b35c192536bc793d79083a30fde7ae8f30e29b6a --- .agignore | 1 - watcher/lib/inject_stats.js | 2 +- watcher/lib/test_no_execute.js | 2 +- watcher/test/cluster_status.js | 20 ++++++++++---------- watcher/watches/cluster_status.json | 8 ++++---- watcher/watches/cpu_usage.json | 4 ++-- watcher/watches/fielddata.json | 4 ++-- watcher/watches/file_descriptors.json | 4 ++-- watcher/watches/heap_used.json | 4 ++-- 9 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.agignore b/.agignore index 0847b65d054..10fb4038cbd 100644 --- a/.agignore +++ b/.agignore @@ -1,4 +1,3 @@ -public agent node_modules bower_components diff --git a/watcher/lib/inject_stats.js b/watcher/lib/inject_stats.js index c81e85c7270..9a2b1cb6b96 100644 --- a/watcher/lib/inject_stats.js +++ b/watcher/lib/inject_stats.js @@ -18,7 +18,7 @@ module.exports = function (fixture) { function createEntries(row) { var index = workingDate.format(indexPattern); - var entry = { '@timestamp': workingDate.toISOString() }; + var entry = { 'timestamp': workingDate.toISOString() }; row.forEach(function (val, index) { _.set(entry, fields[index], val); }); diff --git a/watcher/lib/test_no_execute.js b/watcher/lib/test_no_execute.js index 7c94d531487..c90fe915a67 100644 --- a/watcher/lib/test_no_execute.js +++ b/watcher/lib/test_no_execute.js @@ -10,7 +10,7 @@ module.exports = function testNoExecute(options, message, generateRawData) { var fixture = { indexPattern: options.indexPattern, type: options.type, - dateField: '@timestamp', + dateField: 'timestamp', rawData: rawData }; return executeWatcher(options.watcher, fixture).then(function (resp) { diff --git a/watcher/test/cluster_status.js b/watcher/test/cluster_status.js index 91eeb774c92..17e97701144 100644 --- a/watcher/test/cluster_status.js +++ b/watcher/test/cluster_status.js @@ -22,12 +22,12 @@ describe('Marvel Watchers', function () { this.timeout(5000); var workingDate = moment.utc(); var rawData = _.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); var fixture = { indexPattern: '[.marvel-]YYYY.MM.DD', type: 'cluster_stats', - dateField: '@timestamp', + dateField: 'timestamp', rawData: rawData }; return executeWatcher('cluster_status', fixture).then(function (resp) { @@ -54,10 +54,10 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 55 then Yellow for 60 seconds', function () { var workingDate = moment.utc(); var rawData = _.times(11, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); return rawData; }); @@ -65,10 +65,10 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 30 then Yellow for 60 seconds', function () { var workingDate = moment.utc(); var rawData = _.times(6, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); return rawData; }); @@ -76,16 +76,16 @@ describe('Marvel Watchers', function () { testNoExecute('Red for 5 Yellow for 10 Red for 10 Green for 60', function () { var workingDate = moment.utc(); var rawData = _.times(1, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; }); rawData.concat(_.times(2, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' }; })); rawData.concat(_.times(2, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' }; })); rawData.concat(_.times(12, function () { - return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'green' }; + return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'green' }; })); return rawData; }); diff --git a/watcher/watches/cluster_status.json b/watcher/watches/cluster_status.json index 52556996e73..74ec231e69a 100644 --- a/watcher/watches/cluster_status.json +++ b/watcher/watches/cluster_status.json @@ -17,7 +17,7 @@ "must": [ { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -45,10 +45,10 @@ } } }, - "fields": ["@timestamp","status"], + "fields": ["timestamp","status"], "sort": [ { - "@timestamp": { + "timestamp": { "order": "desc" } } @@ -57,7 +57,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/cpu_usage.json b/watcher/watches/cpu_usage.json index edda8624f64..f2a7422f948 100644 --- a/watcher/watches/cpu_usage.json +++ b/watcher/watches/cpu_usage.json @@ -16,7 +16,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -27,7 +27,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "minute" }, "aggs": { diff --git a/watcher/watches/fielddata.json b/watcher/watches/fielddata.json index 08072f35ebf..854be35cbfc 100644 --- a/watcher/watches/fielddata.json +++ b/watcher/watches/fielddata.json @@ -21,7 +21,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-1m", "lte": "now" } @@ -32,7 +32,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/file_descriptors.json b/watcher/watches/file_descriptors.json index 83fbe122aca..bfd5fb48ccc 100644 --- a/watcher/watches/file_descriptors.json +++ b/watcher/watches/file_descriptors.json @@ -21,7 +21,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-1m", "lte": "now" } @@ -32,7 +32,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "5s" }, "aggs": { diff --git a/watcher/watches/heap_used.json b/watcher/watches/heap_used.json index 3f33719a9e8..ba92437475b 100644 --- a/watcher/watches/heap_used.json +++ b/watcher/watches/heap_used.json @@ -16,7 +16,7 @@ "filtered": { "filter": { "range": { - "@timestamp": { + "timestamp": { "gte": "now-2m", "lte": "now" } @@ -27,7 +27,7 @@ "aggs": { "minutes": { "date_histogram": { - "field": "@timestamp", + "field": "timestamp", "interval": "minute" }, "aggs": {