honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-06 19:09:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request elastic/elasticsearch#412 from simianhacker/watchers

Browse Source 
Watchers

Original commit: elastic/x-pack-elasticsearch@0c1ecda21c
This commit is contained in:
Chris Cowan 2015-06-11 15:08:47 -07:00
commit a480d5ab70
23 changed files with 1623 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
watcher/README.md Normal file
View File

@ -0,0 +1,11 @@
# Marvel Watchers
In this directory you will find example Watchers for Marvel that were distributed with the Watcher Beta documentation. The `watches` directory contains the example watches and the `test` directory contains integration test for each watcher.
The testing framework is written in Node.js and use ESVM to create an Elasticsearch instance for running the tests against
### Setup and Running Tests
- Run `npm install` in this directory
- Ensure you have `mocha` installed globally
- Run `mocha` in this directory

4
watcher/lib/client.js Normal file
View File

@ -0,0 +1,4 @@
var Client = require('elasticsearch').Client;
module.exports = new Client({
host: 'http://localhost:9800'
});

11
watcher/lib/create_template.js Normal file
View File

@ -0,0 +1,11 @@
var client = require('./client');
var template = require('./template.json');
module.exports = function () {
return client.indices.putTemplate({
body: template,
name: 'marvel'
});
};

40
watcher/lib/execute_watcher.js Normal file
View File

@ -0,0 +1,40 @@
var Promise = require('bluebird');
var request = require('request');
var injectStats = require('./inject_stats');
var loadWatcher = require('./load_watcher');
module.exports = function (watcher, fixture) {
return injectStats(fixture).then(function () {
return loadWatcher(watcher);
}).then(function () {
return new Promise(function (resolve, reject) {
var options = {
method: 'POST',
url: 'http://localhost:9800/_watcher/watch/' + watcher + '/_execute',
json: true,
body: {
trigger_event: {
schedule: {
scheduled_time: 'now',
triggered_time: 'now'
}
}
}
};
request(options, function (err, resp, body) {
if (err) return reject(err);
if (resp.statusCode === 200) return resolve(body);
var message = options.url + ' responed with ' + resp.statusCode;
var error = new Error(body.error || message);
error.body = body;
error.resp = resp;
error.code = resp.statusCode;
error.options = options;
reject(error);
});
});
});
};

13
watcher/lib/find_port.js Normal file
View File

@ -0,0 +1,13 @@
var Promise = require('bluebird');
var portscanner = require('portscanner');
module.exports = function findPort(start, end, host) {
host = host || 'localhost';
return new Promise(function (resolve, reject) {
portscanner.findAPortNotInUse(start, end, host, function (err, port) {
if (err) return reject(err);
resolve(port);
});
});
};

74
watcher/lib/inject_stats.js Normal file
View File

@ -0,0 +1,74 @@
var _ = require('lodash');
_.mixin(require('lodash-deep'));
var moment = require('moment');
var client = require('./client');
module.exports = function (fixture) {
var indexPattern = fixture.indexPattern;
var type = fixture.type;
var data = fixture.data;
var rawData = fixture.rawData;
var startDate = fixture.startDate;
var duration = fixture.duration;
var dateField = fixture.dateField;
var workingDate = moment.utc();
var indices = [];
var body = [];
var fields;
function createEntries(row) {
var index = workingDate.format(indexPattern);
var entry = { '@timestamp': workingDate.toISOString() };
row.forEach(function (val, index) {
_.deepSet(entry, fields[index], val);
});
indices.push(index);
body.push({
index: {
_index: index,
_type: type
}
});
body.push(entry);
}
if (rawData) {
rawData.forEach(function (row) {
var index = moment.utc(row[dateField]).format(indexPattern);
var entry = {};
_.each(row, function (val, key) {
_.deepSet(entry, key, val);
});
indices.push(index);
body.push({
index: {
_index: index,
_type: type
}
});
body.push(entry);
});
} else {
fields = data.shift();
while(startDate <= workingDate) {
data.forEach(createEntries);
workingDate.subtract(duration);
}
}
return client.deleteByQuery({
index: _.unique(indices),
ignoreUnavailable: true,
allowNoIndices: true,
q: '*'
})
.then(function (arg) {
return client.bulk({
body: body,
refresh: true
});
});
};

26
watcher/lib/load_watcher.js Normal file
View File

@ -0,0 +1,26 @@
var Promise = require('bluebird');
var request = require('request');
var path = require('path');
module.exports = function (name) {
var watch = require(path.join(__dirname, '..', 'watches', name + '.json'));
var options = {
method: 'PUT',
url: 'http://localhost:9800/_watcher/watch/' + name,
json: true,
body: watch
};
return new Promise(function (resolve, reject) {
request(options, function (err, resp, body) {
if (err) return reject(err);
if (resp.statusCode <= 201) resolve({ resp: resp, body: body });
var message = options.url + ' responded with ' + resp.statusCode;
var error = new Error(body.error || message);
error.body = body;
error.resp = resp;
error.code = resp.statusCode;
error.options = options;
reject(error);
});
});
};

71
watcher/lib/setup_es.js Normal file
View File

@ -0,0 +1,71 @@
var path = require('path');
var Promise = require('bluebird');
var libesvm = require('libesvm');
var createTemplate = require('./create_template');
function startEs() {
var options = {
version: '1.5.2',
directory: path.join(__dirname, '..', 'esvm'),
purge: true,
plugins: [
'elasticsearch/watcher/1.0.0-Beta1-5',
'elasticsearch/license/latest'
],
config: {
'script.groovy.sandbox.enabled': true,
'cluster.name': 'test',
'network.host': '127.0.0.1',
'http.port': 9800,
'watcher.actions.email.service.account': {
'local': {
'email_defaults.from': 'admin@example.com',
'smtp': {
'host': 'localhost',
'user': 'test',
'password': 'test',
'port': 5555
}
}
}
}
};
var cluster = libesvm.createCluster(options);
cluster.on('log', function (log) {
if (log.type === 'progress') return;
if (process.env.DEBUG) console.log('%s %s %s %s', log.level, log.node, log.type, log.message);
});
return cluster.install()
.then(function () {
return cluster.installPlugins();
})
.then(function () {
return cluster.start();
})
.then(function () {
after(function () {
this.timeout(60000);
return cluster.shutdown();
});
return cluster;
});
}
before(function () {
var self = this;
this.timeout(60000);
return new Promise(function (resolve, reject) {
startEs().then(function (cluster) {
self.cluster = cluster;
cluster.on('log', function (log) {
if (/watch service has started/.test(log.message)) {
createTemplate().then(function () {
resolve(cluster);
});
}
});
}).catch(reject);
});
});

56
watcher/lib/setup_smtp_server.js Normal file
View File

@ -0,0 +1,56 @@
var Promise = require('bluebird');
var SMTPServer = require('smtp-server').SMTPServer;
var MailParser = require('mailparser').MailParser;
var acceptAny = function (address, session, done) {
return done();
};
var mailbox = [];
function startSMTP() {
return new Promise(function (resolve, reject) {
var server = new SMTPServer({
logger: false,
disabledCommands: ['STARTTLS'],
onAuth: function (auth, session, done) {
done(null, { user: 1 });
},
onMailFrom: acceptAny,
onRcptTo: acceptAny,
onData: function (stream, session, done) {
var mailparser = new MailParser();
mailparser.on('end', function (mailObj) {
mailbox.push(mailObj);
done();
});
stream.pipe(mailparser);
}
});
server.listen(5555, function (err) {
if (err) return reject(err);
after(function (done) {
server.close(done);
});
resolve(server);
});
});
}
before(function () {
var self = this;
return startSMTP().then(function (server) {
this.smtp = server;
return server;
});
});
beforeEach(function () {
this.mailbox = mailbox;
});
afterEach(function () {
mailbox = [];
});
module.exports = mailbox;

447
watcher/lib/template.json Normal file
View File

@ -0,0 +1,447 @@
{
"template": ".marvel*",
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"analysis": {
"analyzer": {
"default": {
"type": "standard",
"stopwords": "_none_"
}
}
},
"mapper.dynamic": true,
"marvel.index_format": 6
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "multi_field",
"fields": {
"{name}": {
"type": "string",
"index": "analyzed",
"omit_norms": true
},
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
}
}
}
]
},
"node_stats": {
"properties": {
"breakers": {
"properties": {
"fielddata": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"request": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"parent": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
}
}
},
"fs": {
"properties": {
"total": {
"properties": {
"disk_io_op": {
"type": "long"
},
"disk_reads": {
"type": "long"
},
"disk_writes": {
"type": "long"
},
"disk_io_size_in_bytes": {
"type": "long"
},
"disk_read_size_in_bytes": {
"type": "long"
},
"disk_write_size_in_bytes": {
"type": "long"
}
}
}
}
},
"jvm": {
"properties": {
"buffer_pools": {
"properties": {
"direct": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
},
"mapped": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
}
}
},
"gc": {
"properties": {
"collectors": {
"properties": {
"young": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
},
"old": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
},
"indices": {
"properties": {
"indexing": {
"properties": {
"throttle_time_in_millis": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"os": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
},
"thread_pool": {
"properties": {
"listener": {
"properties": {
"threads": {
"type": "long"
},
"rejected": {
"type": "long"
},
"completed": {
"type": "long"
},
"queue": {
"type": "long"
},
"largest": {
"type": "long"
}
}
}
}
}
}
},
"index_stats": {
"properties": {
"index": {
"type": "multi_field",
"fields": {
"index": {
"type": "string",
"norms": {
"enabled": false
}
},
"raw": {
"type": "string",
"index": "not_analyzed",
"norms": {
"enabled": false
},
"index_options": "docs",
"include_in_all": false,
"ignore_above": 256
}
}
},
"total": {
"properties": {
"fielddata": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"throttle_time_in_millis": {
"type": "long"
}
}
},
"merges": {
"properties": {
"total_size_in_bytes": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"search": {
"properties": {
"query": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_event": {},
"shard_event": {},
"indices_stats": {
"properties": {
"primaries": {
"properties": {
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
},
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"search": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_stats": {},
"index_event": {},
"node_event": {},
"routing_event": {},
"cluster_state": {
"properties": {
"blocks": {
"type": "object",
"enabled": false
},
"nodes": {
"type": "object",
"enabled": false
},
"routing_nodes": {
"type": "object",
"enabled": false
},
"routing_table": {
"type": "object",
"enabled": false
}
}
}
}
}

28
watcher/lib/test_no_execute.js Normal file
View File

@ -0,0 +1,28 @@
var lib = require('requirefrom')('lib');
var executeWatcher = lib('execute_watcher');
var expect = require('expect.js');
module.exports = function testNoExecute(options, message, generateRawData) {
describe(message, function () {
var response;
beforeEach(function () {
this.timeout(5000);
var rawData = generateRawData();
var fixture = {
indexPattern: options.indexPattern,
type: options.type,
dateField: '@timestamp',
rawData: rawData
};
return executeWatcher(options.watcher, fixture).then(function (resp) {
response = resp;
if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' '));
return resp;
});
});
it('should not meet the script condition', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
});
});
};

23
watcher/package.json Normal file
View File

@ -0,0 +1,23 @@
{
"name": "marvel-watchers",
"version": "1.0.0",
"description": "",
"main": "index.js",
"directories": {
"test": "test"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"libesvm": "0.0.12",
"lodash": "^3.8.0",
"lodash-deep": "^1.6.0",
"mailparser": "^0.5.1",
"moment": "^2.10.3",
"portscanner": "^1.0.0",
"requirefrom": "^0.2.0"
}
}

95
watcher/test/cluster_status.js Normal file
View File

@ -0,0 +1,95 @@
var _ = require('lodash');
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var options = {
indexPattern: '[.marvel-]YYYY.MM.DD',
type: 'cluster_stats',
watcher: 'cluster_status'
};
var testNoExecute = lib('test_no_execute').bind(null, options);
var client = lib('client');
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('Cluster Status', function () {
describe('Red for 60 seconds', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var workingDate = moment.utc();
var rawData = _.times(12, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
var fixture = {
indexPattern: '[.marvel-]YYYY.MM.DD',
type: 'cluster_stats',
dateField: '@timestamp',
rawData: rawData
};
return executeWatcher('cluster_status', fixture).then(function (resp) {
response = resp;
if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' '));
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.subject).to.contain('Watcher Notification - Cluster has been RED for the last 60 seconds');
expect(message.text).to.contain('Your cluster has been red for the last 60 seconds.');
});
});
testNoExecute('Red for 55 then Yellow for 60 seconds', function () {
var workingDate = moment.utc();
var rawData = _.times(11, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(12, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
return rawData;
});
testNoExecute('Red for 30 then Yellow for 60 seconds', function () {
var workingDate = moment.utc();
var rawData = _.times(6, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(12, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
return rawData;
});
testNoExecute('Red for 5 Yellow for 10 Red for 10 Green for 60', function () {
var workingDate = moment.utc();
var rawData = _.times(1, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(2, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
rawData.concat(_.times(2, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
}));
rawData.concat(_.times(12, function () {
return { '@timestamp': workingDate.subtract(5, 's').format(), status: 'green' };
}));
return rawData;
});
});
});

81
watcher/test/cpu_usage.js Normal file
View File

@ -0,0 +1,81 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('CPU Usage', function () {
describe('above 75%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'os.cpu.user'],
['node-01', 75],
['node-02', 85],
['node-03', 60]
]
};
return executeWatcher('cpu_usage', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - CPU Usage is at 75.0%');
expect(message.text).to.contain('"node-02" - CPU Usage is at 85.0%');
});
});
describe('below 75%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'os.cpu.user'],
['node-01', 35],
['node-02', 25],
['node-03', 10]
]
};
return executeWatcher('cpu_usage', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

82
watcher/test/fielddata.js Normal file
View File

@ -0,0 +1,82 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('File Descriptors', function () {
describe('above 80%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'indices.fielddata.memory_size_in_bytes'],
['node-01', 81000],
['node-02', 70000],
['node-03', 90000]
]
};
return executeWatcher('fielddata', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - Fielddata utilization is at 81000.0 bytes (81%)');
expect(message.text).to.contain('"node-03" - Fielddata utilization is at 90000.0 bytes (90%)');
});
});
describe('below 80%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'indices.fielddata.memory_size_in_bytes'],
['node-01', 12039],
['node-02', 54393],
['node-03', 20302]
]
};
return executeWatcher('fielddata', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

82
watcher/test/file_descriptors.js Normal file
View File

@ -0,0 +1,82 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('File Descriptors', function () {
describe('above 80%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'process.open_file_descriptors'],
['node-01', Math.round(65535*0.75)],
['node-02', Math.round(65535*0.81)],
['node-03', Math.round(65535*0.93)]
]
};
return executeWatcher('file_descriptors', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-02" - File Descriptors is at ' + Math.round(65535*0.81) + '.0 ('+ Math.round(((65535*0.81)/65535)*100) + '%)');
expect(message.text).to.contain('"node-03" - File Descriptors is at ' + Math.round(65535*0.93) + '.0 ('+ Math.round(((65535*0.93)/65535)*100) + '%)');
});
});
describe('below 80%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'process.open_file_descriptors'],
['node-01', Math.round(65535*0.05)],
['node-02', Math.round(65535*0.30)],
['node-03', Math.round(65535*0.23)]
]
};
return executeWatcher('file_descriptors', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

81
watcher/test/heap_used.js Normal file
View File

@ -0,0 +1,81 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('Memory Usage', function () {
describe('above 75%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'jvm.mem.heap_used_percent'],
['node-01', 75],
['node-02', 85],
['node-03', 60]
]
};
return executeWatcher('heap_used', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - Memory Usage is at 75.0%');
expect(message.text).to.contain('"node-02" - Memory Usage is at 85.0%');
});
});
describe('below 75%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'jvm.mem.heap_used_percent'],
['node-01', 35],
['node-02', 25],
['node-03', 10]
]
};
return executeWatcher('heap_used', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

2
watcher/test/mocha.opts Normal file
View File

@ -0,0 +1,2 @@
--require expect.js
--reporter spec

92
watcher/watches/cluster_status.json Normal file
View File

@ -0,0 +1,92 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": ".marvel-*",
"types": "cluster_stats",
"body": {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"range": {
"@timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
],
"should": [
{
"term": {
"status.raw": "red"
}
},
{
"term": {
"status.raw": "green"
}
},
{
"term": {
"status.raw": "yellow"
}
}
]
}
}
}
},
"fields": ["@timestamp","status"],
"sort": [
{
"@timestamp": {
"order": "desc"
}
}
],
"size": 1,
"aggs": {
"minutes": {
"date_histogram": {
"field": "@timestamp",
"interval": "5s"
},
"aggs": {
"status": {
"terms": {
"field": "status.raw",
"size": 3
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": {
"inline": "if (ctx.payload.hits.total < 1) return false; def rows = ctx.payload.hits.hits; if (rows[0].fields.status[0] != 'red') return false; if (ctx.payload.aggregations.minutes.buckets.size() < 12) return false; def last60Seconds = ctx.payload.aggregations.minutes.buckets[-12..-1]; return last60Seconds.every { it.status.buckets.every { s -> s.key == 'red' } }"
}
},
"actions": {
"send_email": {
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - Cluster has been RED for the last 60 seconds",
"body": "Your cluster has been red for the last 60 seconds."
}
}
}
}

73
watcher/watches/cpu_usage.json Normal file
View File

@ -0,0 +1,73 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"@timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "@timestamp",
"interval": "minute"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"cpu": "desc"
}
},
"aggs": {
"cpu": {
"avg": {
"field": "os.cpu.user"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.cpu && node.cpu.value >= 75;"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.cpu && it.cpu.value >= 75 };"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - HIGH CPU USAGE",
"body": "Nodes with HIGH CPU Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - CPU Usage is at {{cpu.value}}%\n{{/ctx.payload._value}}"
}
}
}
}

79
watcher/watches/fielddata.json Normal file
View File

@ -0,0 +1,79 @@
{
"metadata": {
"fielddata_cache_size": 100000,
"threshold": 0.8
},
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"types": "node_stats",
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"@timestamp": {
"gte": "now-1m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "@timestamp",
"interval": "5s"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"fielddata": "desc"
}
},
"aggs": {
"fielddata": {
"avg": {
"field": "indices.fielddata.memory_size_in_bytes"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fielddata && node.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold);"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fielddata && it.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold) }).collect({ it.fielddata.percent = Math.round((it.fielddata.value/ctx.metadata.fielddata_cache_size)*100); it });"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - NODES WITH 80% FIELDDATA UTILIZATION",
"body": "Nodes with 80% FIELDDATA UTILIZATION (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Fielddata utilization is at {{fielddata.value}} bytes ({{fielddata.percent}}%)\n{{/ctx.payload._value}}"
}
}
}
}

79
watcher/watches/file_descriptors.json Normal file
View File

@ -0,0 +1,79 @@
{
"metadata": {
"system_fd": 65535,
"threshold": 0.8
},
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"types": "node_stats",
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"@timestamp": {
"gte": "now-1m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "@timestamp",
"interval": "5s"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"fd": "desc"
}
},
"aggs": {
"fd": {
"avg": {
"field": "process.open_file_descriptors"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fd && node.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold);"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fd && it.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold) }).collect({ it.fd.percent = Math.round((it.fd.value/ctx.metadata.system_fd)*100); it });"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - NODES WITH 80% FILE DESCRIPTORS USED",
"body": "Nodes with 80% FILE DESCRIPTORS USED (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - File Descriptors is at {{fd.value}} ({{fd.percent}}%)\n{{/ctx.payload._value}}"
}
}
}
}

73
watcher/watches/heap_used.json Normal file
View File

@ -0,0 +1,73 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"@timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "@timestamp",
"interval": "minute"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"memory": "desc"
}
},
"aggs": {
"memory": {
"avg": {
"field": "jvm.mem.heap_used_percent"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.memory && node.memory.value >= 75;"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.memory && it.memory.value >= 75 };"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - HIGH MEMORY USAGE",
"body": "Nodes with HIGH MEMORY Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Memory Usage is at {{memory.value}}%\n{{/ctx.payload._value}}"
}
}
}
}