move public code to kibana/marvel

Original commit: elastic/x-pack-elasticsearch@13d409da5d
This commit is contained in:
Timothy Sullivan 2015-12-06 00:51:20 -07:00
parent d022e8d276
commit d3fb668346
34 changed files with 0 additions and 2140 deletions

View File

@ -1,3 +0,0 @@
agent
node_modules
bower_components

View File

@ -1,71 +0,0 @@
---
parser: babel-eslint
plugins: [ mocha ]
env:
es6: true
amd: true
node: true
mocha: true
rules:
block-scoped-var: 2
camelcase: [ 2, { properties: never } ]
comma-dangle: 0
comma-style: [ 2, last ]
consistent-return: 0
curly: [ 2, multi-line ]
dot-location: [ 2, property ]
dot-notation: [ 2, { allowKeywords: true } ]
eqeqeq: [ 2, allow-null ]
guard-for-in: 2
indent: [ 2, 2, { SwitchCase: 1 } ]
key-spacing: [ 0, { align: value } ]
max-len: [ 2, 140, 2, { ignoreComments: true, ignoreUrls: true } ]
new-cap: [ 2, { capIsNewExceptions: [ Private ] } ]
no-bitwise: 0
no-caller: 2
no-cond-assign: 0
no-debugger: 2
no-empty: 2
no-eval: 2
no-extend-native: 2
no-extra-parens: 0
no-irregular-whitespace: 2
no-iterator: 2
no-loop-func: 2
no-multi-spaces: 0
no-multi-str: 2
no-nested-ternary: 2
no-new: 0
no-path-concat: 0
no-proto: 2
no-return-assign: 0
no-script-url: 2
no-sequences: 2
no-shadow: 0
no-trailing-spaces: 2
no-undef: 2
no-underscore-dangle: 0
no-unused-expressions: 0
no-unused-vars: 0
no-use-before-define: [ 2, nofunc ]
no-with: 2
one-var: [ 2, never ]
quotes: [ 2, single ]
semi-spacing: [ 2, { before: false, after: true } ]
semi: [ 2, always ]
space-after-keywords: [ 2, always ]
space-before-blocks: [ 2, always ]
space-before-function-paren: [ 2, { anonymous: always, named: never } ]
space-in-parens: [ 2, never ]
space-infix-ops: [ 2, { int32Hint: false } ]
space-return-throw-case: [ 2 ]
space-unary-ops: [ 2 ]
strict: [ 2, never ]
valid-typeof: 2
wrap-iife: [ 2, outside ]
yoda: 0
mocha/no-exclusive-tests: 2
mocha/handle-done-callback: 2

59
.jscsrc
View File

@ -1,59 +0,0 @@
{
"maximumLineLength": {
"value": 140,
"allowComments": true
},
"requireCurlyBraces": [
"for",
"do",
"try",
"catch"
],
"requireSpaceAfterKeywords": [
"if",
"else",
"for",
"while",
"do",
"switch",
"case",
"return",
"try",
"catch",
"function",
"typeof"
],
"requireSpaceBeforeBlockStatements": true,
"requireParenthesesAroundIIFE": true,
"requireSpacesInConditionalExpression": true,
"disallowSpacesInNamedFunctionExpression": {
"beforeOpeningRoundBrace": true
},
"disallowSpacesInFunctionDeclaration": {
"beforeOpeningRoundBrace": true
},
"requireSpaceBetweenArguments": true,
"disallowMultipleVarDecl": "exceptUndefined",
"disallowEmptyBlocks": true,
"requireCommaBeforeLineBreak": true,
"disallowSpaceAfterPrefixUnaryOperators": true,
"disallowSpaceBeforePostfixUnaryOperators": true,
"disallowSpaceBeforeBinaryOperators": [
","
],
"requireSpacesInForStatement": true,
"requireSpaceBeforeBinaryOperators": true,
"requireSpaceAfterBinaryOperators": true,
"disallowKeywords": [
"with"
],
"validateIndentation": 2,
"validateQuoteMarks": { "mark": "'", "escape": true },
"disallowMixedSpacesAndTabs": true,
"disallowTrailingWhitespace": true,
"requireCapitalizedConstructors": true,
"requireDotNotation": true,
"disallowNewlineBeforeBlockStatements": true,
"disallowMultipleLineStrings": true,
"esprima" : "esprima-fb"
}

View File

@ -1,2 +0,0 @@
vendor
Gruntfile.js

View File

@ -1,27 +0,0 @@
{
// for files at project root
"node": true,
// shared with .jshintrc files for browser and node
"unused": false,
"bitwise": false,
"eqnull": true,
"eqeqeq": true,
"forin": true,
"immed": true,
"expr": true,
"latedef": "nofunc",
"noarg": true,
"noempty": true,
"undef": true,
"quotmark": "single",
"plusplus": false,
"boss": true,
"laxbreak": true,
"laxcomma": true,
"validthis": true,
"sub": true,
"-W084": true,
"scripturl": true,
"evil": true
}

View File

@ -1,14 +0,0 @@
{
"extends": "./.jshintrc",
"node": false,
"browser": true,
"predef": {
// require.js
"define": true,
"require": true,
"console": false,
"-event": true,
"-name": true
}
}

View File

@ -1,9 +0,0 @@
{
"extends": "./.jshintrc",
"node": true,
"globals": {
"Promise": true,
"status": true
}
}

View File

@ -1,31 +0,0 @@
# Development Environment Setup
- You must have first setup [Kibana](https://github.com/elastic/kibana) by following their [contributing](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md) docs.
- Next esvm is strongly suggested simply `npm install -g esvm`
- Clone and go into this project
```
git clone https://github.com/elastic/elasticsearch-marvel.git && cd elasticsearch-marvel
```
- Install dependencies
```
npm install
```
- Start it up
- Start Elasticsearch, with esvm `esvm --branch <target version>` or traditionally with `bin/elasticsearch`.
- Start up the mock Marvel agent with `gulp index`.
- Start up the filesystem watching of Marvel code and syncing to Kibana's plugin directory with `gulp dev`.
- Finally, startup Kibana by running `bin/kibana --dev` from the root of the Kibana project.
- Check it out, navigate to your [Kibana App](http://localhost:5601)
- Run tests
```
gulp test
```
- Debug tests
Add a `debugger` line to create a breakpoint, and then:
```
gulp sync && mocha debug --compilers js:babel/register /pathto/kibana/installedPlugins/marvel/pathto/__test__/testfile.js
```

View File

@ -1,5 +0,0 @@
ElasticSearch
Copyright 2009-2013 ElasticSearch
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).

View File

@ -1,28 +0,0 @@
# WARNING THIS IS ALL IN FLUX
# Testing/Installing Marvel 2.1.0
The easiest way to get to know the new Marvel is probably by [reading the docs](https://github.com/elastic/elasticsearch-marvel/blob/master/docs/index.asciidoc).
The second easiest way is to just install it.
- Install the marvel plugin on kibana `./bin/kibana plugin -i elasticsearch/marvel/2.1.0`
- Install the License plugin on your cluster `./bin/plugin install license`
- Install the Marvel agent on your cluster `./bin/plugin install marvel-agent`
Once done, open up the following url (assuming standard kibana config): [http://localhost:5601/app/marvel.](http://localhost:5601/app/marvel)
# Deploying Marvel
The `release task` creates archives and uploads them to download.elasticsearch.org/elasticsearch/marvel/VERSION. You will need S3 credentials in .aws-config.json. Format as so:
```
{
"key":"MY_KEY_HERE",
"secret":"your/long/secret/string"
}
```
To upload the current archive as the "latest" release, use:
`gulp release`

View File

@ -1,269 +0,0 @@
{
"template": ".marvel*",
"settings": {
"marvel.index_format": 7,
"marvel.version": "${project.version}",
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapper.dynamic": false
},
"mappings": {
"_default_": {
"_all": {
"enabled": false
},
"date_detection": false,
"properties": {
"cluster_uuid": {
"type": "string",
"index": "not_analyzed"
},
"timestamp": {
"type": "date",
"format": "date_time"
}
}
},
"index_stats": {
"properties": {
"index_stats": {
"properties": {
"index": {
"type": "string",
"index": "not_analyzed"
},
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
},
"store": {
"properties": {
"size_in_bytes": {
"type": "long"
},
"throttle_time_in_millis": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
},
"index_time_in_millis": {
"type": "long"
},
"throttle_time_in_millis": {
"type": "long"
}
}
},
"merges": {
"properties": {
"total_size_in_bytes": {
"type": "long"
}
}
},
"search": {
"properties": {
"query_total": {
"type": "long"
},
"query_time_in_millis": {
"type": "long"
}
}
},
"segments": {
"properties": {
"memory_in_bytes": {
"type": "long"
}
}
},
"refresh": {
"properties": {
"total_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
},
"cluster_stats": {
"properties": {
"cluster_stats": {
"properties": {
"nodes": {
"type": "object"
},
"indices": {
"type": "object"
}
}
}
}
},
"cluster_state": {
"properties": {
"cluster_state": {
"properties": {
"version": {
"type": "long"
},
"master_node": {
"type": "string",
"index": "not_analyzed"
},
"status": {
"type": "string",
"index": "not_analyzed"
},
"nodes": {
"enabled": false
}
}
}
}
},
"cluster_info": {
"enabled": false
},
"phone_home": {
"enabled": false
},
"node_stats": {
"properties": {
"node_stats": {
"properties": {
"node_id": {
"type": "string",
"index": "not_analyzed"
},
"node_master": {
"type": "boolean"
},
"mlockall": {
"type": "boolean"
},
"disk_threshold_enabled": {
"type": "boolean"
},
"disk_threshold_watermark_high": {
"type": "short"
},
"indices": {
"type": "object"
},
"fs": {
"type": "object"
},
"process": {
"type": "object"
},
"jvm": {
"type": "object"
},
"thread_pool": {
"type": "object"
}
}
}
}
},
"indices_stats": {
"properties": {
"indices_stats": {
"type": "object"
}
}
},
"node": {
"properties": {
"state_uuid": {
"type": "string",
"index": "not_analyzed"
},
"node": {
"properties": {
"id": { "type": "string", "index": "not_analyzed" },
"name": { "type": "string", "index": "not_analyzed" },
"transport_address": { "type": "string", "index": "not_analyzed" },
"master": { "type": "boolean" },
"attributes": { "type": "object" }
}
}
}
},
"nodes": {
"properties": {
"state_uuid": {
"type": "string",
"index": "not_analyzed"
},
"node": {
"properties": {
"id": { "type": "string", "index": "not_analyzed" },
"name": { "type": "string", "index": "not_analyzed" },
"transport_address": { "type": "string", "index": "not_analyzed" },
"master": { "type": "boolean" },
"attributes": { "type": "object" }
}
}
}
},
"shards": {
"properties": {
"state_uuid": {
"type": "string",
"index": "not_analyzed"
},
"shard": {
"properties": {
"state": { "type": "string", "index": "not_analyzed" },
"primary": { "type": "boolean" },
"index": { "type": "string", "index": "not_analyzed" },
"relocating_node": { "type": "string", "index": "not_analyzed" },
"shard": { "type": "long" },
"node": { "type": "string", "index": "not_analyzed" }
}
}
}
},
"index_recovery": {
"properties": {
"index_recovery": {
"properties": {
"shards": {
"type": "object"
}
}
}
}
}
}
}

View File

@ -1,11 +0,0 @@
# Marvel Watchers
In this directory you will find example Watchers for Marvel that were distributed with the Watcher Beta documentation. The `watches` directory contains the example watches and the `test` directory contains integration test for each watcher.
The testing framework is written in Node.js and use ESVM to create an Elasticsearch instance for running the tests against
### Setup and Running Tests
- Run `npm install` in this directory
- Ensure you have `mocha` installed globally
- Run `mocha` in this directory

View File

@ -1,4 +0,0 @@
var Client = require('elasticsearch').Client;
module.exports = new Client({
host: 'http://localhost:9800'
});

View File

@ -1,11 +0,0 @@
var client = require('./client');
var template = require('./template.json');
module.exports = function () {
return client.indices.putTemplate({
body: template,
name: 'marvel'
});
};

View File

@ -1,40 +0,0 @@
var Promise = require('bluebird');
var request = require('request');
var injectStats = require('./inject_stats');
var loadWatcher = require('./load_watcher');
module.exports = function (watcher, fixture) {
return injectStats(fixture).then(function () {
return loadWatcher(watcher);
}).then(function () {
return new Promise(function (resolve, reject) {
var options = {
method: 'POST',
url: 'http://localhost:9800/_watcher/watch/' + watcher + '/_execute',
json: true,
body: {
trigger_event: {
schedule: {
scheduled_time: 'now',
triggered_time: 'now'
}
}
}
};
request(options, function (err, resp, body) {
if (err) return reject(err);
if (resp.statusCode === 200) return resolve(body);
var message = options.url + ' responed with ' + resp.statusCode;
var error = new Error(body.error || message);
error.body = body;
error.resp = resp;
error.code = resp.statusCode;
error.options = options;
reject(error);
});
});
});
};

View File

@ -1,13 +0,0 @@
var Promise = require('bluebird');
var portscanner = require('portscanner');
module.exports = function findPort(start, end, host) {
host = host || 'localhost';
return new Promise(function (resolve, reject) {
portscanner.findAPortNotInUse(start, end, host, function (err, port) {
if (err) return reject(err);
resolve(port);
});
});
};

View File

@ -1,73 +0,0 @@
var _ = require('lodash');
var moment = require('moment');
var client = require('./client');
module.exports = function (fixture) {
var indexPattern = fixture.indexPattern;
var type = fixture.type;
var data = fixture.data;
var rawData = fixture.rawData;
var startDate = fixture.startDate;
var duration = fixture.duration;
var dateField = fixture.dateField;
var workingDate = moment.utc();
var indices = [];
var body = [];
var fields;
function createEntries(row) {
var index = workingDate.format(indexPattern);
var entry = { 'timestamp': workingDate.toISOString() };
row.forEach(function (val, index) {
_.set(entry, fields[index], val);
});
indices.push(index);
body.push({
index: {
_index: index,
_type: type
}
});
body.push(entry);
}
if (rawData) {
rawData.forEach(function (row) {
var index = moment.utc(row[dateField]).format(indexPattern);
var entry = {};
_.each(row, function (val, key) {
_.set(entry, key, val);
});
indices.push(index);
body.push({
index: {
_index: index,
_type: type
}
});
body.push(entry);
});
} else {
fields = data.shift();
while(startDate <= workingDate) {
data.forEach(createEntries);
workingDate.subtract(duration);
}
}
return client.deleteByQuery({
index: _.unique(indices),
ignoreUnavailable: true,
allowNoIndices: true,
q: '*'
})
.then(function (arg) {
return client.bulk({
body: body,
refresh: true
});
});
};

View File

@ -1,26 +0,0 @@
var Promise = require('bluebird');
var request = require('request');
var path = require('path');
module.exports = function (name) {
var watch = require(path.join(__dirname, '..', 'watches', name + '.json'));
var options = {
method: 'PUT',
url: 'http://localhost:9800/_watcher/watch/' + name,
json: true,
body: watch
};
return new Promise(function (resolve, reject) {
request(options, function (err, resp, body) {
if (err) return reject(err);
if (resp.statusCode <= 201) resolve({ resp: resp, body: body });
var message = options.url + ' responded with ' + resp.statusCode;
var error = new Error(body.error || message);
error.body = body;
error.resp = resp;
error.code = resp.statusCode;
error.options = options;
reject(error);
});
});
};

View File

@ -1,71 +0,0 @@
var path = require('path');
var Promise = require('bluebird');
var libesvm = require('libesvm');
var createTemplate = require('./create_template');
function startEs() {
var options = {
version: '1.5.2',
directory: path.join(__dirname, '..', 'esvm'),
purge: true,
plugins: [
'elasticsearch/watcher/1.0.0-Beta1-5',
'elasticsearch/license/latest'
],
config: {
'script.groovy.sandbox.enabled': true,
'cluster.name': 'test',
'network.host': '127.0.0.1',
'http.port': 9800,
'watcher.actions.email.service.account': {
'local': {
'email_defaults.from': 'admin@example.com',
'smtp': {
'host': 'localhost',
'user': 'test',
'password': 'test',
'port': 5555
}
}
}
}
};
var cluster = libesvm.createCluster(options);
cluster.on('log', function (log) {
if (log.type === 'progress') return;
if (process.env.DEBUG) console.log('%s %s %s %s', log.level, log.node, log.type, log.message);
});
return cluster.install()
.then(function () {
return cluster.installPlugins();
})
.then(function () {
return cluster.start();
})
.then(function () {
after(function () {
this.timeout(60000);
return cluster.shutdown();
});
return cluster;
});
}
before(function () {
var self = this;
this.timeout(60000);
return new Promise(function (resolve, reject) {
startEs().then(function (cluster) {
self.cluster = cluster;
cluster.on('log', function (log) {
if (/watch service has started/.test(log.message)) {
createTemplate().then(function () {
resolve(cluster);
});
}
});
}).catch(reject);
});
});

View File

@ -1,56 +0,0 @@
var Promise = require('bluebird');
var SMTPServer = require('smtp-server').SMTPServer;
var MailParser = require('mailparser').MailParser;
var acceptAny = function (address, session, done) {
return done();
};
var mailbox = [];
function startSMTP() {
return new Promise(function (resolve, reject) {
var server = new SMTPServer({
logger: false,
disabledCommands: ['STARTTLS'],
onAuth: function (auth, session, done) {
done(null, { user: 1 });
},
onMailFrom: acceptAny,
onRcptTo: acceptAny,
onData: function (stream, session, done) {
var mailparser = new MailParser();
mailparser.on('end', function (mailObj) {
mailbox.push(mailObj);
done();
});
stream.pipe(mailparser);
}
});
server.listen(5555, function (err) {
if (err) return reject(err);
after(function (done) {
server.close(done);
});
resolve(server);
});
});
}
before(function () {
var self = this;
return startSMTP().then(function (server) {
this.smtp = server;
return server;
});
});
beforeEach(function () {
this.mailbox = mailbox;
});
afterEach(function () {
mailbox = [];
});
module.exports = mailbox;

View File

@ -1,447 +0,0 @@
{
"template": ".marvel*",
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"analysis": {
"analyzer": {
"default": {
"type": "standard",
"stopwords": "_none_"
}
}
},
"mapper.dynamic": true,
"marvel.index_format": 6
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "multi_field",
"fields": {
"{name}": {
"type": "string",
"index": "analyzed",
"omit_norms": true
},
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
}
}
}
}
}
]
},
"node_stats": {
"properties": {
"breakers": {
"properties": {
"fielddata": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"request": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"parent": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
}
}
},
"fs": {
"properties": {
"total": {
"properties": {
"disk_io_op": {
"type": "long"
},
"disk_reads": {
"type": "long"
},
"disk_writes": {
"type": "long"
},
"disk_io_size_in_bytes": {
"type": "long"
},
"disk_read_size_in_bytes": {
"type": "long"
},
"disk_write_size_in_bytes": {
"type": "long"
}
}
}
}
},
"jvm": {
"properties": {
"buffer_pools": {
"properties": {
"direct": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
},
"mapped": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
}
}
},
"gc": {
"properties": {
"collectors": {
"properties": {
"young": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
},
"old": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
},
"indices": {
"properties": {
"indexing": {
"properties": {
"throttle_time_in_millis": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"os": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
},
"thread_pool": {
"properties": {
"listener": {
"properties": {
"threads": {
"type": "long"
},
"rejected": {
"type": "long"
},
"completed": {
"type": "long"
},
"queue": {
"type": "long"
},
"largest": {
"type": "long"
}
}
}
}
}
}
},
"index_stats": {
"properties": {
"index": {
"type": "multi_field",
"fields": {
"index": {
"type": "string",
"norms": {
"enabled": false
}
},
"raw": {
"type": "string",
"index": "not_analyzed",
"norms": {
"enabled": false
},
"index_options": "docs",
"include_in_all": false,
"ignore_above": 256
}
}
},
"total": {
"properties": {
"fielddata": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"throttle_time_in_millis": {
"type": "long"
}
}
},
"merges": {
"properties": {
"total_size_in_bytes": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"search": {
"properties": {
"query": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_event": {},
"shard_event": {},
"indices_stats": {
"properties": {
"primaries": {
"properties": {
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
},
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"search": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_stats": {},
"index_event": {},
"node_event": {},
"routing_event": {},
"cluster_state": {
"properties": {
"blocks": {
"type": "object",
"enabled": false
},
"nodes": {
"type": "object",
"enabled": false
},
"routing_nodes": {
"type": "object",
"enabled": false
},
"routing_table": {
"type": "object",
"enabled": false
}
}
}
}
}

View File

@ -1,28 +0,0 @@
var lib = require('requirefrom')('lib');
var executeWatcher = lib('execute_watcher');
var expect = require('expect.js');
module.exports = function testNoExecute(options, message, generateRawData) {
describe(message, function () {
var response;
beforeEach(function () {
this.timeout(5000);
var rawData = generateRawData();
var fixture = {
indexPattern: options.indexPattern,
type: options.type,
dateField: 'timestamp',
rawData: rawData
};
return executeWatcher(options.watcher, fixture).then(function (resp) {
response = resp;
if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' '));
return resp;
});
});
it('should not meet the script condition', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
});
});
};

View File

@ -1,23 +0,0 @@
{
"name": "marvel-watchers",
"version": "1.0.0",
"description": "",
"main": "index.js",
"directories": {
"test": "test"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"libesvm": "0.0.12",
"lodash": "^3.8.0",
"lodash-deep": "^1.6.0",
"mailparser": "^0.5.1",
"moment": "^2.10.3",
"portscanner": "^1.0.0",
"requirefrom": "^0.2.0"
}
}

View File

@ -1,95 +0,0 @@
var _ = require('lodash');
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var options = {
indexPattern: '[.marvel-]YYYY.MM.DD',
type: 'cluster_stats',
watcher: 'cluster_status'
};
var testNoExecute = lib('test_no_execute').bind(null, options);
var client = lib('client');
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('Cluster Status', function () {
describe('Red for 60 seconds', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var workingDate = moment.utc();
var rawData = _.times(12, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
var fixture = {
indexPattern: '[.marvel-]YYYY.MM.DD',
type: 'cluster_stats',
dateField: 'timestamp',
rawData: rawData
};
return executeWatcher('cluster_status', fixture).then(function (resp) {
response = resp;
if (process.env.DEBUG) console.log(JSON.stringify(resp, null, ' '));
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.subject).to.contain('Watcher Notification - Cluster has been RED for the last 60 seconds');
expect(message.text).to.contain('Your cluster has been red for the last 60 seconds.');
});
});
testNoExecute('Red for 55 then Yellow for 60 seconds', function () {
var workingDate = moment.utc();
var rawData = _.times(11, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(12, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
return rawData;
});
testNoExecute('Red for 30 then Yellow for 60 seconds', function () {
var workingDate = moment.utc();
var rawData = _.times(6, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(12, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
return rawData;
});
testNoExecute('Red for 5 Yellow for 10 Red for 10 Green for 60', function () {
var workingDate = moment.utc();
var rawData = _.times(1, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
});
rawData.concat(_.times(2, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'yellow' };
}));
rawData.concat(_.times(2, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'red' };
}));
rawData.concat(_.times(12, function () {
return { 'timestamp': workingDate.subtract(5, 's').format(), status: 'green' };
}));
return rawData;
});
});
});

View File

@ -1,81 +0,0 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('CPU Usage', function () {
describe('above 75%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'os.cpu.user'],
['node-01', 75],
['node-02', 85],
['node-03', 60]
]
};
return executeWatcher('cpu_usage', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - CPU Usage is at 75.0%');
expect(message.text).to.contain('"node-02" - CPU Usage is at 85.0%');
});
});
describe('below 75%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'os.cpu.user'],
['node-01', 35],
['node-02', 25],
['node-03', 10]
]
};
return executeWatcher('cpu_usage', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

View File

@ -1,82 +0,0 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('File Descriptors', function () {
describe('above 80%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'indices.fielddata.memory_size_in_bytes'],
['node-01', 81000],
['node-02', 70000],
['node-03', 90000]
]
};
return executeWatcher('fielddata', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - Fielddata utilization is at 81000.0 bytes (81%)');
expect(message.text).to.contain('"node-03" - Fielddata utilization is at 90000.0 bytes (90%)');
});
});
describe('below 80%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'indices.fielddata.memory_size_in_bytes'],
['node-01', 12039],
['node-02', 54393],
['node-03', 20302]
]
};
return executeWatcher('fielddata', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

View File

@ -1,82 +0,0 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('File Descriptors', function () {
describe('above 80%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'process.open_file_descriptors'],
['node-01', Math.round(65535*0.75)],
['node-02', Math.round(65535*0.81)],
['node-03', Math.round(65535*0.93)]
]
};
return executeWatcher('file_descriptors', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-02" - File Descriptors is at ' + Math.round(65535*0.81) + '.0 ('+ Math.round(((65535*0.81)/65535)*100) + '%)');
expect(message.text).to.contain('"node-03" - File Descriptors is at ' + Math.round(65535*0.93) + '.0 ('+ Math.round(((65535*0.93)/65535)*100) + '%)');
});
});
describe('below 80%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'process.open_file_descriptors'],
['node-01', Math.round(65535*0.05)],
['node-02', Math.round(65535*0.30)],
['node-03', Math.round(65535*0.23)]
]
};
return executeWatcher('file_descriptors', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

View File

@ -1,81 +0,0 @@
var lib = require('requirefrom')('lib');
var expect = require('expect.js');
var moment = require('moment');
var executeWatcher = lib('execute_watcher');
var client = lib('client');
var indexPattern = '[.marvel-]YYYY.MM.DD';
lib('setup_es');
lib('setup_smtp_server');
describe('Marvel Watchers', function () {
describe('Memory Usage', function () {
describe('above 75%', function () {
var response;
beforeEach(function () {
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'jvm.mem.heap_used_percent'],
['node-01', 75],
['node-02', 85],
['node-03', 60]
]
};
return executeWatcher('heap_used', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should meet the script condition', function () {
expect(response.state).to.be('executed');
expect(response.execution_result.condition.script.met).to.be(true);
});
it('should send an email with multiple hosts', function () {
expect(this.mailbox).to.have.length(1);
var message = this.mailbox[0];
expect(message.text).to.contain('"node-01" - Memory Usage is at 75.0%');
expect(message.text).to.contain('"node-02" - Memory Usage is at 85.0%');
});
});
describe('below 75%', function () {
var response;
beforeEach(function () {
var self = this;
this.timeout(5000);
var fixture = {
indexPattern: indexPattern,
type: 'node_stats',
duration: moment.duration(5, 's'),
startDate: moment.utc().subtract(5, 'm'),
data: [
['node.name', 'jvm.mem.heap_used_percent'],
['node-01', 35],
['node-02', 25],
['node-03', 10]
]
};
return executeWatcher('heap_used', fixture).then(function (resp) {
response = resp;
return resp;
});
});
it('should not send an email', function () {
expect(response.state).to.be('execution_not_needed');
expect(response.execution_result.condition.script.met).to.be(false);
expect(this.mailbox).to.have.length(0);
});
});
});
});

View File

@ -1,2 +0,0 @@
--require expect.js
--reporter spec

View File

@ -1,92 +0,0 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": ".marvel-*",
"types": "cluster_stats",
"body": {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [
{
"range": {
"timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
],
"should": [
{
"term": {
"status.raw": "red"
}
},
{
"term": {
"status.raw": "green"
}
},
{
"term": {
"status.raw": "yellow"
}
}
]
}
}
}
},
"fields": ["timestamp","status"],
"sort": [
{
"timestamp": {
"order": "desc"
}
}
],
"size": 1,
"aggs": {
"minutes": {
"date_histogram": {
"field": "timestamp",
"interval": "5s"
},
"aggs": {
"status": {
"terms": {
"field": "status.raw",
"size": 3
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": {
"inline": "if (ctx.payload.hits.total < 1) return false; def rows = ctx.payload.hits.hits; if (rows[0].fields.status[0] != 'red') return false; if (ctx.payload.aggregations.minutes.buckets.size() < 12) return false; def last60Seconds = ctx.payload.aggregations.minutes.buckets[-12..-1]; return last60Seconds.every { it.status.buckets.every { s -> s.key == 'red' } }"
}
},
"actions": {
"send_email": {
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - Cluster has been RED for the last 60 seconds",
"body": "Your cluster has been red for the last 60 seconds."
}
}
}
}

View File

@ -1,73 +0,0 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "timestamp",
"interval": "minute"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"cpu": "desc"
}
},
"aggs": {
"cpu": {
"avg": {
"field": "os.cpu.user"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.cpu && node.cpu.value >= 75;"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.cpu && it.cpu.value >= 75 };"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - HIGH CPU USAGE",
"body": "Nodes with HIGH CPU Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - CPU Usage is at {{cpu.value}}%\n{{/ctx.payload._value}}"
}
}
}
}

View File

@ -1,79 +0,0 @@
{
"metadata": {
"fielddata_cache_size": 100000,
"threshold": 0.8
},
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"types": "node_stats",
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"timestamp": {
"gte": "now-1m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "timestamp",
"interval": "5s"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"fielddata": "desc"
}
},
"aggs": {
"fielddata": {
"avg": {
"field": "indices.fielddata.memory_size_in_bytes"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fielddata && node.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold);"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fielddata && it.fielddata.value >= (ctx.metadata.fielddata_cache_size * ctx.metadata.threshold) }).collect({ it.fielddata.percent = Math.round((it.fielddata.value/ctx.metadata.fielddata_cache_size)*100); it });"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - NODES WITH 80% FIELDDATA UTILIZATION",
"body": "Nodes with 80% FIELDDATA UTILIZATION (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Fielddata utilization is at {{fielddata.value}} bytes ({{fielddata.percent}}%)\n{{/ctx.payload._value}}"
}
}
}
}

View File

@ -1,79 +0,0 @@
{
"metadata": {
"system_fd": 65535,
"threshold": 0.8
},
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"types": "node_stats",
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"timestamp": {
"gte": "now-1m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "timestamp",
"interval": "5s"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"fd": "desc"
}
},
"aggs": {
"fd": {
"avg": {
"field": "process.open_file_descriptors"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.fd && node.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold);"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll({ return it.fd && it.fd.value >= (ctx.metadata.system_fd * ctx.metadata.threshold) }).collect({ it.fd.percent = Math.round((it.fd.value/ctx.metadata.system_fd)*100); it });"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - NODES WITH 80% FILE DESCRIPTORS USED",
"body": "Nodes with 80% FILE DESCRIPTORS USED (above 80%):\n\n{{#ctx.payload._value}}\"{{key}}\" - File Descriptors is at {{fd.value}} ({{fd.percent}}%)\n{{/ctx.payload._value}}"
}
}
}
}

View File

@ -1,73 +0,0 @@
{
"trigger": {
"schedule": {
"interval": "1m"
}
},
"input": {
"search": {
"request": {
"indices": [
".marvel-*"
],
"search_type": "count",
"body": {
"query": {
"filtered": {
"filter": {
"range": {
"timestamp": {
"gte": "now-2m",
"lte": "now"
}
}
}
}
},
"aggs": {
"minutes": {
"date_histogram": {
"field": "timestamp",
"interval": "minute"
},
"aggs": {
"nodes": {
"terms": {
"field": "node.name.raw",
"size": 10,
"order": {
"memory": "desc"
}
},
"aggs": {
"memory": {
"avg": {
"field": "jvm.mem.heap_used_percent"
}
}
}
}
}
}
}
}
}
}
},
"throttle_period": "30m",
"condition": {
"script": "if (ctx.payload.aggregations.minutes.buckets.size() == 0) return false; def latest = ctx.payload.aggregations.minutes.buckets[-1]; def node = latest.nodes.buckets[0]; return node && node.memory && node.memory.value >= 75;"
},
"actions": {
"send_email": {
"transform": {
"script": "def latest = ctx.payload.aggregations.minutes.buckets[-1]; return latest.nodes.buckets.findAll { return it.memory && it.memory.value >= 75 };"
},
"email": {
"to": "user@example.com",
"subject": "Watcher Notification - HIGH MEMORY USAGE",
"body": "Nodes with HIGH MEMORY Usage (above 75%):\n\n{{#ctx.payload._value}}\"{{key}}\" - Memory Usage is at {{memory.value}}%\n{{/ctx.payload._value}}"
}
}
}
}