From 05db5dc2c8e3b1d41081b9c7fb677b5850e7b3db Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 5 Jun 2015 13:12:05 +0200 Subject: [PATCH] create parent pom project from its original location --- dev-tools/build_release.py | 1046 +++---- dev-tools/pom.xml | 37 + .../resources/forbidden/all-signatures.txt | 47 + .../resources/forbidden/core-signatures.txt | 86 + .../resources/forbidden/test-signatures.txt | 20 + .../third-party-shaded-signatures.txt | 22 + .../third-party-unshaded-signatures.txt | 60 + .../elasticsearch_license_header.txt | 16 + .../license_header_definition.xml | 13 + dev-tools/src/main/resources/pmd/custom.xml | 20 + .../shared-test-resources/log4j.properties | 9 + dev-tools/src/main/resources/site/site_en.xml | 7 + plugins/pom.xml | 122 + pom.xml | 2430 +++++++++-------- 14 files changed, 2173 insertions(+), 1762 deletions(-) create mode 100644 dev-tools/pom.xml create mode 100644 dev-tools/src/main/resources/forbidden/all-signatures.txt create mode 100644 dev-tools/src/main/resources/forbidden/core-signatures.txt create mode 100644 dev-tools/src/main/resources/forbidden/test-signatures.txt create mode 100644 dev-tools/src/main/resources/forbidden/third-party-shaded-signatures.txt create mode 100644 dev-tools/src/main/resources/forbidden/third-party-unshaded-signatures.txt create mode 100644 dev-tools/src/main/resources/license-check/elasticsearch_license_header.txt create mode 100644 dev-tools/src/main/resources/license-check/license_header_definition.xml create mode 100644 dev-tools/src/main/resources/pmd/custom.xml create mode 100644 dev-tools/src/main/resources/shared-test-resources/log4j.properties create mode 100644 dev-tools/src/main/resources/site/site_en.xml create mode 100644 plugins/pom.xml diff --git a/dev-tools/build_release.py b/dev-tools/build_release.py index 8ae300d98fd..af3baed7e7d 100644 --- a/dev-tools/build_release.py +++ b/dev-tools/build_release.py @@ -6,10 +6,10 @@ # not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on +# software distributed under the License is distributed on # an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, # either express or implied. See the License for the specific # language governing permissions and limitations under the License. @@ -19,146 +19,119 @@ import tempfile import shutil import os import datetime -import json -import time -import sys import argparse -import hmac -import urllib -import fnmatch -import socket -import urllib.request -import subprocess +import github3 +import smtplib +import sys -from functools import partial -from http.client import HTTPConnection -from http.client import HTTPSConnection +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from os.path import dirname, abspath -""" - This tool builds a release from the a given elasticsearch branch. +""" + This tool builds a release from the a given elasticsearch plugin branch. In order to execute it go in the top level directory and run: - $ python3 dev_tools/build_release.py --branch 0.90 --publish --remote origin + $ python3 dev_tools/build_release.py --branch master --publish --remote origin By default this script runs in 'dry' mode which essentially simulates a release. If the - '--publish' option is set the actual release is done. The script takes over almost all + '--publish' option is set the actual release is done. + + $ python3 dev_tools/build_release.py --publish --remote origin + + The script takes over almost all steps necessary for a release from a high level point of view it does the following things: - - run prerequisit checks ie. check for Java 1.7 being presend or S3 credentials available as env variables + - run prerequisite checks - detect the version to release from the specified branch (--branch) or the current branch - - creates a release branch & updates pom.xml and Version.java to point to a release version rather than a snapshot - - builds the artifacts and runs smoke-tests on the build zip & tar.gz files - - commits the new version and merges the release branch into the source branch - - creates a tag and pushes the commit to the specified origin (--remote) - - publishes the releases to Sonatype and S3 + - creates a version release branch & updates pom.xml to point to a release version rather than a snapshot + - builds the artifacts + - commits the new version and merges the version release branch into the source branch + - merges the master release branch into the master branch + - creates a tag and pushes branch and master to the specified origin (--remote) + - publishes the releases to sonatype Once it's done it will print all the remaining steps. Prerequisites: - Python 3k for script execution - - Boto for S3 Upload ($ apt-get install python-boto) - - RPM for RPM building ($ apt-get install rpm) - - S3 keys exported via ENV variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) - - GPG data exported via ENV variables (GPG_KEY_ID, GPG_PASSPHRASE, optionally GPG_KEYRING) - - S3 target repository via ENV variables (S3_BUCKET_SYNC_TO, optionally S3_BUCKET_SYNC_FROM) """ env = os.environ -PLUGINS = [('license', 'elasticsearch/license/latest'), - ('bigdesk', 'lukas-vlcek/bigdesk'), - ('paramedic', 'karmi/elasticsearch-paramedic'), - ('segmentspy', 'polyfractal/elasticsearch-segmentspy'), - ('inquisitor', 'polyfractal/elasticsearch-inquisitor'), - ('head', 'mobz/elasticsearch-head')] - LOG = env.get('ES_RELEASE_LOG', '/tmp/elasticsearch_release.log') +ROOT_DIR = abspath(os.path.join(abspath(dirname(__file__)), '../')) +POM_FILE = ROOT_DIR + '/pom.xml' -# console colors -COLOR_OK = '\033[92m' -COLOR_END = '\033[0m' -COLOR_FAIL = '\033[91m' - +########################################################## +# +# Utility methods (log and run) +# +########################################################## +# Log a message def log(msg): - log_plain('\n%s' % msg) + log_plain('\n%s' % msg) + +# Purge the log file +def purge_log(): + try: + os.remove(LOG) + except FileNotFoundError: + pass + + +# Log a message to the LOG file def log_plain(msg): - f = open(LOG, mode='ab') - f.write(msg.encode('utf-8')) - f.close() + f = open(LOG, mode='ab') + f.write(msg.encode('utf-8')) + f.close() + +# Run a command and log it def run(command, quiet=False): - log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) - if os.system('%s >> %s 2>&1' % (command, LOG)): - msg = ' FAILED: %s [see log %s]' % (command, LOG) - if not quiet: - print(msg) - raise RuntimeError(msg) + log('%s: RUN: %s\n' % (datetime.datetime.now(), command)) + if os.system('%s >> %s 2>&1' % (command, LOG)): + msg = ' FAILED: %s [see log %s]' % (command, LOG) + if not quiet: + print(msg) + raise RuntimeError(msg) +########################################################## +# +# Clean logs and check JAVA and Maven +# +########################################################## try: - JAVA_HOME = env['JAVA_HOME'] + purge_log() + JAVA_HOME = env['JAVA_HOME'] except KeyError: - raise RuntimeError(""" + raise RuntimeError(""" Please set JAVA_HOME in the env before running release tool On OSX use: export JAVA_HOME=`/usr/libexec/java_home -v '1.7*'`""") try: - JAVA_HOME = env['JAVA7_HOME'] -except KeyError: - pass #no JAVA7_HOME - we rely on JAVA_HOME + MVN = 'mvn' + # make sure mvn3 is used if mvn3 is available + # some systems use maven 2 as default + run('mvn3 --version', quiet=True) + MVN = 'mvn3' +except RuntimeError: + pass -try: - # make sure mvn3 is used if mvn3 is available - # some systems use maven 2 as default - subprocess.check_output('mvn3 --version', shell=True, stderr=subprocess.STDOUT) - MVN = 'mvn3' -except subprocess.CalledProcessError: - MVN = 'mvn' - def java_exe(): - path = JAVA_HOME - return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) + path = JAVA_HOME + return 'export JAVA_HOME="%s" PATH="%s/bin:$PATH" JAVACMD="%s/bin/java"' % (path, path, path) -def verify_java_version(version): - s = os.popen('%s; java -version 2>&1' % java_exe()).read() - if ' version "%s.' % version not in s: - raise RuntimeError('got wrong version for java %s:\n%s' % (version, s)) - -# Verifies the java version. We guarantee that we run with Java 1.7 -# If 1.7 is not available fail the build! -def verify_mvn_java_version(version, mvn): - s = os.popen('%s; %s --version 2>&1' % (java_exe(), mvn)).read() - if 'Java version: %s' % version not in s: - raise RuntimeError('got wrong java version for %s %s:\n%s' % (mvn, version, s)) - -# Returns the hash of the current git HEAD revision -def get_head_hash(): - return os.popen(' git rev-parse --verify HEAD 2>&1').read().strip() - -# Returns the hash of the given tag revision -def get_tag_hash(tag): - return os.popen('git show-ref --tags %s --hash 2>&1' % (tag)).read().strip() - -# Returns the name of the current branch -def get_current_branch(): - return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() +########################################################## +# +# String and file manipulation utils +# +########################################################## # Utility that returns the name of the release branch for a given version -def release_branch(version): - return 'release_branch_%s' % version - -# runs get fetch on the given remote -def fetch(remote): - run('git fetch %s' % remote) - -# Creates a new release branch from the given source branch -# and rebases the source branch from the remote before creating -# the release branch. Note: This fails if the source branch -# doesn't exist on the provided remote. -def create_release_branch(remote, src_branch, release): - run('git checkout %s' % src_branch) - run('git pull --rebase %s %s' % (remote, src_branch)) - run('git checkout -b %s' % (release_branch(release))) +def release_branch(branchsource, version): + return 'release_branch_%s_%s' % (branchsource, version) # Reads the given file and applies the @@ -166,367 +139,204 @@ def create_release_branch(remote, src_branch, release): # a line the given file is replaced with # the modified input. def process_file(file_path, line_callback): - fh, abs_path = tempfile.mkstemp() - modified = False - with open(abs_path,'w', encoding='utf-8') as new_file: - with open(file_path, encoding='utf-8') as old_file: - for line in old_file: - new_line = line_callback(line) - modified = modified or (new_line != line) - new_file.write(new_line) - os.close(fh) - if modified: - #Remove original file - os.remove(file_path) - #Move new file - shutil.move(abs_path, file_path) - return True - else: - # nothing to do - just remove the tmp file - os.remove(abs_path) - return False + fh, abs_path = tempfile.mkstemp() + modified = False + with open(abs_path, 'w', encoding='utf-8') as new_file: + with open(file_path, encoding='utf-8') as old_file: + for line in old_file: + new_line = line_callback(line) + modified = modified or (new_line != line) + new_file.write(new_line) + os.close(fh) + if modified: + #Remove original file + os.remove(file_path) + #Move new file + shutil.move(abs_path, file_path) + return True + else: + # nothing to do - just remove the tmp file + os.remove(abs_path) + return False + + +# Split a version x.y.z as an array of digits [x,y,z] +def split_version_to_digits(version): + return list(map(int, re.findall(r'\d+', version))) + + +# Guess the next snapshot version number (increment last digit) +def guess_snapshot(version): + digits = split_version_to_digits(version) + source = '%s.%s.%s' % (digits[0], digits[1], digits[2]) + destination = '%s.%s.%s' % (digits[0], digits[1], digits[2] + 1) + return version.replace(source, destination) + + +# Guess the anchor in generated documentation +# Looks like this "#version-230-for-elasticsearch-13" +def get_doc_anchor(release, esversion): + plugin_digits = split_version_to_digits(release) + es_digits = split_version_to_digits(esversion) + return '#version-%s%s%s-for-elasticsearch-%s%s' % ( + plugin_digits[0], plugin_digits[1], plugin_digits[2], es_digits[0], es_digits[1]) -# Walks the given directory path (defaults to 'docs') -# and replaces all 'coming[$version]' tags with -# 'added[$version]'. This method only accesses asciidoc files. -def update_reference_docs(release_version, path='docs'): - pattern = 'coming[%s' % (release_version) - replacement = 'added[%s' % (release_version) - pending_files = [] - def callback(line): - return line.replace(pattern, replacement) - for root, _, file_names in os.walk(path): - for file_name in fnmatch.filter(file_names, '*.asciidoc'): - full_path = os.path.join(root, file_name) - if process_file(full_path, callback): - pending_files.append(os.path.join(root, file_name)) - return pending_files # Moves the pom.xml file from a snapshot to a release def remove_maven_snapshot(pom, release): - pattern = '%s-SNAPSHOT' % (release) - replacement = '%s' % (release) - def callback(line): - return line.replace(pattern, replacement) - process_file(pom, callback) + pattern = '%s-SNAPSHOT' % release + replacement = '%s' % release -# Moves the Version.java file from a snapshot to a release -def remove_version_snapshot(version_file, release): - # 1.0.0.Beta1 -> 1_0_0_Beta1 - release = release.replace('.', '_') - pattern = 'new Version(V_%s_ID, true' % (release) - replacement = 'new Version(V_%s_ID, false' % (release) - def callback(line): - return line.replace(pattern, replacement) - process_file(version_file, callback) + def callback(line): + return line.replace(pattern, replacement) -# Stages the given files for the next git commit -def add_pending_files(*files): - for file in files: - run('git add %s' % (file)) + process_file(pom, callback) -# Executes a git commit with 'release [version]' as the commit message -def commit_release(release): - run('git commit -m "release [%s]"' % release) -def commit_feature_flags(release): - run('git commit -m "Update Documentation Feature Flags [%s]"' % release) +# Moves the pom.xml file to the next snapshot +def add_maven_snapshot(pom, release, snapshot): + pattern = '%s' % release + replacement = '%s-SNAPSHOT' % snapshot -def tag_release(release): - run('git tag -a v%s -m "Tag release version %s"' % (release, release)) + def callback(line): + return line.replace(pattern, replacement) -def run_mvn(*cmd): - for c in cmd: - run('%s; %s %s' % (java_exe(), MVN, c)) + process_file(pom, callback) -def build_release(run_tests=False, dry_run=True, cpus=1, bwc_version=None): - target = 'deploy' - if dry_run: - target = 'package' - if run_tests: - run_mvn('clean', - 'test -Dtests.jvms=%s -Des.node.mode=local' % (cpus), - 'test -Dtests.jvms=%s -Des.node.mode=network' % (cpus)) - if bwc_version: - print('Running Backwards compatibility tests against version [%s]' % (bwc_version)) - run_mvn('clean', 'test -Dtests.filter=@backwards -Dtests.bwc.version=%s -Dtests.bwc=true -Dtests.jvms=1' % bwc_version) - run_mvn('clean test-compile -Dforbidden.test.signatures="org.apache.lucene.util.LuceneTestCase\$AwaitsFix @ Please fix all bugs before release"') - gpg_args = '-Dgpg.key="%s" -Dgpg.passphrase="%s" -Ddeb.sign=true' % (env.get('GPG_KEY_ID'), env.get('GPG_PASSPHRASE')) - if env.get('GPG_KEYRING'): - gpg_args += ' -Dgpg.keyring="%s"' % env.get('GPG_KEYRING') - run_mvn('clean %s -DskipTests %s' % (target, gpg_args)) - success = False - try: - run_mvn('-DskipTests rpm:rpm %s' % (gpg_args)) - success = True - finally: - if not success: - print(""" - RPM Bulding failed make sure "rpm" tools are installed. - Use on of the following commands to install: - $ brew install rpm # on OSX - $ apt-get install rpm # on Ubuntu et.al - """) -# Uses the github API to fetch open tickets for the given release version -# if it finds any tickets open for that version it will throw an exception -def ensure_no_open_tickets(version): - version = "v%s" % version - conn = HTTPSConnection('api.github.com') - try: - log('Checking for open tickets on Github for version %s' % version) - log('Check if node is available') - conn.request('GET', '/repos/elastic/elasticsearch/issues?state=open&labels=%s' % version, headers= {'User-Agent' : 'Elasticsearch version checker'}) - res = conn.getresponse() - if res.status == 200: - issues = json.loads(res.read().decode("utf-8")) - if issues: - urls = [] - for issue in issues: - urls.append(issue['html_url']) - raise RuntimeError('Found open issues for release version %s:\n%s' % (version, '\n'.join(urls))) - else: - log("No open issues found for version %s" % version) - else: - raise RuntimeError('Failed to fetch issue list from Github for release version %s' % version) - except socket.error as e: - log("Failed to fetch issue list from Github for release version %s' % version - Exception: [%s]" % (version, e)) - #that is ok it might not be there yet - finally: - conn.close() - -def wait_for_node_startup(host='127.0.0.1', port=9200,timeout=15): - for _ in range(timeout): - conn = HTTPConnection(host, port, timeout) - try: - log('Waiting until node becomes available for 1 second') - time.sleep(1) - log('Check if node is available') - conn.request('GET', '') - res = conn.getresponse() - if res.status == 200: - return True - except socket.error as e: - log("Failed while waiting for node - Exception: [%s]" % e) - #that is ok it might not be there yet - finally: - conn.close() - - return False - -# Ensures we are using a true Lucene release, not a snapshot build: -def verify_lucene_version(): - s = open('pom.xml', encoding='utf-8').read() - if 'download.elastic.co/lucenesnapshots' in s: - raise RuntimeError('pom.xml contains download.elastic.co/lucenesnapshots repository: remove that before releasing') - - m = re.search(r'(.*?)', s) - if m is None: - raise RuntimeError('unable to locate lucene.version in pom.xml') - lucene_version = m.group(1) - - m = re.search(r'(.*?)', s) - if m is None: - raise RuntimeError('unable to locate lucene.maven.version in pom.xml') - lucene_maven_version = m.group(1) - if lucene_version != lucene_maven_version: - raise RuntimeError('pom.xml is still using a snapshot release of lucene (%s): cutover to a real lucene release before releasing' % lucene_maven_version) - -# Checks the pom.xml for the release version. +# Checks the pom.xml for the release version. 2.0.0-SNAPSHOT # This method fails if the pom file has no SNAPSHOT version set ie. # if the version is already on a release version we fail. # Returns the next version string ie. 0.90.7 def find_release_version(src_branch): - run('git checkout %s' % src_branch) - with open('pom.xml', encoding='utf-8') as file: - for line in file: - match = re.search(r'(.+)-SNAPSHOT', line) - if match: - return match.group(1) - raise RuntimeError('Could not find release version in branch %s' % src_branch) + git_checkout(src_branch) + with open(POM_FILE, encoding='utf-8') as file: + for line in file: + match = re.search(r'(.+)-SNAPSHOT', line) + if match: + return match.group(1) + raise RuntimeError('Could not find release version in branch %s' % src_branch) -def artifact_names(release, path = ''): - return [os.path.join(path, 'elasticsearch-%s.%s' % (release, t)) for t in ['deb', 'tar.gz', 'zip']] -def get_artifacts(release): - common_artifacts = artifact_names(release, 'target/releases/') - for f in common_artifacts: - if not os.path.isfile(f): - raise RuntimeError('Could not find required artifact at %s' % f) - rpm = os.path.join('target/rpm/elasticsearch/RPMS/noarch/', 'elasticsearch-%s-1.noarch.rpm' % release) - if os.path.isfile(rpm): - log('RPM [%s] contains: ' % rpm) - run('rpm -pqli %s' % rpm) - # this is an oddness of RPM that is attches -1 so we have to rename it - renamed_rpm = os.path.join('target/rpm/elasticsearch/RPMS/noarch/', 'elasticsearch-%s.noarch.rpm' % release) - shutil.move(rpm, renamed_rpm) - common_artifacts.append(renamed_rpm) - else: - raise RuntimeError('Could not find required artifact at %s' % rpm) - return common_artifacts +# extract a value from pom.xml +def find_from_pom(tag): + with open(POM_FILE, encoding='utf-8') as file: + for line in file: + match = re.search(r'<%s>(.+)' % (tag, tag), line) + if match: + return match.group(1) + raise RuntimeError('Could not find <%s> in pom.xml file' % (tag)) -# Checks the jar files in each package -# Barfs if any of the package jar files differ -def check_artifacts_for_same_jars(artifacts): - jars = [] - for file in artifacts: - if file.endswith('.zip'): - jars.append(subprocess.check_output("unzip -l %s | grep '\.jar$' | awk -F '/' '{ print $NF }' | sort" % file, shell=True)) - if file.endswith('.tar.gz'): - jars.append(subprocess.check_output("tar tzvf %s | grep '\.jar$' | awk -F '/' '{ print $NF }' | sort" % file, shell=True)) - if file.endswith('.rpm'): - jars.append(subprocess.check_output("rpm -pqli %s | grep '\.jar$' | awk -F '/' '{ print $NF }' | sort" % file, shell=True)) - if file.endswith('.deb'): - jars.append(subprocess.check_output("dpkg -c %s | grep '\.jar$' | awk -F '/' '{ print $NF }' | sort" % file, shell=True)) - if len(set(jars)) != 1: - raise RuntimeError('JAR contents of packages are not the same, please check the package contents. Use [unzip -l], [tar tzvf], [dpkg -c], [rpm -pqli] to inspect') -# Generates sha1 checsums for all files -# and returns the checksum files as well -# as the given files in a list -def generate_checksums(files): - res = [] - for release_file in files: - directory = os.path.dirname(release_file) - file = os.path.basename(release_file) - checksum_file = '%s.sha1.txt' % file +########################################################## +# +# GIT commands +# +########################################################## +# Returns the hash of the current git HEAD revision +def get_head_hash(): + return os.popen('git rev-parse --verify HEAD 2>&1').read().strip() - if os.system('cd %s; shasum %s > %s' % (directory, file, checksum_file)): - raise RuntimeError('Failed to generate checksum for file %s' % release_file) - res = res + [os.path.join(directory, checksum_file), release_file] - return res -def download_and_verify(release, files, plugins=None, base_url='https://download.elastic.co/elasticsearch/elasticsearch'): - print('Downloading and verifying release %s from %s' % (release, base_url)) - tmp_dir = tempfile.mkdtemp() - try: - downloaded_files = [] +# Returns the name of the current branch +def get_current_branch(): + return os.popen('git rev-parse --abbrev-ref HEAD 2>&1').read().strip() + + +# runs get fetch on the given remote +def fetch(remote): + run('git fetch %s' % remote) + + +# Creates a new release branch from the given source branch +# and rebases the source branch from the remote before creating +# the release branch. Note: This fails if the source branch +# doesn't exist on the provided remote. +def create_release_branch(remote, src_branch, release): + git_checkout(src_branch) + run('git pull --rebase %s %s' % (remote, src_branch)) + run('git checkout -b %s' % (release_branch(src_branch, release))) + + +# Stages the given files for the next git commit +def add_pending_files(*files): for file in files: - name = os.path.basename(file) - url = '%s/%s' % (base_url, name) - abs_file_path = os.path.join(tmp_dir, name) - print(' Downloading %s' % (url)) - downloaded_files.append(abs_file_path) - urllib.request.urlretrieve(url, abs_file_path) - url = ''.join([url, '.sha1.txt']) - checksum_file = os.path.join(tmp_dir, ''.join([abs_file_path, '.sha1.txt'])) - urllib.request.urlretrieve(url, checksum_file) - print(' Verifying checksum %s' % (checksum_file)) - run('cd %s && sha1sum -c %s' % (tmp_dir, os.path.basename(checksum_file))) - smoke_test_release(release, downloaded_files, get_tag_hash('v%s' % release), plugins) - print(' SUCCESS') - finally: - shutil.rmtree(tmp_dir) + run('git add %s' % file) -def smoke_test_release(release, files, expected_hash, plugins): - for release_file in files: - if not os.path.isfile(release_file): - raise RuntimeError('Smoketest failed missing file %s' % (release_file)) - tmp_dir = tempfile.mkdtemp() - if release_file.endswith('tar.gz'): - run('tar -xzf %s -C %s' % (release_file, tmp_dir)) - elif release_file.endswith('zip'): - run('unzip %s -d %s' % (release_file, tmp_dir)) + +# Executes a git commit with 'release [version]' as the commit message +def commit_release(artifact_id, release): + run('git commit -m "prepare release %s-%s"' % (artifact_id, release)) + + +# Commit documentation changes on the master branch +def commit_master(release): + run('git commit -m "update documentation with release %s"' % release) + + +# Commit next snapshot files +def commit_snapshot(): + run('git commit -m "prepare for next development iteration"') + + +# Put the version tag on on the current commit +def tag_release(release): + run('git tag -a v%s -m "Tag release version %s"' % (release, release)) + + +# Checkout a given branch +def git_checkout(branch): + run('git checkout %s' % branch) + + +# Merge the release branch with the actual branch +def git_merge(src_branch, release_version): + git_checkout(src_branch) + run('git merge %s' % release_branch(src_branch, release_version)) + + +# Push the actual branch and master branch +def git_push(remote, src_branch, release_version, dry_run): + if not dry_run: + run('git push %s %s master' % (remote, src_branch)) # push the commit and the master + run('git push %s v%s' % (remote, release_version)) # push the tag else: - log('Skip SmokeTest for [%s]' % release_file) - continue # nothing to do here - es_run_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release), 'bin/elasticsearch') - print(' Smoke testing package [%s]' % release_file) - es_plugin_path = os.path.join(tmp_dir, 'elasticsearch-%s' % (release),'bin/plugin') - plugin_names = {} - for name, plugin in plugins: - print(' Install plugin [%s] from [%s]' % (name, plugin)) - run('%s; %s %s %s' % (java_exe(), es_plugin_path, '-install', plugin)) - plugin_names[name] = True + print(' dryrun [True] -- skipping push to remote %s %s master' % (remote, src_branch)) - if release.startswith("0.90."): - background = '' # 0.90.x starts in background automatically - else: - background = '-d' - print(' Starting elasticsearch deamon from [%s]' % os.path.join(tmp_dir, 'elasticsearch-%s' % release)) - run('%s; %s -Des.node.name=smoke_tester -Des.cluster.name=prepare_release -Des.discovery.zen.ping.multicast.enabled=false -Des.script.inline=on -Des.script.indexed=on %s' - % (java_exe(), es_run_path, background)) - conn = HTTPConnection('127.0.0.1', 9200, 20); - wait_for_node_startup() - try: - try: - conn.request('GET', '') - res = conn.getresponse() - if res.status == 200: - version = json.loads(res.read().decode("utf-8"))['version'] - if release != version['number']: - raise RuntimeError('Expected version [%s] but was [%s]' % (release, version['number'])) - if version['build_snapshot']: - raise RuntimeError('Expected non snapshot version') - if version['build_hash'].strip() != expected_hash: - raise RuntimeError('HEAD hash does not match expected [%s] but got [%s]' % (expected_hash, version['build_hash'])) - print(' Running REST Spec tests against package [%s]' % release_file) - run_mvn('test -Dtests.cluster=%s -Dtests.class=*.*RestTests' % ("127.0.0.1:9300")) - print(' Verify if plugins are listed in _nodes') - conn.request('GET', '/_nodes?plugin=true&pretty=true') - res = conn.getresponse() - if res.status == 200: - nodes = json.loads(res.read().decode("utf-8"))['nodes'] - for _, node in nodes.items(): - node_plugins = node['plugins'] - for node_plugin in node_plugins: - if not plugin_names.get(node_plugin['name'], False): - raise RuntimeError('Unexpeced plugin %s' % node_plugin['name']) - del plugin_names[node_plugin['name']] - if plugin_names: - raise RuntimeError('Plugins not loaded %s' % list(plugin_names.keys())) - else: - raise RuntimeError('Expected HTTP 200 but got %s' % res.status) - else: - raise RuntimeError('Expected HTTP 200 but got %s' % res.status) - finally: - conn.request('POST', '/_cluster/nodes/_local/_shutdown') - time.sleep(1) # give the node some time to shut down - if conn.getresponse().status != 200: - raise RuntimeError('Expected HTTP 200 but got %s on node shutdown' % res.status) +########################################################## +# +# Maven commands +# +########################################################## +# Run a given maven command +def run_mvn(*cmd): + for c in cmd: + run('%s; %s -f %s %s' % (java_exe(), MVN, POM_FILE, c)) - finally: - conn.close() - shutil.rmtree(tmp_dir) -def merge_tag_push(remote, src_branch, release_version, dry_run): - run('git checkout %s' % src_branch) - run('git merge %s' % release_branch(release_version)) - run('git tag v%s' % release_version) - if not dry_run: - run('git push %s %s' % (remote, src_branch)) # push the commit - run('git push %s v%s' % (remote, release_version)) # push the tag - else: - print(' dryrun [True] -- skipping push to remote %s' % remote) - -def publish_artifacts(artifacts, base='elasticsearch/elasticsearch', dry_run=True): - location = os.path.dirname(os.path.realpath(__file__)) - for artifact in artifacts: +# Run deploy or package depending on dry_run +# Default to run mvn package +# When run_tests=True a first mvn clean test is run +def build_release(run_tests=False, dry_run=True): + target = 'deploy' + tests = '-DskipTests' + if run_tests: + tests = '' if dry_run: - print('Skip Uploading %s to Amazon S3' % artifact) - else: - print('Uploading %s to Amazon S3' % artifact) - # requires boto to be installed but it is not available on python3k yet so we use a dedicated tool - run('python %s/upload-s3.py --file %s ' % (location, os.path.abspath(artifact))) + target = 'package' + run_mvn('clean %s %s' % (target, tests)) -def publish_repositories(version, dry_run=True): - if dry_run: - print('Skipping package repository update') - else: - print('Triggering repository update - calling dev-tools/build_repositories.sh %s' % version) - # src_branch is a version like 1.5/1.6/2.0/etc.. so we can use this - run('dev-tools/build_repositories.sh %s' % src_branch) def print_sonatype_notice(): - settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') - if os.path.isfile(settings): - with open(settings, encoding='utf-8') as settings_file: - for line in settings_file: - if line.strip() == 'sonatype-nexus-snapshots': - # moving out - we found the indicator no need to print the warning - return - print(""" + settings = os.path.join(os.path.expanduser('~'), '.m2/settings.xml') + if os.path.isfile(settings): + with open(settings, encoding='utf-8') as settings_file: + for line in settings_file: + if line.strip() == 'sonatype-nexus-snapshots': + # moving out - we found the indicator no need to print the warning + return + print(""" NOTE: No sonatype settings detected, make sure you have configured your sonatype credentials in '~/.m2/settings.xml': @@ -548,256 +358,146 @@ def print_sonatype_notice(): """) -def check_command_exists(name, cmd): - try: - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - raise RuntimeError('Could not run command %s - please make sure it is installed' % (name)) -VERSION_FILE = 'src/main/java/org/elasticsearch/Version.java' -POM_FILE = 'pom.xml' - -# finds the highest available bwc version to test against -def find_bwc_version(release_version, bwc_dir='backwards'): - log(' Lookup bwc version in directory [%s]' % bwc_dir) - bwc_version = None - if os.path.exists(bwc_dir) and os.path.isdir(bwc_dir): - max_version = [int(x) for x in release_version.split('.')] - for dir in os.listdir(bwc_dir): - if os.path.isdir(os.path.join(bwc_dir, dir)) and dir.startswith('elasticsearch-'): - version = [int(x) for x in dir[len('elasticsearch-'):].split('.')] - if version < max_version: # bwc tests only against smaller versions - if (not bwc_version) or version > [int(x) for x in bwc_version.split('.')]: - bwc_version = dir[len('elasticsearch-'):] - log(' Using bwc version [%s]' % bwc_version) - else: - log(' bwc directory [%s] does not exists or is not a directory - skipping' % bwc_dir) - return bwc_version - -def ensure_checkout_is_clean(branchName): - # Make sure no local mods: - s = subprocess.check_output('git diff --shortstat', shell=True) - if len(s) > 0: - raise RuntimeError('git diff --shortstat is non-empty: got:\n%s' % s) - - # Make sure no untracked files: - s = subprocess.check_output('git status', shell=True).decode('utf-8', errors='replace') - if 'Untracked files:' in s: - raise RuntimeError('git status shows untracked files: got:\n%s' % s) - - # Make sure we are on the right branch (NOTE: a bit weak, since we default to current branch): - if 'On branch %s' % branchName not in s: - raise RuntimeError('git status does not show branch %s: got:\n%s' % (branchName, s)) - - # Make sure we have all changes from origin: - if 'is behind' in s: - raise RuntimeError('git status shows not all changes pulled from origin; try running "git pull origin %s": got:\n%s' % (branchName, s)) - - # Make sure we no local unpushed changes (this is supposed to be a clean area): - if 'is ahead' in s: - raise RuntimeError('git status shows local commits; try running "git fetch origin", "git checkout %s", "git reset --hard origin/%s": got:\n%s' % (branchName, branchName, s)) - -# Checks all source files for //NORELEASE comments -def check_norelease(path='src'): - pattern = re.compile(r'\bnorelease\b', re.IGNORECASE) - for root, _, file_names in os.walk(path): - for file_name in fnmatch.filter(file_names, '*.java'): - full_path = os.path.join(root, file_name) - line_number = 0 - with open(full_path, 'r', encoding='utf-8') as current_file: - for line in current_file: - line_number = line_number + 1 - if pattern.search(line): - raise RuntimeError('Found //norelease comment in %s line %s' % (full_path, line_number)) - -def run_and_print(text, run_function): - try: - print(text, end='') - run_function() - print(COLOR_OK + 'OK' + COLOR_END) - return True - except RuntimeError: - print(COLOR_FAIL + 'NOT OK' + COLOR_END) - return False - -def check_env_var(text, env_var): - try: - print(text, end='') - env[env_var] - print(COLOR_OK + 'OK' + COLOR_END) - return True - except KeyError: - print(COLOR_FAIL + 'NOT OK' + COLOR_END) - return False - -def check_environment_and_commandline_tools(check_only): - checks = list() - checks.append(check_env_var('Checking for AWS env configuration AWS_SECRET_ACCESS_KEY_ID... ', 'AWS_SECRET_ACCESS_KEY')) - checks.append(check_env_var('Checking for AWS env configuration AWS_ACCESS_KEY_ID... ', 'AWS_ACCESS_KEY_ID')) - checks.append(check_env_var('Checking for SONATYPE env configuration SONATYPE_USERNAME... ', 'SONATYPE_USERNAME')) - checks.append(check_env_var('Checking for SONATYPE env configuration SONATYPE_PASSWORD... ', 'SONATYPE_PASSWORD')) - checks.append(check_env_var('Checking for GPG env configuration GPG_KEY_ID... ', 'GPG_KEY_ID')) - checks.append(check_env_var('Checking for GPG env configuration GPG_PASSPHRASE... ', 'GPG_PASSPHRASE')) - checks.append(check_env_var('Checking for S3 repo upload env configuration S3_BUCKET_SYNC_TO... ', 'S3_BUCKET_SYNC_TO')) - checks.append(check_env_var('Checking for git env configuration GIT_AUTHOR_NAME... ', 'GIT_AUTHOR_NAME')) - checks.append(check_env_var('Checking for git env configuration GIT_AUTHOR_EMAIL... ', 'GIT_AUTHOR_EMAIL')) - - checks.append(run_and_print('Checking command: rpm... ', partial(check_command_exists, 'rpm', 'rpm --version'))) - checks.append(run_and_print('Checking command: dpkg... ', partial(check_command_exists, 'dpkg', 'dpkg --version'))) - checks.append(run_and_print('Checking command: gpg... ', partial(check_command_exists, 'gpg', 'gpg --version'))) - checks.append(run_and_print('Checking command: expect... ', partial(check_command_exists, 'expect', 'expect -v'))) - checks.append(run_and_print('Checking command: createrepo... ', partial(check_command_exists, 'createrepo', 'createrepo --version'))) - checks.append(run_and_print('Checking command: s3cmd... ', partial(check_command_exists, 's3cmd', 's3cmd --version'))) - checks.append(run_and_print('Checking command: apt-ftparchive... ', partial(check_command_exists, 'apt-ftparchive', 'apt-ftparchive --version'))) - - # boto, check error code being returned - location = os.path.dirname(os.path.realpath(__file__)) - command = 'python %s/upload-s3.py -h' % (location) - checks.append(run_and_print('Testing boto python dependency... ', partial(check_command_exists, 'python-boto', command))) - - checks.append(run_and_print('Checking java version... ', partial(verify_java_version, '1.7'))) - checks.append(run_and_print('Checking java mvn version... ', partial(verify_mvn_java_version, '1.7', MVN))) - - if check_only: - sys.exit(0) - - if False in checks: - print("Exiting due to failing checks") - sys.exit(0) +# we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml +print_sonatype_notice() if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Release') - parser.add_argument('--branch', '-b', metavar='RELEASE_BRANCH', default=get_current_branch(), - help='The branch to release from. Defaults to the current branch.') - parser.add_argument('--cpus', '-c', metavar='1', default=1, - help='The number of cpus to use for running the test. Default is [1]') - parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', - help='Skips tests before release. Tests are run by default.') - parser.set_defaults(tests=True) - parser.add_argument('--remote', '-r', metavar='origin', default='origin', - help='The remote to push the release commit and tag to. Default is [origin]') - parser.add_argument('--publish', '-d', dest='dryrun', action='store_false', - help='Publishes the release. Disable by default.') - parser.add_argument('--smoke', '-s', dest='smoke', default='', - help='Smoke tests the given release') - parser.add_argument('--bwc', '-w', dest='bwc', metavar='backwards', default='backwards', - help='Backwards compatibility version path to use to run compatibility tests against') - parser.add_argument('--check-only', dest='check_only', action='store_true', - help='Checks and reports for all requirements and then exits') + parser = argparse.ArgumentParser(description='Builds and publishes a Elasticsearch Plugin Release') + parser.add_argument('--branch', '-b', metavar='master', default=get_current_branch(), + help='The branch to release from. Defaults to the current branch.') + parser.add_argument('--skiptests', '-t', dest='tests', action='store_false', + help='Skips tests before release. Tests are run by default.') + parser.set_defaults(tests=True) + parser.add_argument('--remote', '-r', metavar='origin', default='origin', + help='The remote to push the release commit and tag to. Default is [origin]') + parser.add_argument('--publish', '-p', dest='dryrun', action='store_false', + help='Publishes the release. Disable by default.') + parser.add_argument('--disable_mail', '-dm', dest='mail', action='store_false', + help='Do not send a release email. Email is sent by default.') - parser.set_defaults(dryrun=True) - parser.set_defaults(smoke=None) - parser.set_defaults(check_only=False) - args = parser.parse_args() - bwc_path = args.bwc - src_branch = args.branch - remote = args.remote - run_tests = args.tests - dry_run = args.dryrun - cpus = args.cpus - build = not args.smoke - smoke_test_version = args.smoke + parser.set_defaults(dryrun=True) + parser.set_defaults(mail=True) + args = parser.parse_args() - check_environment_and_commandline_tools(args.check_only) + src_branch = args.branch + remote = args.remote + run_tests = args.tests + dry_run = args.dryrun + mail = args.mail - # we print a notice if we can not find the relevant infos in the ~/.m2/settings.xml - print_sonatype_notice() + if src_branch == 'master': + raise RuntimeError('Can not release the master branch. You need to create another branch before a release') - # we require to build with 1.7 - verify_java_version('1.7') - verify_mvn_java_version('1.7', MVN) - - if os.path.exists(LOG): - raise RuntimeError('please remove old release log %s first' % LOG) - - if not dry_run: - print('WARNING: dryrun is set to "false" - this will push and publish the release') - input('Press Enter to continue...') - - print(''.join(['-' for _ in range(80)])) - print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) - print(' JAVA_HOME is [%s]' % JAVA_HOME) - print(' Running with maven command: [%s] ' % (MVN)) - if build: - check_norelease(path='src') - ensure_checkout_is_clean(src_branch) - verify_lucene_version() - release_version = find_release_version(src_branch) - ensure_no_open_tickets(release_version) if not dry_run: - smoke_test_version = release_version - head_hash = get_head_hash() - run_mvn('clean') # clean the env! + print('WARNING: dryrun is set to "false" - this will push and publish the release') + input('Press Enter to continue...') + + print(''.join(['-' for _ in range(80)])) + print('Preparing Release from branch [%s] running tests: [%s] dryrun: [%s]' % (src_branch, run_tests, dry_run)) + print(' JAVA_HOME is [%s]' % JAVA_HOME) + print(' Running with maven command: [%s] ' % (MVN)) + + release_version = find_release_version(src_branch) + artifact_id = find_from_pom('artifactId') + artifact_name = find_from_pom('name') + artifact_description = find_from_pom('description') + elasticsearch_version = find_from_pom('elasticsearch.version') + print(' Artifact Id: [%s]' % artifact_id) print(' Release version: [%s]' % release_version) - create_release_branch(remote, src_branch, release_version) - print(' Created release branch [%s]' % (release_branch(release_version))) + print(' Elasticsearch: [%s]' % elasticsearch_version) + if elasticsearch_version.find('-SNAPSHOT') != -1: + raise RuntimeError('Can not release with a SNAPSHOT elasticsearch dependency: %s' % elasticsearch_version) + + # extract snapshot + default_snapshot_version = guess_snapshot(release_version) + snapshot_version = input('Enter next snapshot version [%s]:' % default_snapshot_version) + snapshot_version = snapshot_version or default_snapshot_version + + print(' Next version: [%s-SNAPSHOT]' % snapshot_version) + print(' Artifact Name: [%s]' % artifact_name) + print(' Artifact Description: [%s]' % artifact_description) + + if not dry_run: + smoke_test_version = release_version + + try: + git_checkout(src_branch) + version_hash = get_head_hash() + run_mvn('clean') # clean the env! + create_release_branch(remote, src_branch, release_version) + print(' Created release branch [%s]' % (release_branch(src_branch, release_version))) + except RuntimeError: + print('Logs:') + with open(LOG, 'r') as log_file: + print(log_file.read()) + sys.exit(-1) + success = False try: - pending_files = [POM_FILE, VERSION_FILE] - remove_maven_snapshot(POM_FILE, release_version) - remove_version_snapshot(VERSION_FILE, release_version) - print(' Done removing snapshot version') - add_pending_files(*pending_files) # expects var args use * to expand - commit_release(release_version) - pending_files = update_reference_docs(release_version) - version_head_hash = None - # split commits for docs and version to enable easy cherry-picking - if pending_files: - add_pending_files(*pending_files) # expects var args use * to expand - commit_feature_flags(release_version) - version_head_hash = get_head_hash() - print(' Committed release version [%s]' % release_version) - print(''.join(['-' for _ in range(80)])) - print('Building Release candidate') - input('Press Enter to continue...') - if not dry_run: - print(' Running maven builds now and publish to Sonatype - run-tests [%s]' % run_tests) - else: - print(' Running maven builds now run-tests [%s]' % run_tests) - build_release(run_tests=run_tests, dry_run=dry_run, cpus=cpus, bwc_version=find_bwc_version(release_version, bwc_path)) - artifacts = get_artifacts(release_version) - print('Checking if all artifacts contain the same jars') - check_artifacts_for_same_jars(artifacts) - artifacts_and_checksum = generate_checksums(artifacts) - smoke_test_release(release_version, artifacts, get_head_hash(), PLUGINS) - print(''.join(['-' for _ in range(80)])) - print('Finish Release -- dry_run: %s' % dry_run) - input('Press Enter to continue...') - print(' merge release branch, tag and push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) - merge_tag_push(remote, src_branch, release_version, dry_run) - print(' publish artifacts to S3 -- dry_run: %s' % dry_run) - publish_artifacts(artifacts_and_checksum, dry_run=dry_run) - print(' Updating package repositories -- dry_run: %s' % dry_run) - publish_repositories(src_branch, dry_run=dry_run) - cherry_pick_command = '.' - if version_head_hash: - cherry_pick_command = ' and cherry-pick the documentation changes: \'git cherry-pick %s\' to the development branch' % (version_head_hash) - pending_msg = """ - Release successful pending steps: - * create a new vX.Y.Z label on github for the next release, with label color #dddddd (https://github.com/elastic/elasticsearch/labels) - * publish the maven artifacts on Sonatype: https://oss.sonatype.org/index.html - - here is a guide: http://central.sonatype.org/pages/releasing-the-deployment.html - * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/elasticsearch/%(version)s - * announce the release on the website / blog post - * tweet about the release - * announce the release in the google group/mailinglist - * Move to a Snapshot version to the current branch for the next point release%(cherry_pick)s - """ - print(pending_msg % { 'version' : release_version, 'cherry_pick' : cherry_pick_command} ) - success = True + ######################################## + # Start update process in version branch + ######################################## + pending_files = [POM_FILE] + remove_maven_snapshot(POM_FILE, release_version) + print(' Done removing snapshot version') + add_pending_files(*pending_files) # expects var args use * to expand + commit_release(artifact_id, release_version) + print(' Committed release version [%s]' % release_version) + print(''.join(['-' for _ in range(80)])) + print('Building Release candidate') + input('Press Enter to continue...') + if not dry_run: + print(' Running maven builds now and publish to sonatype - run-tests [%s]' % run_tests) + else: + print(' Running maven builds now run-tests [%s]' % run_tests) + build_release(run_tests=run_tests, dry_run=dry_run) + + print(''.join(['-' for _ in range(80)])) + + print('Finish Release -- dry_run: %s' % dry_run) + input('Press Enter to continue...') + + print(' merge release branch') + git_merge(src_branch, release_version) + print(' tag') + tag_release(release_version) + + add_maven_snapshot(POM_FILE, release_version, snapshot_version) + add_pending_files(*pending_files) + commit_snapshot() + + print(' push to %s %s -- dry_run: %s' % (remote, src_branch, dry_run)) + git_push(remote, src_branch, release_version, dry_run) + + pending_msg = """ +Release successful pending steps: + * close and release sonatype repo: https://oss.sonatype.org/ + * check if the release is there https://oss.sonatype.org/content/repositories/releases/org/elasticsearch/%(artifact_id)s/%(version)s +""" + print(pending_msg % {'version': release_version, + 'artifact_id': artifact_id}) + success = True finally: - if not success: - run('git reset --hard HEAD') - run('git checkout %s' % src_branch) - elif dry_run: - run('git reset --hard %s' % head_hash) - run('git tag -d v%s' % release_version) - # we delete this one anyways - run('git branch -D %s' % (release_branch(release_version))) - else: - print("Skipping build - smoketest only against version %s" % smoke_test_version) - run_mvn('clean') # clean the env! - - if smoke_test_version: - fetch(remote) - download_and_verify(smoke_test_version, artifact_names(smoke_test_version), plugins=PLUGINS) + if not success: + print('Logs:') + with open(LOG, 'r') as log_file: + print(log_file.read()) + git_checkout(src_branch) + run('git reset --hard %s' % version_hash) + try: + run('git tag -d v%s' % release_version) + except RuntimeError: + pass + elif dry_run: + print('End of dry_run') + input('Press Enter to reset changes...') + git_checkout(src_branch) + run('git reset --hard %s' % version_hash) + run('git tag -d v%s' % release_version) + + # we delete this one anyways + run('git branch -D %s' % (release_branch(src_branch, release_version))) + + # Checkout the branch we started from + git_checkout(src_branch) diff --git a/dev-tools/pom.xml b/dev-tools/pom.xml new file mode 100644 index 00000000000..bebf7d45cfc --- /dev/null +++ b/dev-tools/pom.xml @@ -0,0 +1,37 @@ + + 4.0.0 + org.elasticsearch + dev-tools + 2.0.0-SNAPSHOT + Elasticsearch Build Resources + + org.sonatype.oss + oss-parent + 7 + + + + UTF-8 + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + 1.5 + + + + bundle + + + + + + **/* + + + + + + diff --git a/dev-tools/src/main/resources/forbidden/all-signatures.txt b/dev-tools/src/main/resources/forbidden/all-signatures.txt new file mode 100644 index 00000000000..3b5eb7a0b03 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/all-signatures.txt @@ -0,0 +1,47 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +@defaultMessage Convert to URI +java.net.URL#getPath() +java.net.URL#getFile() + +@defaultMessage Use java.nio.file instead of java.io.File API +java.util.jar.JarFile +java.util.zip.ZipFile +java.io.File +java.io.FileInputStream +java.io.FileOutputStream +java.io.PrintStream#(java.lang.String,java.lang.String) +java.io.PrintWriter#(java.lang.String,java.lang.String) +java.util.Formatter#(java.lang.String,java.lang.String,java.util.Locale) +java.io.RandomAccessFile +java.nio.file.Path#toFile() + +@defaultMessage Don't use deprecated lucene apis +org.apache.lucene.index.DocsEnum +org.apache.lucene.index.DocsAndPositionsEnum +org.apache.lucene.queries.TermFilter +org.apache.lucene.queries.TermsFilter +org.apache.lucene.search.TermRangeFilter +org.apache.lucene.search.NumericRangeFilter +org.apache.lucene.search.PrefixFilter + +java.nio.file.Paths @ Use PathUtils.get instead. +java.nio.file.FileSystems#getDefault() @ use PathUtils.getDefault instead. + +@defaultMessage Specify a location for the temp file/directory instead. +java.nio.file.Files#createTempDirectory(java.lang.String,java.nio.file.attribute.FileAttribute[]) +java.nio.file.Files#createTempFile(java.lang.String,java.lang.String,java.nio.file.attribute.FileAttribute[]) diff --git a/dev-tools/src/main/resources/forbidden/core-signatures.txt b/dev-tools/src/main/resources/forbidden/core-signatures.txt new file mode 100644 index 00000000000..796f3eec850 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/core-signatures.txt @@ -0,0 +1,86 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +# For shaded dependencies, please put signatures in third-party-shaded.txt +# and third-party-unshaded.txt instead of here. + +@defaultMessage spawns threads with vague names; use a custom thread factory and name threads so that you can tell (by its name) which executor it is associated with + +java.util.concurrent.Executors#newFixedThreadPool(int) +java.util.concurrent.Executors#newSingleThreadExecutor() +java.util.concurrent.Executors#newCachedThreadPool() +java.util.concurrent.Executors#newSingleThreadScheduledExecutor() +java.util.concurrent.Executors#newScheduledThreadPool(int) +java.util.concurrent.Executors#defaultThreadFactory() +java.util.concurrent.Executors#privilegedThreadFactory() + +java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars +java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars + +java.io.StringReader#(java.lang.String) @ Use FastStringReader instead + +@defaultMessage Reference management is tricky, leave it to SearcherManager +org.apache.lucene.index.IndexReader#decRef() +org.apache.lucene.index.IndexReader#incRef() +org.apache.lucene.index.IndexReader#tryIncRef() + +@defaultMessage Pass the precision step from the mappings explicitly instead +org.apache.lucene.search.NumericRangeQuery#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) +org.apache.lucene.search.NumericRangeQuery#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) +org.apache.lucene.search.NumericRangeQuery#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) +org.apache.lucene.search.NumericRangeQuery#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) +org.apache.lucene.search.NumericRangeFilter#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) +org.apache.lucene.search.NumericRangeFilter#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) +org.apache.lucene.search.NumericRangeFilter#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) +org.apache.lucene.search.NumericRangeFilter#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) + +@defaultMessage Only use wait / notify when really needed try to use concurrency primitives, latches or callbacks instead. +java.lang.Object#wait() +java.lang.Object#wait(long) +java.lang.Object#wait(long,int) +java.lang.Object#notify() +java.lang.Object#notifyAll() + +@defaultMessage Beware of the behavior of this method on MIN_VALUE +java.lang.Math#abs(int) +java.lang.Math#abs(long) + +@defaultMessage Please do not try to stop the world +java.lang.System#gc() + +@defaultMessage Use Channels.* methods to write to channels. Do not write directly. +java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer) +java.nio.channels.FileChannel#write(java.nio.ByteBuffer, long) +java.nio.channels.GatheringByteChannel#write(java.nio.ByteBuffer[], int, int) +java.nio.channels.GatheringByteChannel#write(java.nio.ByteBuffer[]) +java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer) +java.nio.channels.ScatteringByteChannel#read(java.nio.ByteBuffer[]) +java.nio.channels.ScatteringByteChannel#read(java.nio.ByteBuffer[], int, int) +java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long) + +@defaultMessage Use Lucene.parseLenient instead it strips off minor version +org.apache.lucene.util.Version#parseLeniently(java.lang.String) + +@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#estimatedTimeInMillisCounter instead +org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread() +org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter() + +@defaultMessage Don't interrupt threads use FutureUtils#cancel(Future) instead +java.util.concurrent.Future#cancel(boolean) + +@defaultMessage Don't try reading from paths that are not configured in Environment, resolve from Environment instead +org.elasticsearch.common.io.PathUtils#get(java.lang.String, java.lang.String[]) +org.elasticsearch.common.io.PathUtils#get(java.net.URI) diff --git a/dev-tools/src/main/resources/forbidden/test-signatures.txt b/dev-tools/src/main/resources/forbidden/test-signatures.txt new file mode 100644 index 00000000000..7ea2c6a45f1 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/test-signatures.txt @@ -0,0 +1,20 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +com.carrotsearch.randomizedtesting.RandomizedTest#globalTempDir() @ Use newTempDirPath() instead +com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded seeds + +org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead diff --git a/dev-tools/src/main/resources/forbidden/third-party-shaded-signatures.txt b/dev-tools/src/main/resources/forbidden/third-party-shaded-signatures.txt new file mode 100644 index 00000000000..798265d8ad9 --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/third-party-shaded-signatures.txt @@ -0,0 +1,22 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +@defaultMessage Use Long.compare instead we are on Java7 +org.elasticsearch.common.primitives.Longs#compare(long,long) + +@defaultMessage unsafe encoders/decoders have problems in the lzf compress library. Use variants of encode/decode functions which take Encoder/Decoder. +org.elasticsearch.common.compress.lzf.impl.UnsafeChunkDecoder#() +org.elasticsearch.common.compress.lzf.util.ChunkDecoderFactory#optimalInstance() diff --git a/dev-tools/src/main/resources/forbidden/third-party-unshaded-signatures.txt b/dev-tools/src/main/resources/forbidden/third-party-unshaded-signatures.txt new file mode 100644 index 00000000000..379951e8c3a --- /dev/null +++ b/dev-tools/src/main/resources/forbidden/third-party-unshaded-signatures.txt @@ -0,0 +1,60 @@ +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on +# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +@defaultMessage Use Long.compare instead we are on Java7 +com.google.common.primitives.Longs#compare(long,long) + +@defaultMessage unsafe encoders/decoders have problems in the lzf compress library. Use variants of encode/decode functions which take Encoder/Decoder. +com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int) +com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int) +com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.impl.UnsafeChunkDecoder#() +com.ning.compress.lzf.parallel.CompressTask +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance() +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int) +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int) +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(com.ning.compress.BufferRecycler) +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.util.ChunkDecoderFactory#optimalInstance() +com.ning.compress.lzf.util.LZFFileInputStream#(java.io.File) +com.ning.compress.lzf.util.LZFFileInputStream#(java.io.FileDescriptor) +com.ning.compress.lzf.util.LZFFileInputStream#(java.lang.String) +com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File) +com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File, boolean) +com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.FileDescriptor) +com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String) +com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String, boolean) +com.ning.compress.lzf.LZFEncoder#encode(byte[]) +com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int) +com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int) +com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.LZFCompressingInputStream#(java.io.InputStream) +com.ning.compress.lzf.LZFDecoder#fastDecoder() +com.ning.compress.lzf.LZFDecoder#decode(byte[]) +com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int) +com.ning.compress.lzf.LZFDecoder#decode(byte[], byte[]) +com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int, byte[]) +com.ning.compress.lzf.LZFInputStream#(java.io.InputStream) +com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, boolean) +com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler, boolean) +com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream) +com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream, com.ning.compress.BufferRecycler) +com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler) +com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler, com.ning.compress.BufferRecycler) diff --git a/dev-tools/src/main/resources/license-check/elasticsearch_license_header.txt b/dev-tools/src/main/resources/license-check/elasticsearch_license_header.txt new file mode 100644 index 00000000000..ca943323401 --- /dev/null +++ b/dev-tools/src/main/resources/license-check/elasticsearch_license_header.txt @@ -0,0 +1,16 @@ +Licensed to Elasticsearch under one or more contributor +license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright +ownership. Elasticsearch licenses this file to you under +the Apache License, Version 2.0 (the "License"); you may +not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. diff --git a/dev-tools/src/main/resources/license-check/license_header_definition.xml b/dev-tools/src/main/resources/license-check/license_header_definition.xml new file mode 100644 index 00000000000..1cc53bda001 --- /dev/null +++ b/dev-tools/src/main/resources/license-check/license_header_definition.xml @@ -0,0 +1,13 @@ + + + + /* + * + */ + + (\s|\t)*/\*.*$ + .*\*/(\s|\t)*$ + false + true + + diff --git a/dev-tools/src/main/resources/pmd/custom.xml b/dev-tools/src/main/resources/pmd/custom.xml new file mode 100644 index 00000000000..58788337ae9 --- /dev/null +++ b/dev-tools/src/main/resources/pmd/custom.xml @@ -0,0 +1,20 @@ + + + + Default ruleset for elasticsearch server project + + + + + + + + + + + + + diff --git a/dev-tools/src/main/resources/shared-test-resources/log4j.properties b/dev-tools/src/main/resources/shared-test-resources/log4j.properties new file mode 100644 index 00000000000..22f54ef68e5 --- /dev/null +++ b/dev-tools/src/main/resources/shared-test-resources/log4j.properties @@ -0,0 +1,9 @@ +es.logger.level=INFO +log4j.rootLogger=${es.logger.level}, out + +log4j.logger.org.apache.http=INFO, out +log4j.additivity.org.apache.http=false + +log4j.appender.out=org.apache.log4j.ConsoleAppender +log4j.appender.out.layout=org.apache.log4j.PatternLayout +log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/dev-tools/src/main/resources/site/site_en.xml b/dev-tools/src/main/resources/site/site_en.xml new file mode 100644 index 00000000000..f3011083e06 --- /dev/null +++ b/dev-tools/src/main/resources/site/site_en.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/plugins/pom.xml b/plugins/pom.xml new file mode 100644 index 00000000000..50236253332 --- /dev/null +++ b/plugins/pom.xml @@ -0,0 +1,122 @@ + + + 4.0.0 + + org.elasticsearch + elasticsearch-plugin + 2.0.0-SNAPSHOT + pom + Elasticsearch Plugin POM + 2009 + + + org.elasticsearch + elasticsearch-parent + 2.0.0-SNAPSHOT + + + + ${basedir}/src/main/assemblies/plugin.xml + false + + + + + + org.hamcrest + hamcrest-all + + + junit + junit + + + org.apache.lucene + lucene-test-framework + + + org.elasticsearch + elasticsearch + + + org.elasticsearch + elasticsearch + test-jar + + + log4j + log4j + test + + + + + + + + src/main/resources + true + + **/*.properties + + + + + + + src/test/java + + **/*.json + **/*.txt + + + + src/test/resources + + elasticsearch.yml + **/*.properties + + + + src/test/resources + true + + elasticsearch.yml + **/*.properties + + + + + ${elasticsearch.tools.directory}/shared-test-resources + false + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + ${elasticsearch.assembly.appendId} + ${project.build.directory}/releases/ + + ${elasticsearch.assembly.descriptor} + + + + + package + + single + + + + + + + + + diff --git a/pom.xml b/pom.xml index b6df03658ef..44c53db8372 100644 --- a/pom.xml +++ b/pom.xml @@ -3,13 +3,21 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + org.elasticsearch - elasticsearch + elasticsearch-parent 2.0.0-SNAPSHOT - jar - Elasticsearch core - Elasticsearch - Open Source, Distributed, RESTful Search Engine + pom + Elasticsearch Parent POM + Elasticsearch Parent POM 2009 + + + org.sonatype.oss + oss-parent + 7 + + The Apache Software License, Version 2.0 @@ -19,38 +27,98 @@ - scm:git:git@github.com:elasticsearch/elasticsearch.git - scm:git:git@github.com:elasticsearch/elasticsearch.git - http://github.com/elasticsearch/elasticsearch + scm:git:git@github.com:elasticsearch/elasticsearch-parent.git + scm:git:git@github.com:elasticsearch/elasticsearch-parent.git + http://github.com/elasticsearch/elasticsearch-parent - - org.elasticsearch - elasticsearch-parent - 2.0.0-SNAPSHOT - - - - unshaded - - /usr/share/elasticsearch - /usr/share/elasticsearch/bin - /etc/elasticsearch - /var/lib/elasticsearch - elasticsearch - elasticsearch - /var/log/elasticsearch - ${packaging.elasticsearch.home.dir}/plugins - /var/run/elasticsearch - /usr/lib/systemd/system - /usr/lib/sysctl.d - /usr/lib/tmpfiles.d - false - dpkg-sig + + 2.0.0-SNAPSHOT + ${java.home}${file.separator}bin${file.separator}java + 1.7 + 1.7 + + + 5.2.0 + 1682335 + 5.2.0-snapshot-${lucene.snapshot.revision} + 2.1.14 + 2.5.3 + 1.6.2 + 1.2.17 + + + ${project.build.directory}/dev-tools + shaded + ${elasticsearch.tools.directory}/license-check/elasticsearch_license_header.txt + ${elasticsearch.tools.directory}/license-check/license_header_definition.xml + + + auto + true + onerror + + ${project.basedir}/backwards + random + random + false + ERROR + 512m + ${basedir}/logs/ + 5 + .local-${elasticsearch.version}-execution-hints.log + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + fail + + elasticsearch-releases + http://maven.elasticsearch.org/releases + + true + + + false + + + + lucene-snapshots + Lucene snapshots + https://download.elasticsearch.org/lucenesnapshots/${lucene.snapshot.revision} + oss-snapshots Sonatype OSS Snapshots @@ -58,1098 +126,1129 @@ - - - org.hamcrest - hamcrest-all - test - - - com.carrotsearch.randomizedtesting - randomizedtesting-runner - test - - - org.apache.lucene - lucene-test-framework - test - - - org.apache.httpcomponents - httpclient - test - - - com.google.jimfs - jimfs - test - + + + + org.elasticsearch + dev-tools + ${elasticsearch.version} + - - org.apache.lucene - lucene-core - compile - - - org.apache.lucene - lucene-backward-codecs - compile - - - org.apache.lucene - lucene-analyzers-common - compile - - - org.apache.lucene - lucene-queries - compile - - - org.apache.lucene - lucene-memory - compile - - - org.apache.lucene - lucene-highlighter - compile - - - org.apache.lucene - lucene-queryparser - compile - - - org.apache.lucene - lucene-suggest - compile - - - org.apache.lucene - lucene-join - compile - - - org.apache.lucene - lucene-spatial - compile - - - org.apache.lucene - lucene-expressions - compile - true - - - com.spatial4j - spatial4j - compile - true - - - com.vividsolutions - jts - compile - true - - - - com.github.spullara.mustache.java - compiler - true - - + + com.carrotsearch.randomizedtesting + randomizedtesting-runner + ${testframework.version} + test + + + org.hamcrest + hamcrest-all + 1.3 + test + - - - com.google.guava - guava - compile - + + junit + junit + 4.11 + test + - - com.carrotsearch - hppc - + + com.google.jimfs + jimfs + 1.0 + test + - - joda-time - joda-time - compile - - - org.joda - joda-convert - compile - + + org.apache.lucene + lucene-test-framework + ${lucene.maven.version} + test + - - com.fasterxml.jackson.core - jackson-core - compile - + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + test-jar + test + - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - compile - + + org.apache.httpcomponents + httpclient + 4.3.5 + - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - - - com.fasterxml.jackson.core - jackson-databind - - - + + org.apache.lucene + lucene-core + ${lucene.maven.version} + + + org.apache.lucene + lucene-backward-codecs + ${lucene.maven.version} + + + org.apache.lucene + lucene-analyzers-common + ${lucene.maven.version} + + + org.apache.lucene + lucene-queries + ${lucene.maven.version} + + + jakarta-regexp + jakarta-regexp + + + + + org.apache.lucene + lucene-memory + ${lucene.maven.version} + + + org.apache.lucene + lucene-highlighter + ${lucene.maven.version} + + + org.apache.lucene + lucene-queryparser + ${lucene.maven.version} + + + jakarta-regexp + jakarta-regexp + + + + + org.apache.lucene + lucene-suggest + ${lucene.maven.version} + + + org.apache.lucene + lucene-join + ${lucene.maven.version} + + + + org.apache.lucene + lucene-spatial + ${lucene.maven.version} + + + org.apache.lucene + lucene-expressions + ${lucene.maven.version} + true + + + com.spatial4j + spatial4j + 0.4.1 + true + + + com.vividsolutions + jts + 1.13 + true + + + xerces + xercesImpl + + + + + + com.github.spullara.mustache.java + compiler + 0.8.13 + true + - - com.fasterxml.jackson.dataformat - jackson-dataformat-cbor - compile - + + + + org.apache.lucene + lucene-analyzers-phonetic + ${lucene.maven.version} + + + + org.apache.lucene + lucene-analyzers-kuromoji + ${lucene.maven.version} + + + + org.apache.lucene + lucene-analyzers-stempel + ${lucene.maven.version} + + + + org.apache.lucene + lucene-analyzers-icu + ${lucene.maven.version} + + + + org.apache.lucene + lucene-analyzers-smartcn + ${lucene.maven.version} + - - io.netty - netty - compile - + + + com.google.guava + guava + 18.0 + - - com.ning - compress-lzf - compile - + + com.carrotsearch + hppc + 0.7.1 + - - com.tdunning - t-digest - compile - + + joda-time + joda-time + + + 2.7 + + + org.joda + joda-convert + 1.2 + - - org.apache.commons - commons-lang3 - + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + - - commons-cli - commons-cli - + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${jackson.version} + - + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + - - org.codehaus.groovy - groovy-all - indy - compile - true - + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${jackson.version} + - - log4j - log4j - compile - true - + + io.netty + netty + 3.10.0.Final + - - log4j - apache-log4j-extras - compile - true - + + com.ning + compress-lzf + 1.0.2 + - - org.slf4j - slf4j-api - compile - true - + + com.tdunning + t-digest + 3.0 + - - net.java.dev.jna - jna - compile - true - + + org.apache.commons + commons-lang3 + 3.3.2 + - - org.fusesource - sigar - compile - true - + + commons-cli + commons-cli + 1.2 + - - - + + + + org.codehaus.groovy + groovy-all + 2.4.0 + indy + true + + + + log4j + log4j + ${log4j.version} + true + + + + log4j + apache-log4j-extras + ${log4j.version} + true + + + + org.slf4j + slf4j-api + ${slf4j.version} + true + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + true + + + + net.java.dev.jna + jna + 4.1.0 + true + + + + org.fusesource + sigar + 1.6.4 + true + + + + + + + + org.jacoco + org.jacoco.agent + runtime + 0.6.4.201312101107 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.codehaus.mojo + buildnumber-maven-plugin + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + + + org.apache.maven.plugins + maven-source-plugin + + + org.apache.maven.plugins + maven-remote-resources-plugin + + + de.thetaphi + forbiddenapis + + + org.apache.maven.plugins + maven-antrun-plugin + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.4 + + + org.apache.maven.plugins + maven-clean-plugin + 2.6.1 + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.plugins + maven-install-plugin + 2.5.2 + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4 + + + enforce-versions + + enforce + + + + + [${maven.compiler.source},) + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + true + 512m + + false + + -XDignore.symbol.file + + + + + com.carrotsearch.randomizedtesting + junit4-maven-plugin + ${testframework.version} + + + tests + test + + junit4 + + + ${jvm.executable} + ${tests.jvm.argline} + 10 + warn + true + ${tests.ifNoTests} + + + + + + + + + + + + + + + + + + + + + + + ${tests.jvms} + + + + + + + **/*Tests.class + **/*Test.class + + + **/Abstract*.class + **/*StressTest.class + + + -Xmx${tests.heap.size} + -Xms${tests.heap.size} + ${java.permGenSpace} + -Djava.library.path=${project.basedir}/lib/sigar + -XX:MaxDirectMemorySize=512m + -Des.logger.prefix= + -XX:+HeapDumpOnOutOfMemoryError + -XX:HeapDumpPath=${tests.heapdump.path} + + ${tests.shuffle} + ${tests.verbose} + ${tests.seed} + ${tests.failfast} + + true + + + ./temp + + ${tests.bwc} + ${tests.bwc.path} + ${tests.bwc.version} + ${tests.jvm.argline} + ${tests.appendseed} + ${tests.iters} + ${tests.maxfailures} + ${tests.failfast} + ${tests.class} + ${tests.method} + ${tests.nightly} + ${tests.verbose} + ${tests.badapples} + ${tests.weekly} + ${tests.awaitsfix} + ${tests.slow} + ${tests.timeoutSuite} + ${tests.showSuccess} + ${tests.integration} + ${tests.thirdparty} + ${tests.config} + ${tests.client.ratio} + ${tests.enable_mock_modules} + ${tests.assertion.disabled} + ${tests.rest} + ${tests.rest.suite} + ${tests.rest.blacklist} + ${tests.rest.spec} + ${tests.network} + ${tests.cluster} + ${tests.heap.size} + ${tests.filter} + ${elasticsearch.version} + ${tests.locale} + ${tests.timezone} + ${project.basedir} + ${settings.localRepository} + ${env.ES_TEST_LOCAL} + ${es.node.mode} + ${es.logger.level} + ${tests.security.manager} + ${tests.compatibility} + true + + true + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar + + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + 1.5 + + + org.elasticsearch:dev-tools:${elasticsearch.version} + + ${elasticsearch.tools.directory} + + false + false + + + + + process + + + + + + de.thetaphi + forbiddenapis + 1.8 - - - ${project.basedir}/src/packaging/common/packaging.properties - - - - - ${project.basedir}/src/main/java - - **/*.json - **/*.yml - - - - ${project.basedir}/src/main/resources - - **/*.* - - true - - - - - - ${project.basedir}/src/test/java - - **/*.json - **/*.yml - **/*.txt - **/*.properties - - true - - - ${project.basedir}/src/test/java - - **/*.gz - - - - ${project.basedir}/src/test/resources - - **/*.* - - - - ${project.basedir}/rest-api-spec - rest-api-spec - - api/*.json - test/**/*.yaml - - - - - ${elasticsearch.tools.directory}/shared-test-resources - false - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.4.0 - - - - exec - - - - - ${jvm.executable} - - -Des.security.manager.enabled=false - -classpath - - org.elasticsearch.bootstrap.Bootstrap - -Xms256m - -Xmx1g - -Djava.awt.headless=true - -XX:+UseParNewGC - -XX:+UseConcMarkSweepGC - -XX:CMSInitiatingOccupancyFraction=75 - -XX:+UseCMSInitiatingOccupancyOnly - -XX:+HeapDumpOnOutOfMemoryError - -XX:+DisableExplicitGC - -Dfile.encoding=UTF-8 - -Djna.nosys=true - -Delasticsearch - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-test-sources - - test-jar - - - - org/elasticsearch/test/**/* - org/elasticsearch/bootstrap/BootstrapForTesting.class - org/elasticsearch/common/cli/CliToolTestCase.class - org/elasticsearch/common/cli/CliToolTestCase$*.class - - - - org/elasticsearch/test/rest/test/**/* - - org/elasticsearch/test/test/**/* - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - - - true - true - true - - - com.google.guava:guava - com.carrotsearch:hppc - com.fasterxml.jackson.core:jackson-core - com.fasterxml.jackson.dataformat:jackson-dataformat-smile - com.fasterxml.jackson.dataformat:jackson-dataformat-yaml - com.fasterxml.jackson.dataformat:jackson-dataformat-cbor - joda-time:joda-time - org.joda:joda-convert - io.netty:netty - com.ning:compress-lzf - com.github.spullara.mustache.java:compiler - com.tdunning:t-digest - org.apache.commons:commons-lang3 - commons-cli:commons-cli - - - - - + + + check-forbidden-apis + + ${maven.compiler.target} + + true + + false + + jsr166e/** + + + + jdk-unsafe + jdk-deprecated + jdk-system-out + + + ${elasticsearch.tools.directory}/forbidden/core-signatures.txt + ${elasticsearch.tools.directory}/forbidden/all-signatures.txt + ${elasticsearch.tools.directory}/forbidden/third-party-${elasticsearch.thirdparty.config}-signatures.txt + + ${forbidden.signatures} + **.SuppressForbidden + + compile + + check + + + + check-forbidden-test-apis + + ${maven.compiler.target} + + true + + false + + + jdk-unsafe + jdk-deprecated + + + ${elasticsearch.tools.directory}/forbidden/test-signatures.txt + ${elasticsearch.tools.directory}/forbidden/all-signatures.txt + + ${forbidden.test.signatures} + **.SuppressForbidden + + test-compile + + testCheck + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + org.apache.maven.plugins + maven-site-plugin + 3.4 + + ${elasticsearch.tools.directory}/site + + + + org.codehaus.mojo + buildnumber-maven-plugin + 1.3 + + + validate + + create + + + + + false + false + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + + + org.vafer + jdeb + 1.3 + + + + org.codehaus.mojo + rpm-maven-plugin + 2.1.2 + + + org.apache.maven.plugins + maven-jar-plugin + 2.5 + + + + true + - true + ${scmBranch} + ${buildNumber} + ${timestamp} + ${elasticsearch.version} + ${lucene.version} + ${maven.compiler.source} + ${maven.compiler.target} - - - - - com.google.common - org.elasticsearch.common - - - com.carrotsearch.hppc - org.elasticsearch.common.hppc - - - jsr166e - org.elasticsearch.common.util.concurrent.jsr166e - - - com.fasterxml.jackson - org.elasticsearch.common.jackson - - - org.joda.time - org.elasticsearch.common.joda.time - - - org.jboss.netty - org.elasticsearch.common.netty - - - com.ning.compress - org.elasticsearch.common.compress - - - com.github.mustachejava - org.elasticsearch.common.mustache - - - com.tdunning.math.stats - org.elasticsearch.common.stats - - - org.apache.commons.lang - org.elasticsearch.common.lang - - - org.apache.commons.cli - org.elasticsearch.common.cli.commons - - - - - *:* - - META-INF/license/** - META-INF/* - META-INF/maven/** - LICENSE - NOTICE - /*.txt - build.properties - - - - io.netty:netty - - org/jboss/netty/** - - - - - - - maven-resources-plugin - 2.6 - - - copy-resources - prepare-package - - copy-resources - - - ${project.build.directory}/bin - - - ${project.basedir}/bin - true - - *.exe - - - - ${project.basedir}/bin - false - - *.exe - - - - - - - - copy-resources-deb - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/deb/ - - ${project.basedir}/src/packaging/common/packaging.properties - ${project.basedir}/src/packaging/deb/packaging.properties - - - - ${project.basedir}/src/packaging/common/ - true - - **/* - - - packaging.properties - - - - ${project.basedir}/src/packaging/deb/ - true - - **/* - - - packaging.properties - - - - ${project.basedir} - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - - - - - - - - copy-resources-rpm - prepare-package - - copy-resources - - - ${project.build.directory}/generated-packaging/rpm/ - - ${project.basedir}/src/packaging/common/packaging.properties - ${project.basedir}/src/packaging/rpm/packaging.properties - - - - src/packaging/common/ - true - - **/* - - - packaging.properties - - - - src/packaging/rpm/ - true - - **/* - - - packaging.properties - - - - ${project.basedir} - true - - bin/elasticsearch - bin/elasticsearch.in.sh - bin/plugin - - - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - false - ${project.build.directory}/releases/ - - ${project.basedir}/src/main/assemblies/targz-bin.xml - ${project.basedir}/src/main/assemblies/zip-bin.xml - - - - - package - - single - - - - - - org.apache.maven.plugins - maven-dependency-plugin - - - prepare-package - - copy-dependencies - - - ${project.build.directory}/lib - runtime - - - - + + + - - - - jdeb - org.vafer - 1.4 - - ${project.build.directory}/releases/${project.artifactId}-${project.version}.deb - ${project.build.directory}/generated-packaging/deb/scripts - - - - package - - jdeb - - - ${deb.sign} - ${gpg.keyring} - ${gpg.key} - ${gpg.passphrase} - ${deb.sign.method} - - - - ${project.build.directory}/generated-packaging/deb/bin - directory - elasticsearch,elasticsearch.in.sh,plugin - - perm - ${packaging.elasticsearch.bin.dir} - 755 - root - root - - - - - ${project.basedir}/config - directory - .DS_Store - - perm - ${packaging.elasticsearch.conf.dir} - root - root - - - - - ${project.build.directory}/generated-packaging/deb/env/elasticsearch - file - - perm - /etc/default - 644 - root - root - - - - - ${project.build.directory}/ - ${project.build.finalName}.jar - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - ${project.basedir}/lib/sigar/ - sigar-*.jar, libsigar-*-linux.* - directory - - perm - ${packaging.elasticsearch.home.dir}/lib/sigar - root - root - - - - ${project.build.directory}/lib - lucene*, *log4j*, jna*, spatial4j*, jts*, groovy*, antlr-runtime*, asm* - directory - - perm - ${packaging.elasticsearch.home.dir}/lib - root - root - - - - - ${project.build.directory}/generated-packaging/deb/init.d/ - directory - .DS_Store - - perm - /etc/init.d - 755 - root - root - - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.service - ${packaging.elasticsearch.systemd.dir}/elasticsearch.service - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/sysctl/elasticsearch.conf - ${packaging.elasticsearch.systemd.sysctl.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/systemd/elasticsearch.conf - ${packaging.elasticsearch.tmpfilesd.dir}/elasticsearch.conf - file - - - - ${project.build.directory}/generated-packaging/deb/lintian - directory - .DS_Store - - perm - /usr/share/lintian/overrides - root - root - - - - - ${project.basedir}/ - *.txt, *.textile - LICENSE.txt, .DS_Store - directory - - perm - ${packaging.elasticsearch.home.dir} - root - root - - - - - ${project.build.directory}/generated-packaging/deb/copyright - /usr/share/doc/elasticsearch/copyright - file - - - - template - - ${packaging.elasticsearch.data.dir} - ${packaging.elasticsearch.log.dir} - ${packaging.elasticsearch.plugins.dir} - ${packaging.elasticsearch.pid.dir} - - - perm - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - + + org.codehaus.mojo + license-maven-plugin + 1.8 + - - - - + + com.mycila + license-maven-plugin + 2.5 + +
${elasticsearch.license.header}
+ + ${elasticsearch.license.headerDefinition} + + + src/main/java/org/elasticsearch/**/*.java + src/test/java/org/elasticsearch/**/*.java + + + + + check-license-headers + compile + + check + + + + - - - org.codehaus.mojo - rpm-maven-plugin - 2.1.2 - - Elasticsearch - Application/Internet - Elasticsearch - /usr - src/changelog - - _unpackaged_files_terminate_build 0 - _binaries_in_noarch_packages_terminate_build 0 - - 644 - 755 - root - root - ${gpg.key} - ${gpg.keyring} - - ${gpg.passphrase} - - - - - ${packaging.elasticsearch.bin.dir}/ - 755 - - - ${project.build.directory}/generated-packaging/rpm/bin - - elasticsearch - elasticsearch.in.sh - plugin - - - - - - - ${packaging.elasticsearch.conf.dir}/ - noreplace - - - ${project.basedir}/config/ - - *.yml - - - - - - - /etc/sysconfig/ - false - noreplace - - - ${project.build.directory}/generated-packaging/rpm/env/ - - elasticsearch - - - - - - - ${packaging.elasticsearch.home.dir}/lib - - - target/lib/ - - lucene* - *log4j* - jna* - spatial4j* - jts* - groovy* - antlr-runtime* - asm* - - - - ${project.build.directory}/ - - ${project.build.finalName}.jar - - - - - - ${packaging.elasticsearch.home.dir}/lib/sigar - - - lib/sigar - - sigar*.jar - libsigar-*-linux.* - - - - - - - /etc/init.d - false - 755 - true - - - ${project.build.directory}/generated-packaging/rpm/init.d - - elasticsearch - - - - - - - ${packaging.elasticsearch.systemd.dir} - false - true - - - ${project.build.directory}/generated-packaging/rpm/systemd - - elasticsearch.service - - - - - - - ${packaging.elasticsearch.systemd.sysctl.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/sysctl - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.tmpfilesd.dir} - true - - - ${project.build.directory}/generated-packaging/rpm/systemd/ - - elasticsearch.conf - - - - - - - ${packaging.elasticsearch.home.dir} - - - . - - LICENSE.txt - NOTICE.txt - README.textile - - - - - - - ${packaging.elasticsearch.data.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.log.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.plugins.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - ${packaging.elasticsearch.pid.dir} - 755 - ${packaging.elasticsearch.user} - ${packaging.elasticsearch.group} - - - - ${project.build.directory}/generated-packaging/rpm/scripts/preinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postinst - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/prerm - utf-8 - - - ${project.build.directory}/generated-packaging/rpm/scripts/postrm - utf-8 - - - + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + [1.0.0,) + + copy-dependencies + + + + + + + + + + de.thetaphi + forbiddenapis + [1.0.0,) + + testCheck + check + + + + + + + + + + exec-maven-plugin + org.codehaus.mojo + [1.0.0,) + + exec + + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + [1.0.0,) + + enforce + + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + [1.0.0,) + + run + + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + [1.0.0,) + + copy-resources + + + + + + + + + com.mycila + license-maven-plugin + [1.0.0,) + + check + + + + + + + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.9 + + eclipse-build + true + false + + - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-package - - test-jar - - - - rest-api-spec/**/* - org/elasticsearch/test/**/* - org/elasticsearch/bootstrap/BootstrapForTesting.class - org/elasticsearch/common/cli/CliToolTestCase.class - org/elasticsearch/common/cli/CliToolTestCase$*.class - - - - org/elasticsearch/test/rest/test/**/* - - org/elasticsearch/test/test/**/* - - - - - - - - - - org.jacoco - jacoco-maven-plugin - - - jsr166e/** - org/apache/lucene/** - - - - - com.mycila - license-maven-plugin - - - - src/main/java/org/elasticsearch/common/inject/** - src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java - src/main/java/org/apache/lucene/**/X*.java - - src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/TDigestState.java - - src/main/java/org/elasticsearch/http/netty/pipelining/** - - - - - + + + org.apache.maven.plugins + maven-jxr-plugin + 2.5 + + + org.apache.maven.plugins + maven-pmd-plugin + 3.3 + + + ${elasticsearch.tools.directory}/pmd/custom.xml + + ${maven.compiler.target} + + + + org.codehaus.mojo + findbugs-maven-plugin + 3.0.0 + + true + target/site + true + 2048 + 1800000 + org.elasticsearch.- + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.7 + + + org.jacoco + jacoco-maven-plugin + 0.7.2.201409121644 + + + default-prepare-agent + + prepare-agent + + + + default-report + prepare-package + + report + + + + default-check + + check + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + print-jvm + validate + + run + + + + Using ${java.runtime.name} ${java.runtime.version} ${java.vendor} + + + + + set-permgen + validate + + run + + + + + + + + + true + + + + check-invalid-patterns + validate + + run + + + + + + + + + + + + + + + + + + + + + + + + The following files contain tabs or + nocommits:${line.separator}${validate.patternsFound} + + + + + + generate-test-resources + create-heapdump-directory + + + + + + + + run + + + + tests-top-hints + test + + ${skipTests} + + false + + + + + + + + + + + + + + + + + + run + + + + + + + org.apache.maven.plugins + maven-invoker-plugin + 1.10 + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + + org.apache.maven.plugins + maven-verifier-plugin + 1.1 + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.15 + + + + + default + + true + + + + + com.mycila + license-maven-plugin + + + + license @@ -1161,5 +1260,158 @@ + + + pom + + false + + ${basedir}/src + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-heapdump-directory + none + + + tests-top-hints + none + + + + + + + + + dev + + + + de.thetaphi + forbiddenapis + + + check-forbidden-apis + none + + + check-forbidden-test-apis + none + + + + + com.mycila + license-maven-plugin + + + check-license-headers + none + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + check-invalid-patterns + none + + + + + + + + + coverage + + + tests.coverage + true + + + + + + org.jacoco + org.jacoco.agent + runtime + test + + + + false + + + + + org.jacoco + jacoco-maven-plugin + + + + + + + static + + + tests.static + true + + + + false + + + + + org.codehaus.mojo + findbugs-maven-plugin + + + + + + + org.apache.maven.plugins + maven-jxr-plugin + + + org.apache.maven.plugins + maven-pmd-plugin + + + org.codehaus.mojo + findbugs-maven-plugin + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.7 + + + + index + + + + + + + + + dev-tools + plugins +