From f1e9292fad91d953c124525c5e7e4a665800c595 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Sat, 13 Feb 2016 16:43:58 -0800 Subject: [PATCH 01/31] Build: Remove legacy testing and releasing scripts. These scripts are no longer needed: * build_randomization.rb - it used by CI a long time ago, but no longer * client_tests_urls.prop - not sure when this was used, but it refers to ancient branches * download-s3.py - replaced by s3cmd in release scripts * upload-s3.py - replaced by s3cmd in release scripts * upgrade-tests.py - these were the old upgrade tests, before the static index bwc tests --- dev-tools/build_randomization.rb | 481 ------------------------------- dev-tools/client_tests_urls.prop | 24 -- dev-tools/download-s3.py | 65 ----- dev-tools/upgrade-tests.py | 319 -------------------- dev-tools/upload-s3.py | 67 ----- 5 files changed, 956 deletions(-) delete mode 100644 dev-tools/build_randomization.rb delete mode 100644 dev-tools/client_tests_urls.prop delete mode 100644 dev-tools/download-s3.py delete mode 100644 dev-tools/upgrade-tests.py delete mode 100644 dev-tools/upload-s3.py diff --git a/dev-tools/build_randomization.rb b/dev-tools/build_randomization.rb deleted file mode 100644 index 4e10e5889d5..00000000000 --- a/dev-tools/build_randomization.rb +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env ruby -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License -# -# NAME -# build_randomization.rb -- Generate property file for the JDK randomization test -# -# SYNOPSIS -# build_randomization.rb [-d] [-l|t] -# -# DESCRIPTION -# This script takes the randomization choices described in RANDOM_CHOICE and generates appropriate JAVA property file 'prop.txt' -# This property file also contain the appropriate JDK selection, randomized. JDK randomization is based on what is available on the Jenkins tools -# directory. This script is used by Jenkins test system to conduct Elasticsearch server randomization testing. -# -# In hash RANDOM_CHOICES, the key of randomization hash maps to key of java property. The value of the hash describes the possible value of the randomization -# -# For example RANDOM_CHOICES = { 'es.node.mode' => {:choices => ['local', 'network'], :method => :get_random_one} } means -# es.node.mode will be set to either 'local' or 'network', each with 50% of probability -# -# OPTIONS SUMMARY -# The options are as follows: -# -# -d, --debug Increase logging verbosity for debugging purpose -# -t, --test Run in test mode. The script will execute unit tests. -# -l, --local Run in local mode. In this mode, directory structure will be created under current directory to mimic -# Jenkins' server directory layout. This mode is mainly used for development. -require 'enumerator' -require 'getoptlong' -require 'log4r' -require 'optparse' -require 'rubygems' -require 'yaml' -include Log4r - -RANDOM_CHOICES = { - 'tests.jvm.argline' => [ - {:choices => ['-server'], :method => 'get_random_one'}, - {:choices => ['-XX:+UseConcMarkSweepGC', '-XX:+UseParallelGC', '-XX:+UseSerialGC', '-XX:+UseG1GC'], :method => 'get_random_one'}, - {:choices => ['-XX:+UseCompressedOops', '-XX:-UseCompressedOops'], :method => 'get_random_one'}, - {:choices => ['-XX:+AggressiveOpts'], :method => 'get_50_percent'} - ], - - 'es.node.mode' => {:choices => ['local', 'network'], :method => 'get_random_one'}, - - # bug forced to be false for now :test_nightly => { :method => :true_or_false}, - 'tests.nightly' => {:selections => false}, - 'tests.heap.size' => {:choices => [512, 1024], :method => :random_heap}, - 'tests.assertion.disabled'=> {:choices => 'org.elasticsearch', :method => 'get_10_percent'}, - 'tests.security.manager' => {:choices => [true, false], :method => 'get_90_percent'}, -} - -L = Logger.new 'test_randomizer' -L.outputters = Outputter.stdout -L.level = INFO -C = {:local => false, :test => false} - - -OptionParser.new do |opts| - opts.banner = "Usage: build_randomization.rb [options]" - - opts.on("-d", "--debug", "Debug mode") do |d| - L.level = DEBUG - end - - opts.on("-l", "--local", "Run in local mode") do |l| - C[:local] = true - end - - opts.on("-t", "--test", "Run unit tests") do |t| - C[:test] = true - end -end.parse! - -class Randomizer - attr_accessor :data_array - - def initialize(data_array) - @data_array = data_array - end - - def true_or_false - [true, false][rand(2)] - end - - def random_heap - inner_data_array = [data_array[0], data_array[1], data_array[0] + rand(data_array[1] - data_array[0])] - "%sm" % inner_data_array[rand(inner_data_array.size)] - end - - def get_random_with_distribution(mdata_array, distribution) - L.debug "randomized distribution data %s" % YAML.dump(mdata_array) - L.debug "randomized distribution distribution %s" % YAML.dump(distribution) - carry = 0 - distribution_map = distribution.enum_for(:each_with_index).map { |x,i| pre_carry = carry ; carry += x; {i => x + pre_carry} } - - random_size = distribution_map.last.values.first - selection = rand(random_size) - #get the index that randomize choice mapped to - choice = distribution_map.select do |x| - x.values.first > selection #only keep the index with distribution value that is higher than the random generated number - end.first.keys.first #first hash's first key is the index we want - - L.debug("randomized distribution choice %s" % mdata_array[choice]) - mdata_array[choice] - end - - def get_random_one - data_array[rand(data_array.size)] - end - - def method_missing(meth, *args, &block) - # trap randomization based on percentage - if meth.to_s =~ /^get_(\d+)_percent/ - percentage = $1.to_i - remain = 100 - percentage - #data = args.first - normalized_data = if(!data_array.kind_of?(Array)) - [data_array, nil] - else - data_array - end - get_random_with_distribution(normalized_data, [percentage, remain]) - else - super - end - end - -end - -class JDKSelector - attr_reader :directory, :jdk_list - - def initialize(directory) - @directory = directory - end - - # get selection of available JDKs from Jenkins automatic install directory - def get_jdk - @jdk_list = Dir.entries(directory).select do |x| - x.chars.first == 'J' - end.map do |y| - File.join(directory, y) - end - self - end - - def filter_java_6(files) - files.select{ |i| File.basename(i).split(/[^0-9]/)[-1].to_i > 6 } - end - - # do randomized selection from a given array - def select_one(selection_array = nil) - selection_array = filter_java_6(selection_array || @jdk_list) - Randomizer.new(selection_array).get_random_one - end - - def JDKSelector.generate_jdk_hash(jdk_choice) - file_separator = if Gem.win_platform? - File::ALT_SEPARATOR - else - File::SEPARATOR - end - { - :PATH => [jdk_choice, 'bin'].join(file_separator) + File::PATH_SEPARATOR + ENV['PATH'], - :JAVA_HOME => jdk_choice - } - end -end - -# -# Fix argument JDK selector -# -class FixedJDKSelector < JDKSelector - def initialize(directory) - @directory = [*directory] #selection of directories to pick from - end - - def get_jdk - #since JDK selection is already specified..jdk list is the @directory - @jdk_list = @directory - self - end - - def select_one(selection_array = nil) - #bypass filtering since this is not automatic - selection_array ||= @jdk_list - Randomizer.new(selection_array).get_random_one - end -end - -# -# Property file writer -# -class PropertyWriter - attr_reader :working_directory - - def initialize(mworking_directory) - @working_directory = mworking_directory - end - - # # pick first element out of array of hashes, generate write java property file - def generate_property_file(data) - directory = working_directory - - #array transformation - content = data.to_a.map do |x| - x.join('=') - end.sort - file_name = (ENV['BUILD_ID'] + ENV['BUILD_NUMBER']) || 'prop' rescue 'prop' - file_name = file_name.split(File::SEPARATOR).first + '.txt' - L.debug "Property file name is %s" % file_name - File.open(File.join(directory, file_name), 'w') do |file| - file.write(content.join("\n")) - end - end -end - -# -# Execute randomization logics -# -class RandomizedRunner - attr_reader :random_choices, :jdk, :p_writer - - def initialize(mrandom_choices, mjdk, mwriter) - @random_choices = mrandom_choices - @jdk = mjdk - @p_writer = mwriter - end - - def generate_selections - configuration = random_choices - - L.debug "Enter %s" % __method__ - L.debug "Configuration %s" % YAML.dump(configuration) - - generated = {} - configuration.each do |k, v| - if(v.kind_of?(Hash)) - if(v.has_key?(:method)) - randomizer = Randomizer.new(v[:choices]) - v[:selections] = randomizer.__send__(v[:method]) - end - else - v.each do |x| - if(x.has_key?(:method)) - randomizer = Randomizer.new(x[:choices]) - x[:selections] = randomizer.__send__(x[:method]) - end - end - end - end.each do |k, v| - if(v.kind_of?(Array)) - selections = v.inject([]) do |sum, current_hash| - sum.push(current_hash[:selections]) - end - else - selections = [v[:selections]] unless v[:selections].nil? - end - generated[k] = selections unless (selections.nil? || selections.size == 0) - end - - L.debug "Generated selections %s" % YAML.dump(generated) - generated - end - - def get_env_matrix(jdk_selection, selections) - L.debug "Enter %s" % __method__ - - #normalization - s = {} - selections.each do |k, v| - if(v.size > 1) - s[k] = v.compact.join(' ') #this should be dependent on class of v[0] and perform reduce operation instead... good enough for now - else - s[k] = v.first - end - end - j = JDKSelector.generate_jdk_hash(jdk_selection) - - # create build description line - desc = {} - - # TODO: better error handling - desc[:BUILD_DESC] = "%s,%s,heap[%s],%s%s%s%s" % [ - File.basename(j[:JAVA_HOME]), - s['es.node.mode'], - s['tests.heap.size'], - s['tests.nightly'] ? 'nightly,':'', - s['tests.jvm.argline'].gsub(/-XX:/,''), - s.has_key?('tests.assertion.disabled')? ',assert off' : '', - s['tests.security.manager'] ? ',sec manager on' : '' - ] - result = j.merge(s).merge(desc) - L.debug(YAML.dump(result)) - result - end - - def run! - p_writer.generate_property_file(get_env_matrix(jdk, generate_selections)) - end - -end - - -# -# Main -# -unless(C[:test]) - - # Check to see if this is running locally - unless(C[:local]) - L.debug("Normal Mode") - working_directory = ENV.fetch('WORKSPACE', (Gem.win_platform? ? Dir.pwd : '/var/tmp')) - else - L.debug("Local Mode") - test_directory = 'tools/hudson.model.JDK/' - unless(File.exist?(test_directory)) - L.info "running local mode, setting up running environment" - L.info "properties are written to file prop.txt" - FileUtils.mkpath "%sJDK6" % test_directory - FileUtils.mkpath "%sJDK7" % test_directory - end - working_directory = Dir.pwd - end - - - # script support both window and linux - # TODO: refactor into platform/machine dependent class structure - jdk = if(Gem.win_platform?) - #window mode jdk directories are fixed - #TODO: better logic - L.debug("Window Mode") - if(File.directory?('y:\jdk7\7u55')) #old window system under ec2 - FixedJDKSelector.new('y:\jdk7\7u55') - else #new metal window system - FixedJDKSelector.new(['c:\PROGRA~1\JAVA\jdk1.8.0_05', 'c:\PROGRA~1\JAVA\jdk1.7.0_55']) - end - else - #Jenkins sets pwd prior to execution - L.debug("Linux Mode") - JDKSelector.new(File.join(ENV['PWD'],'tools','hudson.model.JDK')) - end - - runner = RandomizedRunner.new(RANDOM_CHOICES, - jdk.get_jdk.select_one, - PropertyWriter.new(working_directory)) - environment_matrix = runner.run! - exit 0 -else - require "test/unit" -end - -# -# Test -# -class TestJDKSelector < Test::Unit::TestCase - L = Logger.new 'test' - L.outputters = Outputter.stdout - L.level = DEBUG - - def test_hash_generator - jdk_choice = '/dummy/jdk7' - generated = JDKSelector.generate_jdk_hash(jdk_choice) - L.debug "Generated %s" % generated - assert generated[:PATH].include?(jdk_choice), "PATH doesn't included choice" - assert generated[:JAVA_HOME].include?(jdk_choice), "JAVA home doesn't include choice" - end -end - -class TestFixJDKSelector < Test::Unit::TestCase - L = Logger.new 'test' - L.outputters = Outputter.stdout - L.level = DEBUG - - def test_initialize - ['/home/dummy', ['/JDK7', '/home2'], ['home/dummy']].each do |x| - test_object = FixedJDKSelector.new(x) - assert_kind_of Array, test_object.directory - assert_equal [*x], test_object.directory - end - end - - def test_select_one - test_array = %w(one two three) - test_object = FixedJDKSelector.new(test_array) - assert test_array.include?(test_object.get_jdk.select_one) - end - - def test_hash_generator - jdk_choice = '/dummy/jdk7' - generated = FixedJDKSelector.generate_jdk_hash(jdk_choice) - L.debug "Generated %s" % generated - assert generated[:PATH].include?(jdk_choice), "PATH doesn't included choice" - assert generated[:JAVA_HOME].include?(jdk_choice), "JAVA home doesn't include choice" - end -end - -class TestPropertyWriter < Test::Unit::TestCase - L = Logger.new 'test' - L.outputters = Outputter.stdout - L.level = DEBUG - - def test_initialize - ['/home/dummy','/tmp'].each do |x| - test_object = PropertyWriter.new(x) - assert_kind_of String, test_object.working_directory - assert_equal x, test_object.working_directory - end - end - - def test_generate_property - test_file = '/tmp/prop.txt' - File.delete(test_file) if File.exist?(test_file) - test_object = PropertyWriter.new(File.dirname(test_file)) - # default prop.txt - test_object.generate_property_file({:hi => 'there'}) - assert(File.exist?(test_file)) - - File.open(test_file, 'r') do |properties_file| - properties_file.read.each_line do |line| - line.strip! - assert_equal 'hi=there', line, "content %s is not hi=there" % line - end - end - File.delete(test_file) if File.exist?(test_file) - end -end - -class DummyPropertyWriter < PropertyWriter - def generate_property_file(data) - L.debug "generating property file for %s" % YAML.dump(data) - L.debug "on directory %s" % working_directory - end -end - -class TestRandomizedRunner < Test::Unit::TestCase - - def test_initialize - test_object = RandomizedRunner.new(RANDOM_CHOICES, '/tmp/dummy/jdk', po = PropertyWriter.new('/tmp')) - assert_equal RANDOM_CHOICES, test_object.random_choices - assert_equal '/tmp/dummy/jdk', test_object.jdk - assert_equal po, test_object.p_writer - end - - def test_generate_selection_no_method - test_object = RandomizedRunner.new({'tests.one' => {:selections => false }}, '/tmp/dummy/jdk', po = DummyPropertyWriter.new('/tmp')) - selection = test_object.generate_selections - assert_equal false, selection['tests.one'].first, 'randomization without selection method fails' - end - - def test_generate_with_method - test_object = RandomizedRunner.new({'es.node.mode' => {:choices => ['local', 'network'], :method => 'get_random_one'}}, - '/tmp/dummy/jdk', po = DummyPropertyWriter.new('/tmp')) - selection = test_object.generate_selections - assert ['local', 'network'].include?(selection['es.node.mode'].first), 'selection choice is not correct' - end - - def test_get_env_matrix - test_object = RandomizedRunner.new(RANDOM_CHOICES, - '/tmp/dummy/jdk', po = DummyPropertyWriter.new('/tmp')) - selection = test_object.generate_selections - env_matrix = test_object.get_env_matrix('/tmp/dummy/jdk', selection) - puts YAML.dump(env_matrix) - assert_equal '/tmp/dummy/jdk', env_matrix[:JAVA_HOME] - end - -end diff --git a/dev-tools/client_tests_urls.prop b/dev-tools/client_tests_urls.prop deleted file mode 100644 index 81d2e52c53e..00000000000 --- a/dev-tools/client_tests_urls.prop +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -# -# This is used for client testings to pull in master, 090 bits -# -URL_MASTER=http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/master/nightly/JDK7/elasticsearch-latest-SNAPSHOT.zip -URL_1x=http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/1.x/nightly/JDK7/elasticsearch-latest-SNAPSHOT.zip -URL_11=http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/1.1/nightly/JDK6/elasticsearch-latest-SNAPSHOT.zip -URL_10=http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/1.0/nightly/JDK6/elasticsearch-latest-SNAPSHOT.zip -URL_090=http://s3-us-west-2.amazonaws.com/build.elasticsearch.org/origin/0.90/nightly/JDK6/elasticsearch-latest-SNAPSHOT.zip diff --git a/dev-tools/download-s3.py b/dev-tools/download-s3.py deleted file mode 100644 index b9ac22b653b..00000000000 --- a/dev-tools/download-s3.py +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import os -import sys -import argparse -try: - import boto.s3 -except: - raise RuntimeError(""" - S3 download requires boto to be installed - Use one of: - 'pip install -U boto' - 'apt-get install python-boto' - 'easy_install boto' - """) - -import boto.s3 - - -def list_buckets(conn): - return conn.get_all_buckets() - - -def download_s3(conn, path, key, file, bucket): - print 'Downloading %s from Amazon S3 bucket %s/%s' % \ - (file, bucket, os.path.join(path, key)) - def percent_cb(complete, total): - sys.stdout.write('.') - sys.stdout.flush() - bucket = conn.get_bucket(bucket) - k = bucket.get_key(os.path.join(path, key)) - k.get_contents_to_filename(file, cb=percent_cb, num_cb=100) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Downloads a bucket from Amazon S3') - parser.add_argument('--file', '-f', metavar='path to file', - help='path to store the bucket to', required=True) - parser.add_argument('--bucket', '-b', default='downloads.elasticsearch.org', - help='The S3 Bucket to download from') - parser.add_argument('--path', '-p', default='', - help='The key path to use') - parser.add_argument('--key', '-k', default=None, - help='The key - uses the file name as default key') - args = parser.parse_args() - if args.key: - key = args.key - else: - key = os.path.basename(args.file) - connection = boto.connect_s3() - download_s3(connection, args.path, key, args.file, args.bucket); diff --git a/dev-tools/upgrade-tests.py b/dev-tools/upgrade-tests.py deleted file mode 100644 index 69e3c6a3bbd..00000000000 --- a/dev-tools/upgrade-tests.py +++ /dev/null @@ -1,319 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import random -import os -import tempfile -import shutil -import subprocess -import time -import argparse -import logging -import sys -import re - -from datetime import datetime -try: - from elasticsearch import Elasticsearch - from elasticsearch.exceptions import ConnectionError - from elasticsearch.exceptions import TransportError -except ImportError as e: - print('Can\'t import elasticsearch please install `sudo pip install elasticsearch`') - raise e - - -'''This file executes a basic upgrade test by running a full cluster restart. - -The upgrade test starts 2 or more nodes of an old elasticsearch version, indexes -a random number of documents into the running nodes and executes a full cluster restart. -After the nodes are recovered a small set of basic checks are executed to ensure all -documents are still searchable and field data can be loaded etc. - -NOTE: This script requires the elasticsearch python client `elasticsearch-py` run the following command to install: - - `sudo pip install elasticsearch` - -if you are running python3 you need to install the client using pip3. On OSX `pip3` will be included in the Python 3.4 -release available on `https://www.python.org/download/`: - - `sudo pip3 install elasticsearch` - -See `https://github.com/elasticsearch/elasticsearch-py` for details - -In order to run this test two different version of elasticsearch are required. Both need to be unpacked into -the same directory: - -``` - $ cd /path/to/elasticsearch/clone - $ mkdir backwards && cd backwards - $ wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.3.1.tar.gz - $ wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.13.tar.gz - $ tar -zxvf elasticsearch-1.3.1.tar.gz && tar -zxvf elasticsearch-0.90.13.tar.gz - $ cd .. - $ python dev-tools/upgrade-tests.py --version.backwards 0.90.13 --version.current 1.3.1 -``` -''' - -BLACK_LIST = {'1.2.0' : { 'reason': 'Contains a major bug where routing hashes are not consistent with previous version', - 'issue': 'https://github.com/elasticsearch/elasticsearch/pull/6393'}, - '1.3.0' : { 'reason': 'Lucene Related bug prevents upgrades from 0.90.7 and some earlier versions ', - 'issue' : 'https://github.com/elasticsearch/elasticsearch/pull/7055'}} -# sometimes returns True -def rarely(): - return random.randint(0, 10) == 0 - -# usually returns True -def frequently(): - return not rarely() - -# asserts the correctness of the given hits given they are sorted asc -def assert_sort(hits): - values = [hit['sort'] for hit in hits['hits']['hits']] - assert len(values) > 0, 'expected non emtpy result' - val = min(values) - for x in values: - assert x >= val, '%s >= %s' % (x, val) - val = x - -# asserts that the cluster health didn't timeout etc. -def assert_health(cluster_health, num_shards, num_replicas): - assert cluster_health['timed_out'] == False, 'cluster health timed out %s' % cluster_health - - -# Starts a new elasticsearch node from a released & untared version. -# This node uses unicast discovery with the provided unicast host list and starts -# the nodes with the given data directory. This allows shutting down and starting up -# nodes on the same data dir simulating a full cluster restart. -def start_node(version, data_dir, node_dir, unicast_host_list, tcp_port, http_port): - es_run_path = os.path.join(node_dir, 'elasticsearch-%s' % (version), 'bin/elasticsearch') - if version.startswith('0.90.'): - foreground = '-f' # 0.90.x starts in background automatically - else: - foreground = '' - return subprocess.Popen([es_run_path, - '-Des.path.data=%s' % data_dir, '-Des.cluster.name=upgrade_test', - '-Des.discovery.zen.ping.unicast.hosts=%s' % unicast_host_list, - '-Des.transport.tcp.port=%s' % tcp_port, - '-Des.http.port=%s' % http_port, - foreground], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - -# Indexes the given number of document into the given index -# and randomly runs refresh, optimize and flush commands -def index_documents(es, index_name, type, num_docs): - logging.info('Indexing %s docs' % num_docs) - for id in range(0, num_docs): - es.index(index=index_name, doc_type=type, id=id, body={'string': str(random.randint(0, 100)), - 'long_sort': random.randint(0, 100), - 'double_sort' : float(random.randint(0, 100))}) - if rarely(): - es.indices.refresh(index=index_name) - if rarely(): - es.indices.flush(index=index_name, force=frequently()) - if rarely(): - es.indices.optimize(index=index_name) - es.indices.refresh(index=index_name) - -# Runs a basic number of assertions including: -# - document counts -# - match all search with sort on double / long -# - Realtime GET operations -# TODO(simonw): we should add stuff like: -# - dates including sorting -# - string sorting -# - docvalues if available -# - global ordinal if available -def run_basic_asserts(es, index_name, type, num_docs): - count = es.count(index=index_name)['count'] - assert count == num_docs, 'Expected %r but got %r documents' % (num_docs, count) - for _ in range(0, num_docs): - random_doc_id = random.randint(0, num_docs-1) - doc = es.get(index=index_name, doc_type=type, id=random_doc_id) - assert doc, 'Expected document for id %s but got %s' % (random_doc_id, doc) - - assert_sort(es.search(index=index_name, - body={ - 'sort': [ - {'double_sort': {'order': 'asc'}} - ] - })) - - assert_sort(es.search(index=index_name, - body={ - 'sort': [ - {'long_sort': {'order': 'asc'}} - ] - })) - - -# picks a random version or and entire random version tuple from the directory -# to run the backwards tests against. -def pick_random_upgrade_version(directory, lower_version=None, upper_version=None): - if lower_version and upper_version: - return lower_version, upper_version - assert os.path.isdir(directory), 'No such directory %s' % directory - versions = [] - for version in map(lambda x : x[len('elasticsearch-'):], filter(lambda x : re.match(r'^elasticsearch-\d+[.]\d+[.]\d+$', x), os.listdir(directory))): - if not version in BLACK_LIST: - versions.append(build_tuple(version)) - versions.sort() - - if lower_version: # lower version is set - picking a higher one - versions = filter(lambda x : x > build_tuple(lower_version), versions) - assert len(versions) >= 1, 'Expected at least 1 higher version than %s version in %s ' % (lower_version, directory) - random.shuffle(versions) - return lower_version, build_version(versions[0]) - if upper_version: - versions = filter(lambda x : x < build_tuple(upper_version), versions) - assert len(versions) >= 1, 'Expected at least 1 lower version than %s version in %s ' % (upper_version, directory) - random.shuffle(versions) - return build_version(versions[0]), upper_version - assert len(versions) >= 2, 'Expected at least 2 different version in %s but found %s' % (directory, len(versions)) - random.shuffle(versions) - versions = versions[0:2] - versions.sort() - return build_version(versions[0]), build_version(versions[1]) - -def build_version(version_tuple): - return '.'.join([str(x) for x in version_tuple]) - -def build_tuple(version_string): - return [int(x) for x in version_string.split('.')] - -# returns a new elasticsearch client and ensures the all nodes have joined the cluster -# this method waits at most 30 seconds for all nodes to join -def new_es_instance(num_nodes, http_port, timeout = 30): - logging.info('Waiting for %s nodes to join the cluster' % num_nodes) - for _ in range(0, timeout): - # TODO(simonw): ask Honza if there is a better way to do this? - try: - es = Elasticsearch([ - {'host': '127.0.0.1', 'port': http_port + x} - for x in range(0, num_nodes)]) - es.cluster.health(wait_for_nodes=num_nodes) - es.count() # can we actually search or do we get a 503? -- anyway retry - return es - except (ConnectionError, TransportError): - pass - time.sleep(1) - assert False, 'Timed out waiting for %s nodes for %s seconds' % (num_nodes, timeout) - -def assert_versions(bwc_version, current_version, node_dir): - assert [int(x) for x in bwc_version.split('.')] < [int(x) for x in current_version.split('.')],\ - '[%s] must be < than [%s]' % (bwc_version, current_version) - for version in [bwc_version, current_version]: - assert not version in BLACK_LIST, 'Version %s is blacklisted - %s, see %s' \ - % (version, BLACK_LIST[version]['reason'], - BLACK_LIST[version]['issue']) - dir = os.path.join(node_dir, 'elasticsearch-%s' % current_version) - assert os.path.isdir(dir), 'Expected elasticsearch-%s install directory does not exists: %s' % (version, dir) - -def full_cluster_restart(node_dir, current_version, bwc_version, tcp_port, http_port): - assert_versions(bwc_version, current_version, node_dir) - num_nodes = random.randint(2, 3) - nodes = [] - data_dir = tempfile.mkdtemp() - logging.info('Running upgrade test from [%s] to [%s] seed: [%s] es.path.data: [%s] es.http.port [%s] es.tcp.port [%s]' - % (bwc_version, current_version, seed, data_dir, http_port, tcp_port)) - try: - logging.info('Starting %s BWC nodes of version %s' % (num_nodes, bwc_version)) - unicast_addresses = ','.join(['127.0.0.1:%s' % (tcp_port+x) for x in range(0, num_nodes)]) - for id in range(0, num_nodes): - nodes.append(start_node(bwc_version, data_dir, node_dir, unicast_addresses, tcp_port+id, http_port+id)) - es = new_es_instance(num_nodes, http_port) - es.indices.delete(index='test_index', ignore=404) - num_shards = random.randint(1, 10) - num_replicas = random.randint(0, 1) - logging.info('Create index with [%s] shards and [%s] replicas' % (num_shards, num_replicas)) - es.indices.create(index='test_index', body={ - # TODO(simonw): can we do more here in terms of randomization - seems hard due to all the different version - 'settings': { - 'number_of_shards': num_shards, - 'number_of_replicas': num_replicas - } - }) - logging.info('Nodes joined, waiting for green status') - health = es.cluster.health(wait_for_status='green', wait_for_relocating_shards=0) - assert_health(health, num_shards, num_replicas) - num_docs = random.randint(10, 100) - index_documents(es, 'test_index', 'test_type', num_docs) - logging.info('Run basic asserts before full cluster restart') - run_basic_asserts(es, 'test_index', 'test_type', num_docs) - logging.info('kill bwc nodes -- prepare upgrade') - for node in nodes: - node.terminate() - - # now upgrade the nodes and rerun the checks - tcp_port = tcp_port + len(nodes) # bump up port to make sure we can claim them - http_port = http_port + len(nodes) - logging.info('Full Cluster restart starts upgrading to version [elasticsearch-%s] es.http.port [%s] es.tcp.port [%s]' - % (current_version, http_port, tcp_port)) - nodes = [] - unicast_addresses = ','.join(['127.0.0.1:%s' % (tcp_port+x) for x in range(0, num_nodes)]) - for id in range(0, num_nodes+1): # one more to trigger relocation - nodes.append(start_node(current_version, data_dir, node_dir, unicast_addresses, tcp_port+id, http_port+id)) - es = new_es_instance(num_nodes+1, http_port) - logging.info('Nodes joined, waiting for green status') - health = es.cluster.health(wait_for_status='green', wait_for_relocating_shards=0) - assert_health(health, num_shards, num_replicas) - run_basic_asserts(es, 'test_index', 'test_type', num_docs) - # by running the indexing again we try to catch possible mapping problems after the upgrade - index_documents(es, 'test_index', 'test_type', num_docs) - run_basic_asserts(es, 'test_index', 'test_type', num_docs) - logging.info("[SUCCESS] - all test passed upgrading from version [%s] to version [%s]" % (bwc_version, current_version)) - finally: - for node in nodes: - node.terminate() - time.sleep(1) # wait a second until removing the data dirs to give the nodes a chance to shutdown - shutil.rmtree(data_dir) # remove the temp data dir - -if __name__ == '__main__': - logging.basicConfig(format='[%(levelname)s] [%(asctime)s] %(message)s', level=logging.INFO, - datefmt='%Y-%m-%d %I:%M:%S %p') - logging.getLogger('elasticsearch').setLevel(logging.ERROR) - logging.getLogger('urllib3').setLevel(logging.WARN) - parser = argparse.ArgumentParser(description='Tests Full Cluster Restarts across major version') - parser.add_argument('--version.backwards', '-b', dest='backwards_version', metavar='V', - help='The elasticsearch version to upgrade from') - parser.add_argument('--version.current', '-c', dest='current_version', metavar='V', - help='The elasticsearch version to upgrade to') - parser.add_argument('--seed', '-s', dest='seed', metavar='N', type=int, - help='The random seed to use') - parser.add_argument('--backwards.dir', '-d', dest='bwc_directory', default='backwards', metavar='dir', - help='The directory to the backwards compatibility sources') - - parser.add_argument('--tcp.port', '-p', dest='tcp_port', default=9300, metavar='port', type=int, - help='The port to use as the minimum port for TCP communication') - parser.add_argument('--http.port', '-t', dest='http_port', default=9200, metavar='port', type=int, - help='The port to use as the minimum port for HTTP communication') - - parser.set_defaults(bwc_directory='backwards') - parser.set_defaults(seed=int(time.time())) - args = parser.parse_args() - node_dir = args.bwc_directory - current_version = args.current_version - bwc_version = args.backwards_version - seed = args.seed - random.seed(seed) - bwc_version, current_version = pick_random_upgrade_version(node_dir, bwc_version, current_version) - tcp_port = args.tcp_port - http_port = args.http_port - try: - full_cluster_restart(node_dir, current_version, bwc_version, tcp_port, http_port) - except: - logging.warn('REPRODUCE WITH: \n\t`python %s --version.backwards %s --version.current %s --seed %s --tcp.port %s --http.port %s`' - % (sys.argv[0], bwc_version, current_version, seed, tcp_port, http_port)) - raise diff --git a/dev-tools/upload-s3.py b/dev-tools/upload-s3.py deleted file mode 100644 index 95ea576e65c..00000000000 --- a/dev-tools/upload-s3.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on -# an 'AS IS' BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -# either express or implied. See the License for the specific -# language governing permissions and limitations under the License. - -import os -import sys -import argparse -try: - import boto.s3 -except: - raise RuntimeError(""" - S3 upload requires boto to be installed - Use one of: - 'pip install -U boto' - 'apt-get install python-boto' - 'easy_install boto' - """) - -import boto.s3 - - -def list_buckets(conn): - return conn.get_all_buckets() - - -def upload_s3(conn, path, key, file, bucket): - print 'Uploading %s to Amazon S3 bucket %s/%s' % \ - (file, bucket, os.path.join(path, key)) - def percent_cb(complete, total): - sys.stdout.write('.') - sys.stdout.flush() - bucket = conn.create_bucket(bucket) - k = bucket.new_key(os.path.join(path, key)) - k.set_contents_from_filename(file, cb=percent_cb, num_cb=100) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Uploads files to Amazon S3') - parser.add_argument('--file', '-f', metavar='path to file', - help='the branch to release from', required=True) - parser.add_argument('--bucket', '-b', metavar='B42', default='download.elasticsearch.org', - help='The S3 Bucket to upload to') - parser.add_argument('--path', '-p', metavar='elasticsearch/elasticsearch', default='elasticsearch/elasticsearch', - help='The key path to use') - parser.add_argument('--key', '-k', metavar='key', default=None, - help='The key - uses the file name as default key') - args = parser.parse_args() - if args.key: - key = args.key - else: - key = os.path.basename(args.file) - - connection = boto.connect_s3() - upload_s3(connection, args.path, key, args.file, args.bucket); - From dd2f26ca402c9e7eede68844f18e14f8ac93f3a1 Mon Sep 17 00:00:00 2001 From: Drew Raines Date: Thu, 8 Jan 2015 17:26:44 -0600 Subject: [PATCH 02/31] [cat/recovery] Make recovery time a TimeValue() Recovery `time` should be a TimeValue() to match other cat APIs. Closes #9209 --- .../org/elasticsearch/rest/action/cat/RestRecoveryAction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java index 47265d9efaf..759fac2eb19 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.Table; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestController; @@ -142,7 +143,7 @@ public class RestRecoveryAction extends AbstractCatAction { t.startRow(); t.addCell(index); t.addCell(state.getShardId().id()); - t.addCell(state.getTimer().time()); + t.addCell(new TimeValue(state.getTimer().time())); t.addCell(state.getType().toString().toLowerCase(Locale.ROOT)); t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT)); t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName()); From dc32aaa8c87ca939094f9208a1c36f29aafd10d5 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Sat, 20 Feb 2016 16:27:30 -0800 Subject: [PATCH 03/31] [cat/recovery] Make recovery time a TimeValue(): add doc Recovery `time` should be a TimeValue() to match other cat APIs. Closes #9209 --- docs/reference/cat/recovery.asciidoc | 40 +++++++++++++++------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index 64265677609..b9a16b2913d 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -15,10 +15,12 @@ are no shards in transit from one node to another: [source,sh] ---------------------------------------------------------------------------- > curl -XGET 'localhost:9200/_cat/recovery?v' -index shard time type stage source target files percent bytes percent -wiki 0 73 store done hostA hostA 36 100.0% 24982806 100.0% -wiki 1 245 store done hostA hostA 33 100.0% 24501912 100.0% -wiki 2 230 store done hostA hostA 36 100.0% 30267222 100.0% +index shard time type stage source_host target_host repository snapshot files files_percent bytes bytes_percent total_files total_bytes translog translog_percent total_translog +index 0 87ms store done 127.0.0.1 127.0.0.1 n/a n/a 0 0.0% 0 0.0% 0 0 0 100.0% 0 +index 1 97ms store done 127.0.0.1 127.0.0.1 n/a n/a 0 0.0% 0 0.0% 0 0 0 100.0% 0 +index 2 93ms store done 127.0.0.1 127.0.0.1 n/a n/a 0 0.0% 0 0.0% 0 0 0 100.0% 0 +index 3 90ms store done 127.0.0.1 127.0.0.1 n/a n/a 0 0.0% 0 0.0% 0 0 0 100.0% 0 +index 4 9ms store done 127.0.0.1 127.0.0.1 n/a n/a 0 0.0% 0 0.0% 0 0 0 100.0% 0 --------------------------------------------------------------------------- In the above case, the source and target nodes are the same because the recovery @@ -33,14 +35,14 @@ what a live shard recovery looks like. > curl -XPUT 'localhost:9200/wiki/_settings' -d'{"number_of_replicas":1}' {"acknowledged":true} -> curl -XGET 'localhost:9200/_cat/recovery?v' -index shard time type stage source target files percent bytes percent -wiki 0 1252 store done hostA hostA 4 100.0% 23638870 100.0% -wiki 0 1672 replica index hostA hostB 4 75.0% 23638870 48.8% -wiki 1 1698 replica index hostA hostB 4 75.0% 23348540 49.4% -wiki 1 4812 store done hostA hostA 33 100.0% 24501912 100.0% -wiki 2 1689 replica index hostA hostB 4 75.0% 28681851 40.2% -wiki 2 5317 store done hostA hostA 36 100.0% 30267222 100.0% +> curl -XGET 'localhost:9200/_cat/recovery?v&h=i,s,t,ty,st,shost,thost,f,fp,b,bp' +i s t ty st shost thost f fp b bp +wiki 0 1252ms store done hostA hostA 4 100.0% 23638870 100.0% +wiki 0 1672ms replica index hostA hostB 4 75.0% 23638870 48.8% +wiki 1 1698ms replica index hostA hostB 4 75.0% 23348540 49.4% +wiki 1 4812ms store done hostA hostA 33 100.0% 24501912 100.0% +wiki 2 1689ms replica index hostA hostB 4 75.0% 28681851 40.2% +wiki 2 5317ms store done hostA hostA 36 100.0% 30267222 100.0% ---------------------------------------------------------------------------- We can see in the above listing that our 3 initial shards are in various stages @@ -55,13 +57,13 @@ API. -------------------------------------------------------------------------------- > curl -XPOST 'localhost:9200/_snapshot/imdb/snapshot_2/_restore' {"acknowledged":true} -> curl -XGET 'localhost:9200/_cat/recovery?v' -index shard time type stage repository snapshot files percent bytes percent -imdb 0 1978 snapshot done imdb snap_1 79 8.0% 12086 9.0% -imdb 1 2790 snapshot index imdb snap_1 88 7.7% 11025 8.1% -imdb 2 2790 snapshot index imdb snap_1 85 0.0% 12072 0.0% -imdb 3 2796 snapshot index imdb snap_1 85 2.4% 12048 7.2% -imdb 4 819 snapshot init imdb snap_1 0 0.0% 0 0.0% +> curl -XGET 'localhost:9200/_cat/recovery?v&h=i,s,t,ty,st,rep,snap,f,fp,b,bp' +i s t ty st rep snap f fp b bp +imdb 0 1978ms snapshot done imdb snap_1 79 8.0% 12086 9.0% +imdb 1 2790ms snapshot index imdb snap_1 88 7.7% 11025 8.1% +imdb 2 2790ms snapshot index imdb snap_1 85 0.0% 12072 0.0% +imdb 3 2796ms snapshot index imdb snap_1 85 2.4% 12048 7.2% +imdb 4 819ms snapshot init imdb snap_1 0 0.0% 0 0.0% -------------------------------------------------------------------------------- From 99052c3fef16d6192af0839558ce3dbf23aa148d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Sat, 16 Jan 2016 11:39:59 -0700 Subject: [PATCH 04/31] Limit the accepted length of the _id Elasticsearch should reject ids that are this long, to ensure a document always remains retrievable for clients that impose a maximum URI length Closes #16034 --- .../action/index/IndexRequest.java | 5 +++++ .../action/index/IndexRequestTests.java | 22 +++++++++++++++++++ docs/reference/migration/migrate_3_0.asciidoc | 5 +++++ .../rest-api-spec/test/index/10_with_id.yaml | 8 +++++++ 4 files changed, 40 insertions(+) diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 976e4879a27..33bf17f0653 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -215,6 +215,11 @@ public class IndexRequest extends ReplicationRequest implements Do validationException = addValidationError("ttl must not be negative", validationException); } } + + if (id != null && id.getBytes(StandardCharsets.UTF_8).length > 512) { + validationException = addValidationError("id is too long, must be no longer than 512 bytes but was: " + + id.getBytes(StandardCharsets.UTF_8).length, validationException); + } return validationException; } diff --git a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java index ad246ebc530..8be67cb0fcf 100644 --- a/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/index/IndexRequestTests.java @@ -71,6 +71,28 @@ public class IndexRequestTests extends ESTestCase { assertThat(request.validate().validationErrors(), not(empty())); } + public void testIndexingRejectsLongIds() { + String id = randomAsciiOfLength(511); + IndexRequest request = new IndexRequest("index", "type", id); + request.source("{}"); + ActionRequestValidationException validate = request.validate(); + assertNull(validate); + + id = randomAsciiOfLength(512); + request = new IndexRequest("index", "type", id); + request.source("{}"); + validate = request.validate(); + assertNull(validate); + + id = randomAsciiOfLength(513); + request = new IndexRequest("index", "type", id); + request.source("{}"); + validate = request.validate(); + assertThat(validate, notNullValue()); + assertThat(validate.getMessage(), + containsString("id is too long, must be no longer than 512 bytes but was: 513")); +} + public void testSetTTLAsTimeValue() { IndexRequest indexRequest = new IndexRequest(); TimeValue ttl = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl"); diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index 12224270ebb..4d1848d3a96 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -97,6 +97,11 @@ characteristics as the former `scan` search type. [[breaking_30_rest_api_changes]] === REST API changes +==== id values longer than 512 bytes are rejected + +When specifying an `_id` value longer than 512 bytes, the request will be +rejected. + ==== search exists api removed The search exists api has been removed in favour of using the search api with diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yaml index 745e1117402..8ac55ec79f6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yaml @@ -24,3 +24,11 @@ - match: { _id: "1"} - match: { _version: 1} - match: { _source: { foo: bar }} + + - do: + catch: request + index: + index: idx + type: type + id: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + body: { foo: bar } From 3c15200f6f6c01872fee71b57634458d97285c9c Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 22 Feb 2016 11:37:43 -0800 Subject: [PATCH 05/31] Expose http address in cat/nodes and cat/nodeattrs APIs We expose a lot of information like IP address and port but never expose the http address/ip:port in the CAT API. It's nice to have it there too since otherwise json parsing is required to get this information --- .../rest/action/cat/RestNodeAttrsAction.java | 38 ++++++++++++------- .../rest/action/cat/RestNodesAction.java | 8 ++++ .../test/cat.nodes/10_basic.yaml | 9 +++++ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java index d67d6bc2d28..b02609323ce 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java @@ -43,6 +43,9 @@ import org.elasticsearch.rest.action.support.RestActionListener; import org.elasticsearch.rest.action.support.RestResponseListener; import org.elasticsearch.rest.action.support.RestTable; +import java.util.HashMap; +import java.util.Map; + import static org.elasticsearch.rest.RestRequest.Method.GET; public class RestNodeAttrsAction extends AbstractCatAction { @@ -112,23 +115,32 @@ public class RestNodeAttrsAction extends AbstractCatAction { for (DiscoveryNode node : nodes) { NodeInfo info = nodesInfo.getNodesMap().get(node.id()); for(ObjectObjectCursor att : node.attributes()) { - table.startRow(); - table.addCell(node.name()); - table.addCell(fullId ? node.id() : Strings.substring(node.getId(), 0, 4)); - table.addCell(info == null ? null : info.getProcess().getId()); - table.addCell(node.getHostName()); - table.addCell(node.getHostAddress()); - if (node.address() instanceof InetSocketTransportAddress) { - table.addCell(((InetSocketTransportAddress) node.address()).address().getPort()); - } else { - table.addCell("-"); + buildRow(fullId, table, node, info, att.key, att.value); + } + if (info.getServiceAttributes() != null) { + for (Map.Entry entry : info.getServiceAttributes().entrySet()) { + buildRow(fullId, table, node, info, entry.getKey(), entry.getValue()); } - table.addCell(att.key); - table.addCell(att.value); - table.endRow(); } } return table; } + + private final void buildRow(boolean fullId, Table table, DiscoveryNode node, NodeInfo info, String key, String value) { + table.startRow(); + table.addCell(node.name()); + table.addCell(fullId ? node.id() : Strings.substring(node.getId(), 0, 4)); + table.addCell(info == null ? null : info.getProcess().getId()); + table.addCell(node.getHostName()); + table.addCell(node.getHostAddress()); + if (node.address() instanceof InetSocketTransportAddress) { + table.addCell(((InetSocketTransportAddress) node.address()).address().getPort()); + } else { + table.addCell("-"); + } + table.addCell(key); + table.addCell(value); + table.endRow(); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java index 104d9295299..a6a44a9ed78 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java @@ -64,6 +64,7 @@ import org.elasticsearch.script.ScriptStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import java.util.Locale; +import java.util.Map; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -117,6 +118,7 @@ public class RestNodesAction extends AbstractCatAction { table.addCell("pid", "default:false;alias:p;desc:process id"); table.addCell("ip", "alias:i;desc:ip address"); table.addCell("port", "default:false;alias:po;desc:bound transport port"); + table.addCell("http_address", "default:false;alias:http;desc:bound http adress"); table.addCell("version", "default:false;alias:v;desc:es version"); table.addCell("build", "default:false;alias:b;desc:es build hash"); @@ -247,6 +249,12 @@ public class RestNodesAction extends AbstractCatAction { } else { table.addCell("-"); } + final Map serviceAttributes = info.getServiceAttributes(); + if (serviceAttributes != null) { + table.addCell(serviceAttributes.getOrDefault("http_address", "-")); + } else { + table.addCell("-"); + } table.addCell(node.getVersion().toString()); table.addCell(info == null ? null : info.getBuild().shortHash()); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml index 505d9e0cfe8..d60edf01102 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.nodes/10_basic.yaml @@ -48,3 +48,12 @@ $body: | /^ file_desc\.current \s+ file_desc\.percent \s+ file_desc\.max \n (\s+ (-1|\d+) \s+ \d+ \s+ (-1|\d+) \n)+ $/ + + - do: + cat.nodes: + h: http + v: true + + - match: + $body: | + /^ http \n ((\d{1,3}\.){3}\d{1,3}:\d{1,5}\n)+ $/ From a0a6eff0d0314a7ee2dca797b23c45b9b98ef558 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 22 Feb 2016 13:37:11 -0800 Subject: [PATCH 06/31] Fix test for [cat/recovery] Make recovery time a TimeValue() Related to #16743 --- .../resources/rest-api-spec/test/cat.recovery/10_basic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml index b081aa4d8cc..432b0e50ae4 100755 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml @@ -27,7 +27,7 @@ ( index1 \s+ \d \s+ # shard - \d+ \s+ # time + \d+ms \s+ # time (store|replica|snapshot|relocating) \s+ # type (init|index|verify_index|translog|finalize|done) \s+ # stage [-\w./]+ \s+ # source_host From 79da609439d2e33028eb9e6fe323c477a4eafe23 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 22 Feb 2016 14:45:49 -0800 Subject: [PATCH 07/31] Add issue template This commits adds an issue template for contributors that open bug reports or feature requests. Additionally, this commit adds a .github subdirectory to the project and moves the CONTRIBUTING.md file to that directory. Closes #16773 --- CONTRIBUTING.md => .github/CONTRIBUTING.md | 0 .github/ISSUE_TEMPLATE.md | 34 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) rename CONTRIBUTING.md => .github/CONTRIBUTING.md (100%) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..5ea58046f49 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,34 @@ + + + + +**Elasticsearch version**: + +**JVM version**: + +**OS version**: + +**Description of the problem including expected versus actual behavior**: + +**Steps to reproduce**: + 1. + 2. + 3. + +**Provide logs (if relevant)**: + + + +**Describe the feature**: From 1499d83e4e68f88fb7515491df9a205a3e888dc8 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 22 Feb 2016 16:10:02 -0800 Subject: [PATCH 08/31] Fix broken link to testing doc in contributing doc --- .github/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 070ea23d4e0..1b755a8869a 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -54,7 +54,7 @@ Once your changes and tests are ready to submit for review: 1. Test your changes Run the test suite to make sure that nothing is broken. See the -[TESTING](TESTING.asciidoc) file for help running tests. +[TESTING](../TESTING.asciidoc) file for help running tests. 2. Sign the Contributor License Agreement From 6e840b39f53f23bc801f84ca2bf4c6746bcc0886 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 22 Feb 2016 15:08:32 -0800 Subject: [PATCH 09/31] Remove ability to disable Netty gathering writes Java NIO has the notion of gathering writes. These are writes that gather data from multiple buffers into a single channel. These gathering writes in Netty have been enabled by default with the possibility to disable them using "es.netty.gathering". This flag was added in case having gathering writes on by default did not work out. We have not published this ability and sufficient time has passed to render judgement that using gathering writes is okay. Closes #16774 --- .../org/elasticsearch/common/netty/NettyUtils.java | 11 +---------- docs/reference/migration/migrate_3_0.asciidoc | 7 +++++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java b/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java index 164ed57e268..92b82fd2d8c 100644 --- a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java +++ b/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.common.netty; -import org.elasticsearch.common.Booleans; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.transport.netty.NettyInternalESLoggerFactory; import org.jboss.netty.logging.InternalLogger; import org.jboss.netty.logging.InternalLoggerFactory; @@ -74,7 +72,7 @@ public class NettyUtils { * sized pages, and if its a single one, makes sure that it gets sliced and wrapped in a composite * buffer. */ - public static final boolean DEFAULT_GATHERING; + public static final boolean DEFAULT_GATHERING = true; private static EsThreadNameDeterminer ES_THREAD_NAME_DETERMINER = new EsThreadNameDeterminer(); @@ -95,13 +93,6 @@ public class NettyUtils { }); ThreadRenamingRunnable.setThreadNameDeterminer(ES_THREAD_NAME_DETERMINER); - - /** - * This is here just to give us an option to rollback the change, if its stable, we should remove - * the option to even set it. - */ - DEFAULT_GATHERING = Booleans.parseBoolean(System.getProperty("es.netty.gathering"), true); - Loggers.getLogger(NettyUtils.class).debug("using gathering [{}]", DEFAULT_GATHERING); } public static void setup() { diff --git a/docs/reference/migration/migrate_3_0.asciidoc b/docs/reference/migration/migrate_3_0.asciidoc index 4d1848d3a96..e447bdc9ce7 100644 --- a/docs/reference/migration/migrate_3_0.asciidoc +++ b/docs/reference/migration/migrate_3_0.asciidoc @@ -316,6 +316,13 @@ Elasticsearch process has been removed. This same information can be obtained from the <> API, and a warning is logged on startup if it is set too low. +==== Removed es.netty.gathering + +Disabling Netty from using NIO gathering could be done via the escape +hatch of setting the system property "es.netty.gathering" to "false". +Time has proven enabling gathering by default is a non-issue and this +non-documented setting has been removed. + [[breaking_30_mapping_changes]] === Mapping changes From 5da9059d0b31ff057eb7c5aed4947639824552b9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 19 Feb 2016 21:20:45 -0800 Subject: [PATCH 10/31] Add G1GC check on startup This commit adds a check on startup for G1 GC while running on early versions of HotSpot version 25. This is to prevent potential data corruption issues that can occur on those versions. Closes #16737 --- .../org/elasticsearch/bootstrap/JVMCheck.java | 126 +++++++++++++++--- .../elasticsearch/monitor/jvm/JvmInfo.java | 9 ++ 2 files changed, 113 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java index 5c402bd83ce..3f81cd035bd 100644 --- a/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java +++ b/core/src/main/java/org/elasticsearch/bootstrap/JVMCheck.java @@ -21,17 +21,19 @@ package org.elasticsearch.bootstrap; import org.apache.lucene.util.Constants; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.monitor.jvm.JvmInfo; import java.lang.management.ManagementFactory; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Optional; /** Checks that the JVM is ok and won't cause index corruption */ final class JVMCheck { /** no instantiation */ private JVMCheck() {} - + /** * URL with latest JVM recommendations */ @@ -42,24 +44,60 @@ final class JVMCheck { */ static final String JVM_BYPASS = "es.bypass.vm.check"; + /** + * Metadata and messaging for checking and reporting HotSpot + * issues. + */ + interface HotSpotCheck { + /** + * If this HotSpot check should be executed. + * + * @return true if this HotSpot check should be executed + */ + boolean check(); + + /** + * The error message to display when this HotSpot issue is + * present. + * + * @return the error message for this HotSpot issue + */ + String getErrorMessage(); + + /** + * The warning message for this HotSpot issue if a workaround + * exists and is used. + * + * @return the warning message for this HotSpot issue + */ + Optional getWarningMessage(); + + /** + * The workaround for this HotSpot issue, if one exists. + * + * @return the workaround for this HotSpot issue, if one exists + */ + Optional getWorkaround(); + } + /** * Metadata and messaging for hotspot bugs. */ - static final class HotspotBug { - + static class HotspotBug implements HotSpotCheck { + /** OpenJDK bug URL */ final String bugUrl; - + /** Compiler workaround flag (null if there is no workaround) */ final String workAround; - + HotspotBug(String bugUrl, String workAround) { this.bugUrl = bugUrl; this.workAround = workAround; } - + /** Returns an error message to the user for a broken version */ - String getErrorMessage() { + public String getErrorMessage() { StringBuilder sb = new StringBuilder(); sb.append("Java version: ").append(fullVersion()); sb.append(" suffers from critical bug ").append(bugUrl); @@ -76,9 +114,9 @@ final class JVMCheck { } return sb.toString(); } - + /** Warns the user when a workaround is being used to dodge the bug */ - String getWarningMessage() { + public Optional getWarningMessage() { StringBuilder sb = new StringBuilder(); sb.append("Workaround flag ").append(workAround); sb.append(" for bug ").append(bugUrl); @@ -88,23 +126,67 @@ final class JVMCheck { sb.append(System.lineSeparator()); sb.append("Upgrading is preferred, see ").append(JVM_RECOMMENDATIONS); sb.append(" for current recommendations."); - return sb.toString(); + return Optional.of(sb.toString()); + } + + public boolean check() { + return true; + } + + @Override + public Optional getWorkaround() { + return Optional.of(workAround); } } - + + static class G1GCCheck implements HotSpotCheck { + @Override + public boolean check() { + return JvmInfo.jvmInfo().useG1GC().equals("true"); + } + + /** Returns an error message to the user for a broken version */ + public String getErrorMessage() { + StringBuilder sb = new StringBuilder(); + sb.append("Java version: ").append(fullVersion()); + sb.append(" can cause data corruption"); + sb.append(" when used with G1GC."); + sb.append(System.lineSeparator()); + sb.append("Please upgrade the JVM, see ").append(JVM_RECOMMENDATIONS); + sb.append(" for current recommendations."); + return sb.toString(); + } + + @Override + public Optional getWarningMessage() { + return Optional.empty(); + } + + @Override + public Optional getWorkaround() { + return Optional.empty(); + } + } + /** mapping of hotspot version to hotspot bug information for the most serious bugs */ - static final Map JVM_BROKEN_HOTSPOT_VERSIONS; - + static final Map JVM_BROKEN_HOTSPOT_VERSIONS; + static { - Map bugs = new HashMap<>(); - + Map bugs = new HashMap<>(); + // 1.7.0: loop optimizer bug bugs.put("21.0-b17", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-7070134", "-XX:-UseLoopPredicate")); // register allocation issues (technically only x86/amd64). This impacted update 40, 45, and 51 bugs.put("24.0-b56", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord")); bugs.put("24.45-b08", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord")); bugs.put("24.51-b03", new HotspotBug("https://bugs.openjdk.java.net/browse/JDK-8024830", "-XX:-UseSuperWord")); - + G1GCCheck g1GcCheck = new G1GCCheck(); + bugs.put("25.0-b70", g1GcCheck); + bugs.put("25.11-b03", g1GcCheck); + bugs.put("25.20-b23", g1GcCheck); + bugs.put("25.25-b02", g1GcCheck); + bugs.put("25.31-b07", g1GcCheck); + JVM_BROKEN_HOTSPOT_VERSIONS = Collections.unmodifiableMap(bugs); } @@ -115,10 +197,10 @@ final class JVMCheck { if (Boolean.parseBoolean(System.getProperty(JVM_BYPASS))) { Loggers.getLogger(JVMCheck.class).warn("bypassing jvm version check for version [{}], this can result in data corruption!", fullVersion()); } else if ("Oracle Corporation".equals(Constants.JVM_VENDOR)) { - HotspotBug bug = JVM_BROKEN_HOTSPOT_VERSIONS.get(Constants.JVM_VERSION); - if (bug != null) { - if (bug.workAround != null && ManagementFactory.getRuntimeMXBean().getInputArguments().contains(bug.workAround)) { - Loggers.getLogger(JVMCheck.class).warn(bug.getWarningMessage()); + HotSpotCheck bug = JVM_BROKEN_HOTSPOT_VERSIONS.get(Constants.JVM_VERSION); + if (bug != null && bug.check()) { + if (bug.getWorkaround().isPresent() && ManagementFactory.getRuntimeMXBean().getInputArguments().contains(bug.getWorkaround().get())) { + Loggers.getLogger(JVMCheck.class).warn(bug.getWarningMessage().get()); } else { throw new RuntimeException(bug.getErrorMessage()); } @@ -144,8 +226,8 @@ final class JVMCheck { } } } - - /** + + /** * Returns java + jvm version, looks like this: * {@code Oracle Corporation 1.8.0_45 [Java HotSpot(TM) 64-Bit Server VM 25.45-b02]} */ diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java index 82a99bd0bd1..afe1cc13eb2 100644 --- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java +++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java @@ -117,9 +117,12 @@ public class JvmInfo implements Streamable, ToXContent { Object useCompressedOopsVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops"); Method valueMethod = vmOptionClazz.getMethod("getValue"); info.useCompressedOops = (String)valueMethod.invoke(useCompressedOopsVmOption); + Object useG1GCVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseG1GC"); + info.useG1GC = (String)valueMethod.invoke(useG1GCVmOption); } catch (Throwable t) { // unable to deduce the state of compressed oops info.useCompressedOops = "unknown"; + info.useG1GC = "unknown"; } INSTANCE = info; @@ -158,6 +161,8 @@ public class JvmInfo implements Streamable, ToXContent { private String useCompressedOops; + private String useG1GC; + private JvmInfo() { } @@ -293,6 +298,10 @@ public class JvmInfo implements Streamable, ToXContent { return this.useCompressedOops; } + public String useG1GC() { + return this.useG1GC; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.JVM); From a9eb668497dc7d4da80f8de806f230eacf3de819 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 8 Feb 2016 18:45:41 -0500 Subject: [PATCH 11/31] Add node version check to shard allocation during restore Verifies that the version of a node is compatible with the version of a shard that's being restored on this node. Fixes #16519 --- .../decider/NodeVersionAllocationDecider.java | 21 +++++++++-- .../NodeVersionAllocationDeciderTests.java | 35 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java index 3fe34948acd..eb9c5cf8ee0 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; +import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; @@ -46,8 +47,13 @@ public class NodeVersionAllocationDecider extends AllocationDecider { public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { if (shardRouting.primary()) { if (shardRouting.currentNodeId() == null) { - // fresh primary, we can allocate wherever - return allocation.decision(Decision.YES, NAME, "primary shard can be allocated anywhere"); + if (shardRouting.restoreSource() != null) { + // restoring from a snapshot - check that the node can handle the version + return isVersionCompatible(shardRouting.restoreSource(), node, allocation); + } else { + // fresh primary, we can allocate wherever + return allocation.decision(Decision.YES, NAME, "primary shard can be allocated anywhere"); + } } else { // relocating primary, only migrate to newer host return isVersionCompatible(allocation.routingNodes(), shardRouting.currentNodeId(), node, allocation); @@ -77,4 +83,15 @@ public class NodeVersionAllocationDecider extends AllocationDecider { target.node().version(), source.node().version()); } } + + private Decision isVersionCompatible(RestoreSource restoreSource, final RoutingNode target, RoutingAllocation allocation) { + if (target.node().version().onOrAfter(restoreSource.version())) { + /* we can allocate if we can restore from a snapshot that is older or on the same version */ + return allocation.decision(Decision.YES, NAME, "target node version [%s] is same or newer than snapshot version [%s]", + target.node().version(), restoreSource.version()); + } else { + return allocation.decision(Decision.NO, NAME, "target node version [%s] is older than snapshot version [%s]", + target.node().version(), restoreSource.version()); + } + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java index 8768c311e00..4e5be0f26b7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java @@ -20,14 +20,17 @@ package org.elasticsearch.cluster.routing.allocation; import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RestoreSource; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -39,6 +42,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; @@ -337,6 +341,37 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase { assertThat(result.routingTable().index(shard1.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0)); } + + public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() { + final DiscoveryNode newNode = new DiscoveryNode("newNode", DummyTransportAddress.INSTANCE, Version.CURRENT); + final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", DummyTransportAddress.INSTANCE, VersionUtils.getPreviousVersion()); + final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", DummyTransportAddress.INSTANCE, VersionUtils.getPreviousVersion()); + + int numberOfShards = randomIntBetween(1, 3); + MetaData metaData = MetaData.builder() + .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas + (randomIntBetween(0, 3))) + .build(); + + ClusterState state = ClusterState.builder(ClusterName.DEFAULT) + .metaData(metaData) + .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), + Version.CURRENT, "test")).build()) + .nodes(DiscoveryNodes.builder().put(newNode).put(oldNode1).put(oldNode2)).build(); + AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{ + new ReplicaAfterPrimaryActiveAllocationDecider(Settings.EMPTY), + new NodeVersionAllocationDecider(Settings.EMPTY)}); + AllocationService strategy = new MockAllocationService(Settings.EMPTY, + allocationDeciders, + new ShardsAllocators(Settings.EMPTY, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE); + RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true); + + // Make sure that primary shards are only allocated on the new node + for (int i = 0; i < numberOfShards; i++) { + assertEquals("newNode", result.routingTable().index("test").getShards().get(i).primaryShard().currentNodeId()); + } + } + private ClusterState stabilize(ClusterState clusterState, AllocationService service) { logger.trace("RoutingNodes: {}", clusterState.getRoutingNodes().prettyPrint()); From a1740d66618958aa3fcda5f5177725931a8d8695 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 14:13:09 -0500 Subject: [PATCH 12/31] Inline Base64#encodeBytesToBytes Relates #16725 --- .../java/org/elasticsearch/common/Base64.java | 189 +++++++++--------- 1 file changed, 97 insertions(+), 92 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/Base64.java b/core/src/main/java/org/elasticsearch/common/Base64.java index 37d45261f6a..eb295193160 100644 --- a/core/src/main/java/org/elasticsearch/common/Base64.java +++ b/core/src/main/java/org/elasticsearch/common/Base64.java @@ -18,8 +18,10 @@ */ package org.elasticsearch.common; +import java.io.IOException; import java.nio.charset.Charset; import java.util.Locale; +import java.util.Objects; /** *

Encodes and decodes to and from Base64 notation.

@@ -161,7 +163,7 @@ import java.util.Locale; * @author rob@iharder.net * @version 2.3.7 */ -public class Base64 { +public final class Base64 { /* ******** P U B L I C F I E L D S ******** */ @@ -791,10 +793,7 @@ public class Base64 { * @since 2.3.1 */ public static byte[] encodeBytesToBytes(byte[] source, int off, int len, int options) throws java.io.IOException { - - if (source == null) { - throw new NullPointerException("Cannot serialize a null array."); - } // end if: null + Objects.requireNonNull(source, "Cannot serialize a null array."); if (off < 0) { throw new IllegalArgumentException("Cannot have negative offset: " + off); @@ -809,103 +808,109 @@ public class Base64 { String.format(Locale.ROOT, "Cannot have offset of %d and length of %d with array of length %d", off, len, source.length)); } // end if: off < 0 - // Compress? if ((options & GZIP) != 0) { - java.io.ByteArrayOutputStream baos = null; - java.util.zip.GZIPOutputStream gzos = null; - Base64.OutputStream b64os = null; - - try { - // GZip -> Base64 -> ByteArray - baos = new java.io.ByteArrayOutputStream(); - b64os = new Base64.OutputStream(baos, ENCODE | options); - gzos = new java.util.zip.GZIPOutputStream(b64os); - - gzos.write(source, off, len); - gzos.close(); - } // end try - catch (java.io.IOException e) { - // Catch it and then throw it immediately so that - // the finally{} block is called for cleanup. - throw e; - } // end catch - finally { - try { - gzos.close(); - } catch (Exception e) { - } - try { - b64os.close(); - } catch (Exception e) { - } - try { - baos.close(); - } catch (Exception e) { - } - } // end finally - - return baos.toByteArray(); + return encodeCompressedBytes(source, off, len, options); } // end if: compress // Else, don't compress. Better not to use streams at all then. else { - boolean breakLines = (options & DO_BREAK_LINES) != 0; - - //int len43 = len * 4 / 3; - //byte[] outBuff = new byte[ ( len43 ) // Main 4:3 - // + ( (len % 3) > 0 ? 4 : 0 ) // Account for padding - // + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines - // Try to determine more precisely how big the array needs to be. - // If we get it right, we don't have to do an array copy, and - // we save a bunch of memory. - int encLen = (len / 3) * 4 + (len % 3 > 0 ? 4 : 0); // Bytes needed for actual encoding - if (breakLines) { - encLen += encLen / MAX_LINE_LENGTH; // Plus extra newline characters - } - byte[] outBuff = new byte[encLen]; - - - int d = 0; - int e = 0; - int len2 = len - 2; - int lineLength = 0; - for (; d < len2; d += 3, e += 4) { - encode3to4(source, d + off, 3, outBuff, e, options); - - lineLength += 4; - if (breakLines && lineLength >= MAX_LINE_LENGTH) { - outBuff[e + 4] = NEW_LINE; - e++; - lineLength = 0; - } // end if: end of line - } // en dfor: each piece of array - - if (d < len) { - encode3to4(source, d + off, len - d, outBuff, e, options); - e += 4; - } // end if: some padding needed - - - // Only resize array if we didn't guess it right. - if (e <= outBuff.length - 1) { - // If breaking lines and the last byte falls right at - // the line length (76 bytes per line), there will be - // one extra byte, and the array will need to be resized. - // Not too bad of an estimate on array size, I'd say. - byte[] finalOut = new byte[e]; - System.arraycopy(outBuff, 0, finalOut, 0, e); - //System.err.println("Having to resize array from " + outBuff.length + " to " + e ); - return finalOut; - } else { - //System.err.println("No need to resize array."); - return outBuff; - } - + return encodeNonCompressedBytes(source, off, len, options); } // end else: don't compress } // end encodeBytesToBytes + private static byte[] encodeNonCompressedBytes(byte[] source, int off, int len, int options) { + boolean breakLines = (options & DO_BREAK_LINES) != 0; + + //int len43 = len * 4 / 3; + //byte[] outBuff = new byte[ ( len43 ) // Main 4:3 + // + ( (len % 3) > 0 ? 4 : 0 ) // Account for padding + // + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines + // Try to determine more precisely how big the array needs to be. + // If we get it right, we don't have to do an array copy, and + // we save a bunch of memory. + int encLen = (len / 3) * 4 + (len % 3 > 0 ? 4 : 0); // Bytes needed for actual encoding + if (breakLines) { + encLen += encLen / MAX_LINE_LENGTH; // Plus extra newline characters + } + byte[] outBuff = new byte[encLen]; + + + int d = 0; + int e = 0; + int len2 = len - 2; + int lineLength = 0; + for (; d < len2; d += 3, e += 4) { + encode3to4(source, d + off, 3, outBuff, e, options); + + lineLength += 4; + if (breakLines && lineLength >= MAX_LINE_LENGTH) { + outBuff[e + 4] = NEW_LINE; + e++; + lineLength = 0; + } // end if: end of line + } // en dfor: each piece of array + + if (d < len) { + encode3to4(source, d + off, len - d, outBuff, e, options); + e += 4; + } // end if: some padding needed + + + // Only resize array if we didn't guess it right. + if (e <= outBuff.length - 1) { + // If breaking lines and the last byte falls right at + // the line length (76 bytes per line), there will be + // one extra byte, and the array will need to be resized. + // Not too bad of an estimate on array size, I'd say. + byte[] finalOut = new byte[e]; + System.arraycopy(outBuff, 0, finalOut, 0, e); + //System.err.println("Having to resize array from " + outBuff.length + " to " + e ); + return finalOut; + } else { + //System.err.println("No need to resize array."); + return outBuff; + } + } + + private static byte[] encodeCompressedBytes(byte[] source, int off, int len, int options) throws IOException { + java.io.ByteArrayOutputStream baos = null; + java.util.zip.GZIPOutputStream gzos = null; + OutputStream b64os = null; + + try { + // GZip -> Base64 -> ByteArray + baos = new java.io.ByteArrayOutputStream(); + b64os = new OutputStream(baos, ENCODE | options); + gzos = new java.util.zip.GZIPOutputStream(b64os); + + gzos.write(source, off, len); + gzos.close(); + } // end try + catch (IOException e) { + // Catch it and then throw it immediately so that + // the finally{} block is called for cleanup. + throw e; + } // end catch + finally { + try { + gzos.close(); + } catch (Exception e) { + } + try { + b64os.close(); + } catch (Exception e) { + } + try { + baos.close(); + } catch (Exception e) { + } + } // end finally + + return baos.toByteArray(); + } + /* ******** D E C O D I N G M E T H O D S ******** */ From cf1d9cfefc8af484953ffd31edf720e71e657d71 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 14:39:52 -0500 Subject: [PATCH 13/31] Inline Base64#decode4to3 Relates #16725 --- .../java/org/elasticsearch/common/Base64.java | 77 ++++++------------- 1 file changed, 23 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/Base64.java b/core/src/main/java/org/elasticsearch/common/Base64.java index eb295193160..41e0ce05f7f 100644 --- a/core/src/main/java/org/elasticsearch/common/Base64.java +++ b/core/src/main/java/org/elasticsearch/common/Base64.java @@ -942,17 +942,10 @@ public final class Base64 { * or there is not enough room in the array. * @since 1.3 */ - private static int decode4to3( - byte[] source, int srcOffset, - byte[] destination, int destOffset, int options) { - + private static int decode4to3(byte[] source, int srcOffset, byte[] destination, int destOffset, int options) { // Lots of error checking and exception throwing - if (source == null) { - throw new NullPointerException("Source array was null."); - } // end if - if (destination == null) { - throw new NullPointerException("Destination array was null."); - } // end if + Objects.requireNonNull(source, "Source array was null."); + Objects.requireNonNull(destination, "Destination array was null."); if (srcOffset < 0 || srcOffset + 3 >= source.length) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Source array with length %d cannot have offset of %d and still process four bytes.", source.length, srcOffset)); @@ -962,56 +955,36 @@ public final class Base64 { "Destination array with length %d cannot have offset of %d and still store three bytes.", destination.length, destOffset)); } // end if - byte[] DECODABET = getDecodabet(options); + + // Two ways to do the same thing. Don't know which way I like best. + //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) + // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); + int outBuff = ((DECODABET[source[srcOffset]] & 0xFF) << 18) + | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12); + + destination[destOffset] = (byte) (outBuff >>> 16); + // Example: Dk== if (source[srcOffset + 2] == EQUALS_SIGN) { - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); - int outBuff = ((DECODABET[source[srcOffset]] & 0xFF) << 18) - | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12); - - destination[destOffset] = (byte) (outBuff >>> 16); return 1; } - // Example: DkL= - else if (source[srcOffset + 3] == EQUALS_SIGN) { - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) - // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ); - int outBuff = ((DECODABET[source[srcOffset]] & 0xFF) << 18) - | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12) - | ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6); + outBuff |= ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6); + destination[destOffset + 1] = (byte) (outBuff >>> 8); - destination[destOffset] = (byte) (outBuff >>> 16); - destination[destOffset + 1] = (byte) (outBuff >>> 8); + // Example: DkL= + if (source[srcOffset + 3] == EQUALS_SIGN) { return 2; } + outBuff |= ((DECODABET[source[srcOffset + 3]] & 0xFF)); + destination[destOffset + 2] = (byte) (outBuff); + // Example: DkLE - else { - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) - // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ) - // | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 ); - int outBuff = ((DECODABET[source[srcOffset]] & 0xFF) << 18) - | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12) - | ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6) - | ((DECODABET[source[srcOffset + 3]] & 0xFF)); - - - destination[destOffset] = (byte) (outBuff >> 16); - destination[destOffset + 1] = (byte) (outBuff >> 8); - destination[destOffset + 2] = (byte) (outBuff); - - return 3; - } - } // end decodeToBytes + return 3; + } /** @@ -1056,13 +1029,9 @@ public final class Base64 { * @throws java.io.IOException If bogus characters exist in source data * @since 1.3 */ - public static byte[] decode(byte[] source, int off, int len, int options) - throws java.io.IOException { - + public static byte[] decode(byte[] source, int off, int len, int options) throws java.io.IOException { // Lots of error checking and exception throwing - if (source == null) { - throw new NullPointerException("Cannot decode null source array."); - } // end if + Objects.requireNonNull(source, "Cannot decode null source array."); if (off < 0 || off + len > source.length) { throw new IllegalArgumentException(String.format(Locale.ROOT, "Source array with length %d cannot have offset of %d and process %d bytes.", source.length, off, len)); From 2690f69d772a7f752c10057c3603a237379a8133 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 18:45:10 -0500 Subject: [PATCH 14/31] Inline Base64#decode Relates #16725 --- .../java/org/elasticsearch/common/Base64.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/Base64.java b/core/src/main/java/org/elasticsearch/common/Base64.java index 41e0ce05f7f..fa499a55d4d 100644 --- a/core/src/main/java/org/elasticsearch/common/Base64.java +++ b/core/src/main/java/org/elasticsearch/common/Base64.java @@ -1048,16 +1048,21 @@ public final class Base64 { int len34 = len * 3 / 4; // Estimate on array size byte[] outBuff = new byte[len34]; // Upper limit on size of output - int outBuffPosn = 0; // Keep track of where we're writing + int outBuffPosn = decode(source, off, len, options, DECODABET, outBuff); + + byte[] out = new byte[outBuffPosn]; + System.arraycopy(outBuff, 0, out, 0, outBuffPosn); + return out; + } // end decode + + private static int decode(byte[] source, int off, int len, int options, byte[] DECODABET, byte[] outBuff) throws IOException { + int outBuffPosn = 0; // Keep track of where we're writing byte[] b4 = new byte[4]; // Four byte buffer from source, eliminating white space int b4Posn = 0; // Keep track of four byte input buffer - int i = 0; // Source array counter - byte sbiDecode = 0; // Special value from DECODABET + for (int i = off; i < off + len; i++) { // Loop through source - for (i = off; i < off + len; i++) { // Loop through source - - sbiDecode = DECODABET[source[i] & 0xFF]; + byte sbiDecode = DECODABET[source[i] & 0xFF]; // White space, Equals sign, or legit Base64 character // Note the values such as -5 and -9 in the @@ -1073,7 +1078,7 @@ public final class Base64 { if (source[i] == EQUALS_SIGN) { // check if the equals sign is somewhere in between if (i+1 < len + off) { - throw new java.io.IOException(String.format(Locale.ROOT, + throw new IOException(String.format(Locale.ROOT, "Found equals sign at position %d of the base64 string, not at the end", i)); } break; @@ -1081,7 +1086,7 @@ public final class Base64 { } // end if: quartet built else { if (source[i] == EQUALS_SIGN && len + off > i && source[i+1] != EQUALS_SIGN) { - throw new java.io.IOException(String.format(Locale.ROOT, + throw new IOException(String.format(Locale.ROOT, "Found equals sign at position %d of the base64 string, not at the end", i)); } // enf if: equals sign and next character not as well } // end else: @@ -1089,15 +1094,12 @@ public final class Base64 { } // end if: white space, equals sign or better else { // There's a bad input character in the Base64 stream. - throw new java.io.IOException(String.format(Locale.ROOT, + throw new IOException(String.format(Locale.ROOT, "Bad Base64 input character decimal %d in array position %d", ((int) source[i]) & 0xFF, i)); } // end else: } // each input character - - byte[] out = new byte[outBuffPosn]; - System.arraycopy(outBuff, 0, out, 0, outBuffPosn); - return out; - } // end decode + return outBuffPosn; + } /** From a447c06efccbf64c17c4e92db1f982c77af07c06 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 19:07:35 -0500 Subject: [PATCH 15/31] Inline NettyHttpChannel#getStatus Relates #16725 --- .../http/netty/NettyHttpChannel.java | 152 +++++++----------- 1 file changed, 56 insertions(+), 96 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java index 9a88b375f54..b11ca19640b 100644 --- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java +++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java @@ -43,6 +43,8 @@ import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.HttpVersion; +import java.util.Collections; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -196,101 +198,59 @@ public class NettyHttpChannel extends HttpChannel { private static final HttpResponseStatus TOO_MANY_REQUESTS = new HttpResponseStatus(429, "Too Many Requests"); - private HttpResponseStatus getStatus(RestStatus status) { - switch (status) { - case CONTINUE: - return HttpResponseStatus.CONTINUE; - case SWITCHING_PROTOCOLS: - return HttpResponseStatus.SWITCHING_PROTOCOLS; - case OK: - return HttpResponseStatus.OK; - case CREATED: - return HttpResponseStatus.CREATED; - case ACCEPTED: - return HttpResponseStatus.ACCEPTED; - case NON_AUTHORITATIVE_INFORMATION: - return HttpResponseStatus.NON_AUTHORITATIVE_INFORMATION; - case NO_CONTENT: - return HttpResponseStatus.NO_CONTENT; - case RESET_CONTENT: - return HttpResponseStatus.RESET_CONTENT; - case PARTIAL_CONTENT: - return HttpResponseStatus.PARTIAL_CONTENT; - case MULTI_STATUS: - // no status for this?? - return HttpResponseStatus.INTERNAL_SERVER_ERROR; - case MULTIPLE_CHOICES: - return HttpResponseStatus.MULTIPLE_CHOICES; - case MOVED_PERMANENTLY: - return HttpResponseStatus.MOVED_PERMANENTLY; - case FOUND: - return HttpResponseStatus.FOUND; - case SEE_OTHER: - return HttpResponseStatus.SEE_OTHER; - case NOT_MODIFIED: - return HttpResponseStatus.NOT_MODIFIED; - case USE_PROXY: - return HttpResponseStatus.USE_PROXY; - case TEMPORARY_REDIRECT: - return HttpResponseStatus.TEMPORARY_REDIRECT; - case BAD_REQUEST: - return HttpResponseStatus.BAD_REQUEST; - case UNAUTHORIZED: - return HttpResponseStatus.UNAUTHORIZED; - case PAYMENT_REQUIRED: - return HttpResponseStatus.PAYMENT_REQUIRED; - case FORBIDDEN: - return HttpResponseStatus.FORBIDDEN; - case NOT_FOUND: - return HttpResponseStatus.NOT_FOUND; - case METHOD_NOT_ALLOWED: - return HttpResponseStatus.METHOD_NOT_ALLOWED; - case NOT_ACCEPTABLE: - return HttpResponseStatus.NOT_ACCEPTABLE; - case PROXY_AUTHENTICATION: - return HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED; - case REQUEST_TIMEOUT: - return HttpResponseStatus.REQUEST_TIMEOUT; - case CONFLICT: - return HttpResponseStatus.CONFLICT; - case GONE: - return HttpResponseStatus.GONE; - case LENGTH_REQUIRED: - return HttpResponseStatus.LENGTH_REQUIRED; - case PRECONDITION_FAILED: - return HttpResponseStatus.PRECONDITION_FAILED; - case REQUEST_ENTITY_TOO_LARGE: - return HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE; - case REQUEST_URI_TOO_LONG: - return HttpResponseStatus.REQUEST_URI_TOO_LONG; - case UNSUPPORTED_MEDIA_TYPE: - return HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE; - case REQUESTED_RANGE_NOT_SATISFIED: - return HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE; - case EXPECTATION_FAILED: - return HttpResponseStatus.EXPECTATION_FAILED; - case UNPROCESSABLE_ENTITY: - return HttpResponseStatus.BAD_REQUEST; - case LOCKED: - return HttpResponseStatus.BAD_REQUEST; - case FAILED_DEPENDENCY: - return HttpResponseStatus.BAD_REQUEST; - case TOO_MANY_REQUESTS: - return TOO_MANY_REQUESTS; - case INTERNAL_SERVER_ERROR: - return HttpResponseStatus.INTERNAL_SERVER_ERROR; - case NOT_IMPLEMENTED: - return HttpResponseStatus.NOT_IMPLEMENTED; - case BAD_GATEWAY: - return HttpResponseStatus.BAD_GATEWAY; - case SERVICE_UNAVAILABLE: - return HttpResponseStatus.SERVICE_UNAVAILABLE; - case GATEWAY_TIMEOUT: - return HttpResponseStatus.GATEWAY_TIMEOUT; - case HTTP_VERSION_NOT_SUPPORTED: - return HttpResponseStatus.HTTP_VERSION_NOT_SUPPORTED; - default: - return HttpResponseStatus.INTERNAL_SERVER_ERROR; - } + static Map MAP; + + static { + EnumMap map = new EnumMap<>(RestStatus.class); + map.put(RestStatus.CONTINUE, HttpResponseStatus.CONTINUE); + map.put(RestStatus.SWITCHING_PROTOCOLS, HttpResponseStatus.SWITCHING_PROTOCOLS); + map.put(RestStatus.OK, HttpResponseStatus.OK); + map.put(RestStatus.CREATED, HttpResponseStatus.CREATED); + map.put(RestStatus.ACCEPTED, HttpResponseStatus.ACCEPTED); + map.put(RestStatus.NON_AUTHORITATIVE_INFORMATION, HttpResponseStatus.NON_AUTHORITATIVE_INFORMATION); + map.put(RestStatus.NO_CONTENT, HttpResponseStatus.NO_CONTENT); + map.put(RestStatus.RESET_CONTENT, HttpResponseStatus.RESET_CONTENT); + map.put(RestStatus.PARTIAL_CONTENT, HttpResponseStatus.PARTIAL_CONTENT); + map.put(RestStatus.MULTI_STATUS, HttpResponseStatus.INTERNAL_SERVER_ERROR); // no status for this?? + map.put(RestStatus.MULTIPLE_CHOICES, HttpResponseStatus.MULTIPLE_CHOICES); + map.put(RestStatus.MOVED_PERMANENTLY, HttpResponseStatus.MOVED_PERMANENTLY); + map.put(RestStatus.FOUND, HttpResponseStatus.FOUND); + map.put(RestStatus.SEE_OTHER, HttpResponseStatus.SEE_OTHER); + map.put(RestStatus.NOT_MODIFIED, HttpResponseStatus.NOT_MODIFIED); + map.put(RestStatus.USE_PROXY, HttpResponseStatus.USE_PROXY); + map.put(RestStatus.TEMPORARY_REDIRECT, HttpResponseStatus.TEMPORARY_REDIRECT); + map.put(RestStatus.BAD_REQUEST, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.UNAUTHORIZED, HttpResponseStatus.UNAUTHORIZED); + map.put(RestStatus.PAYMENT_REQUIRED, HttpResponseStatus.PAYMENT_REQUIRED); + map.put(RestStatus.FORBIDDEN, HttpResponseStatus.FORBIDDEN); + map.put(RestStatus.NOT_FOUND, HttpResponseStatus.NOT_FOUND); + map.put(RestStatus.METHOD_NOT_ALLOWED, HttpResponseStatus.METHOD_NOT_ALLOWED); + map.put(RestStatus.NOT_ACCEPTABLE, HttpResponseStatus.NOT_ACCEPTABLE); + map.put(RestStatus.PROXY_AUTHENTICATION, HttpResponseStatus.PROXY_AUTHENTICATION_REQUIRED); + map.put(RestStatus.REQUEST_TIMEOUT, HttpResponseStatus.REQUEST_TIMEOUT); + map.put(RestStatus.CONFLICT, HttpResponseStatus.CONFLICT); + map.put(RestStatus.GONE, HttpResponseStatus.GONE); + map.put(RestStatus.LENGTH_REQUIRED, HttpResponseStatus.LENGTH_REQUIRED); + map.put(RestStatus.PRECONDITION_FAILED, HttpResponseStatus.PRECONDITION_FAILED); + map.put(RestStatus.REQUEST_ENTITY_TOO_LARGE, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE); + map.put(RestStatus.REQUEST_URI_TOO_LONG, HttpResponseStatus.REQUEST_URI_TOO_LONG); + map.put(RestStatus.UNSUPPORTED_MEDIA_TYPE, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE); + map.put(RestStatus.REQUESTED_RANGE_NOT_SATISFIED, HttpResponseStatus.REQUESTED_RANGE_NOT_SATISFIABLE); + map.put(RestStatus.EXPECTATION_FAILED, HttpResponseStatus.EXPECTATION_FAILED); + map.put(RestStatus.UNPROCESSABLE_ENTITY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.LOCKED, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.FAILED_DEPENDENCY, HttpResponseStatus.BAD_REQUEST); + map.put(RestStatus.TOO_MANY_REQUESTS, TOO_MANY_REQUESTS); + map.put(RestStatus.INTERNAL_SERVER_ERROR, HttpResponseStatus.INTERNAL_SERVER_ERROR); + map.put(RestStatus.NOT_IMPLEMENTED, HttpResponseStatus.NOT_IMPLEMENTED); + map.put(RestStatus.BAD_GATEWAY, HttpResponseStatus.BAD_GATEWAY); + map.put(RestStatus.SERVICE_UNAVAILABLE, HttpResponseStatus.SERVICE_UNAVAILABLE); + map.put(RestStatus.GATEWAY_TIMEOUT, HttpResponseStatus.GATEWAY_TIMEOUT); + map.put(RestStatus.HTTP_VERSION_NOT_SUPPORTED, HttpResponseStatus.HTTP_VERSION_NOT_SUPPORTED); + MAP = Collections.unmodifiableMap(map); + } + + private static HttpResponseStatus getStatus(RestStatus status) { + return MAP.getOrDefault(status, HttpResponseStatus.INTERNAL_SERVER_ERROR); } } From 2e9887a3cb7a9a832dcf5e9a2fd212db5b05662c Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 19:18:00 -0500 Subject: [PATCH 16/31] Inline NettyHttpChannel#sendResponse Relates #16725 --- .../http/netty/NettyHttpChannel.java | 80 +++++++++++-------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java index b11ca19640b..0ef5fff11bb 100644 --- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java +++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java @@ -53,10 +53,7 @@ import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.CONNECTION; import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.CLOSE; import static org.jboss.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE; -/** - * - */ -public class NettyHttpChannel extends HttpChannel { +public final class NettyHttpChannel extends HttpChannel { private final NettyHttpServerTransport transport; private final Channel channel; @@ -94,18 +91,11 @@ public class NettyHttpChannel extends HttpChannel { String opaque = nettyRequest.headers().get("X-Opaque-Id"); if (opaque != null) { - resp.headers().add("X-Opaque-Id", opaque); + setHeaderField(resp, "X-Opaque-Id", opaque); } // Add all custom headers - Map> customHeaders = response.getHeaders(); - if (customHeaders != null) { - for (Map.Entry> headerEntry : customHeaders.entrySet()) { - for (String headerValue : headerEntry.getValue()) { - resp.headers().add(headerEntry.getKey(), headerValue); - } - } - } + addCustomHeaders(response, resp); BytesReference content = response.content(); ChannelBuffer buffer; @@ -115,30 +105,11 @@ public class NettyHttpChannel extends HttpChannel { resp.setContent(buffer); // If our response doesn't specify a content-type header, set one - if (!resp.headers().contains(HttpHeaders.Names.CONTENT_TYPE)) { - resp.headers().add(HttpHeaders.Names.CONTENT_TYPE, response.contentType()); - } - + setHeaderField(resp, HttpHeaders.Names.CONTENT_TYPE, response.contentType(), false); // If our response has no content-length, calculate and set one - if (!resp.headers().contains(HttpHeaders.Names.CONTENT_LENGTH)) { - resp.headers().add(HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(buffer.readableBytes())); - } + setHeaderField(resp, HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(buffer.readableBytes()), false); - if (transport.resetCookies) { - String cookieString = nettyRequest.headers().get(HttpHeaders.Names.COOKIE); - if (cookieString != null) { - CookieDecoder cookieDecoder = new CookieDecoder(); - Set cookies = cookieDecoder.decode(cookieString); - if (!cookies.isEmpty()) { - // Reset the cookies if necessary. - CookieEncoder cookieEncoder = new CookieEncoder(true); - for (Cookie cookie : cookies) { - cookieEncoder.addCookie(cookie); - } - resp.headers().add(HttpHeaders.Names.SET_COOKIE, cookieEncoder.encode()); - } - } - } + addCookies(resp); ChannelFuture future; @@ -166,6 +137,45 @@ public class NettyHttpChannel extends HttpChannel { } } + private void setHeaderField(HttpResponse resp, String headerField, String value) { + setHeaderField(resp, headerField, value, true); + } + + private void setHeaderField(HttpResponse resp, String headerField, String value, boolean override) { + if (override || !resp.headers().contains(headerField)) { + resp.headers().add(headerField, value); + } + } + + private void addCookies(HttpResponse resp) { + if (transport.resetCookies) { + String cookieString = nettyRequest.headers().get(HttpHeaders.Names.COOKIE); + if (cookieString != null) { + CookieDecoder cookieDecoder = new CookieDecoder(); + Set cookies = cookieDecoder.decode(cookieString); + if (!cookies.isEmpty()) { + // Reset the cookies if necessary. + CookieEncoder cookieEncoder = new CookieEncoder(true); + for (Cookie cookie : cookies) { + cookieEncoder.addCookie(cookie); + } + setHeaderField(resp, HttpHeaders.Names.SET_COOKIE, cookieEncoder.encode()); + } + } + } + } + + private void addCustomHeaders(RestResponse response, HttpResponse resp) { + Map> customHeaders = response.getHeaders(); + if (customHeaders != null) { + for (Map.Entry> headerEntry : customHeaders.entrySet()) { + for (String headerValue : headerEntry.getValue()) { + setHeaderField(resp, headerEntry.getKey(), headerValue); + } + } + } + } + // Determine if the request protocol version is HTTP 1.0 private boolean isHttp10() { return nettyRequest.getProtocolVersion().equals(HttpVersion.HTTP_1_0); From 001c2c62825e89a7f1e17124207f7be25131e2f9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 20:34:45 -0500 Subject: [PATCH 17/31] Inline XContentBuilder#writeValue Relates #16725 --- .../common/xcontent/XContentBuilder.java | 192 ++++++++++-------- 1 file changed, 111 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index a82a747624b..7ac098f5a64 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -43,9 +43,12 @@ import java.math.RoundingMode; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.function.BiConsumer; /** * @@ -1227,52 +1230,101 @@ public final class XContentBuilder implements BytesStream, Releasable { generator.writeEndObject(); } + @FunctionalInterface + interface Writer { + void write(XContentGenerator g, Object v) throws IOException; + } + + private final static Map, Writer> MAP; + + static { + Map, Writer> map = new HashMap<>(); + map.put(String.class, (g, v) -> g.writeString((String) v)); + map.put(Integer.class, (g, v) -> g.writeNumber((Integer) v)); + map.put(Long.class, (g, v) -> g.writeNumber((Long) v)); + map.put(Float.class, (g, v) -> g.writeNumber((Float) v)); + map.put(Double.class, (g, v) -> g.writeNumber((Double) v)); + map.put(Byte.class, (g, v) -> g.writeNumber((Byte) v)); + map.put(Short.class, (g, v) -> g.writeNumber((Short) v)); + map.put(Boolean.class, (g, v) -> g.writeBoolean((Boolean) v)); + map.put(GeoPoint.class, (g, v) -> { + g.writeStartObject(); + g.writeNumberField("lat", ((GeoPoint) v).lat()); + g.writeNumberField("lon", ((GeoPoint) v).lon()); + g.writeEndObject(); + }); + map.put(int[].class, (g, v) -> { + g.writeStartArray(); + for (int item : (int[]) v) { + g.writeNumber(item); + } + g.writeEndArray(); + }); + map.put(long[].class, (g, v) -> { + g.writeStartArray(); + for (long item : (long[]) v) { + g.writeNumber(item); + } + g.writeEndArray(); + }); + map.put(float[].class, (g, v) -> { + g.writeStartArray(); + for (float item : (float[]) v) { + g.writeNumber(item); + } + g.writeEndArray(); + }); + map.put(double[].class, (g, v) -> { + g.writeStartArray(); + for (double item : (double[])v) { + g.writeNumber(item); + } + g.writeEndArray(); + }); + map.put(byte[].class, (g, v) -> g.writeBinary((byte[]) v)); + map.put(short[].class, (g, v) -> { + g.writeStartArray(); + for (short item : (short[])v) { + g.writeNumber(item); + } + g.writeEndArray(); + }); + map.put(BytesRef.class, (g, v) -> { + BytesRef bytes = (BytesRef) v; + g.writeBinary(bytes.bytes, bytes.offset, bytes.length); + }); + map.put(Text.class, (g, v) -> { + Text text = (Text) v; + if (text.hasBytes() && text.bytes().hasArray()) { + g.writeUTF8String(text.bytes().array(), text.bytes().arrayOffset(), text.bytes().length()); + } else if (text.hasString()) { + g.writeString(text.string()); + } else { + BytesArray bytesArray = text.bytes().toBytesArray(); + g.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length()); + } + }); + MAP = Collections.unmodifiableMap(map); + } + private void writeValue(Object value) throws IOException { if (value == null) { generator.writeNull(); return; } Class type = value.getClass(); - if (type == String.class) { - generator.writeString((String) value); - } else if (type == Integer.class) { - generator.writeNumber(((Integer) value).intValue()); - } else if (type == Long.class) { - generator.writeNumber(((Long) value).longValue()); - } else if (type == Float.class) { - generator.writeNumber(((Float) value).floatValue()); - } else if (type == Double.class) { - generator.writeNumber(((Double) value).doubleValue()); - } else if (type == Byte.class) { - generator.writeNumber(((Byte)value).byteValue()); - } else if (type == Short.class) { - generator.writeNumber(((Short) value).shortValue()); - } else if (type == Boolean.class) { - generator.writeBoolean(((Boolean) value).booleanValue()); - } else if (type == GeoPoint.class) { - generator.writeStartObject(); - generator.writeNumberField("lat", ((GeoPoint) value).lat()); - generator.writeNumberField("lon", ((GeoPoint) value).lon()); - generator.writeEndObject(); + Writer writer = MAP.get(type); + if (writer != null) { + writer.write(generator, value); } else if (value instanceof Map) { writeMap((Map) value); } else if (value instanceof Path) { //Path implements Iterable and causes endless recursion and a StackOverFlow if treated as an Iterable here generator.writeString(value.toString()); } else if (value instanceof Iterable) { - generator.writeStartArray(); - for (Object v : (Iterable) value) { - writeValue(v); - } - generator.writeEndArray(); + writeIterable((Iterable) value); } else if (value instanceof Object[]) { - generator.writeStartArray(); - for (Object v : (Object[]) value) { - writeValue(v); - } - generator.writeEndArray(); - } else if (type == byte[].class) { - generator.writeBinary((byte[]) value); + writeObjectArray((Object[]) value); } else if (value instanceof Date) { generator.writeString(XContentBuilder.defaultDatePrinter.print(((Date) value).getTime())); } else if (value instanceof Calendar) { @@ -1280,56 +1332,9 @@ public final class XContentBuilder implements BytesStream, Releasable { } else if (value instanceof ReadableInstant) { generator.writeString(XContentBuilder.defaultDatePrinter.print((((ReadableInstant) value)).getMillis())); } else if (value instanceof BytesReference) { - BytesReference bytes = (BytesReference) value; - if (!bytes.hasArray()) { - bytes = bytes.toBytesArray(); - } - generator.writeBinary(bytes.array(), bytes.arrayOffset(), bytes.length()); - } else if (value instanceof BytesRef) { - BytesRef bytes = (BytesRef) value; - generator.writeBinary(bytes.bytes, bytes.offset, bytes.length); - } else if (value instanceof Text) { - Text text = (Text) value; - if (text.hasBytes() && text.bytes().hasArray()) { - generator.writeUTF8String(text.bytes().array(), text.bytes().arrayOffset(), text.bytes().length()); - } else if (text.hasString()) { - generator.writeString(text.string()); - } else { - BytesArray bytesArray = text.bytes().toBytesArray(); - generator.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length()); - } + writeBytesReference((BytesReference) value); } else if (value instanceof ToXContent) { ((ToXContent) value).toXContent(this, ToXContent.EMPTY_PARAMS); - } else if (value instanceof double[]) { - generator.writeStartArray(); - for (double v : (double[]) value) { - generator.writeNumber(v); - } - generator.writeEndArray(); - } else if (value instanceof long[]) { - generator.writeStartArray(); - for (long v : (long[]) value) { - generator.writeNumber(v); - } - generator.writeEndArray(); - } else if (value instanceof int[]) { - generator.writeStartArray(); - for (int v : (int[]) value) { - generator.writeNumber(v); - } - generator.writeEndArray(); - } else if (value instanceof float[]) { - generator.writeStartArray(); - for (float v : (float[]) value) { - generator.writeNumber(v); - } - generator.writeEndArray(); - } else if (value instanceof short[]) { - generator.writeStartArray(); - for (short v : (short[]) value) { - generator.writeNumber(v); - } - generator.writeEndArray(); } else { // if this is a "value" object, like enum, DistanceUnit, ..., just toString it // yea, it can be misleading when toString a Java class, but really, jackson should be used in that case @@ -1337,4 +1342,29 @@ public final class XContentBuilder implements BytesStream, Releasable { //throw new ElasticsearchIllegalArgumentException("type not supported for generic value conversion: " + type); } } + + private void writeBytesReference(BytesReference value) throws IOException { + BytesReference bytes = value; + if (!bytes.hasArray()) { + bytes = bytes.toBytesArray(); + } + generator.writeBinary(bytes.array(), bytes.arrayOffset(), bytes.length()); + } + + private void writeIterable(Iterable value) throws IOException { + generator.writeStartArray(); + for (Object v : value) { + writeValue(v); + } + generator.writeEndArray(); + } + + private void writeObjectArray(Object[] value) throws IOException { + generator.writeStartArray(); + for (Object v : value) { + writeValue(v); + } + generator.writeEndArray(); + } + } From 373c3dfa5761591a483b8ab211e934ca328d8721 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 21:44:21 -0500 Subject: [PATCH 18/31] Inline PrimaryPhase#doRun Relates #16725 --- .../TransportReplicationAction.java | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 1d22207723b..16147cc5b3c 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -677,27 +677,36 @@ public abstract class TransportReplicationAction primaryResponse = shardOperationOnPrimary(state.metaData(), request); - if (logger.isTraceEnabled()) { - logger.trace("action [{}] completed on shard [{}] for request [{}] with cluster state version [{}]", transportPrimaryAction, shardId, request, state.version()); - } - ReplicationPhase replicationPhase = new ReplicationPhase(task, primaryResponse.v2(), primaryResponse.v1(), shardId, channel, indexShardReference); - finishAndMoveToReplication(replicationPhase); + executeLocally(); + } else { - // delegate primary phase to relocation target - // it is safe to execute primary phase on relocation target as there are no more in-flight operations where primary - // phase is executed on local shard and all subsequent operations are executed on relocation target as primary phase. - final ShardRouting primary = indexShardReference.routingEntry(); - indexShardReference.close(); - assert primary.relocating() : "indexShard is marked as relocated but routing isn't" + primary; - DiscoveryNode relocatingNode = state.nodes().get(primary.relocatingNodeId()); - transportService.sendRequest(relocatingNode, transportPrimaryAction, request, transportOptions, - TransportChannelResponseHandler.responseHandler(logger, TransportReplicationAction.this::newResponseInstance, channel, - "rerouting indexing to target primary " + primary)); + executeRemotely(); } } + private void executeLocally() throws Exception { + // execute locally + Tuple primaryResponse = shardOperationOnPrimary(state.metaData(), request); + if (logger.isTraceEnabled()) { + logger.trace("action [{}] completed on shard [{}] for request [{}] with cluster state version [{}]", transportPrimaryAction, shardId, request, state.version()); + } + ReplicationPhase replicationPhase = new ReplicationPhase(task, primaryResponse.v2(), primaryResponse.v1(), shardId, channel, indexShardReference); + finishAndMoveToReplication(replicationPhase); + } + + private void executeRemotely() { + // delegate primary phase to relocation target + // it is safe to execute primary phase on relocation target as there are no more in-flight operations where primary + // phase is executed on local shard and all subsequent operations are executed on relocation target as primary phase. + final ShardRouting primary = indexShardReference.routingEntry(); + indexShardReference.close(); + assert primary.relocating() : "indexShard is marked as relocated but routing isn't" + primary; + DiscoveryNode relocatingNode = state.nodes().get(primary.relocatingNodeId()); + transportService.sendRequest(relocatingNode, transportPrimaryAction, request, transportOptions, + TransportChannelResponseHandler.responseHandler(logger, TransportReplicationAction.this::newResponseInstance, channel, + "rerouting indexing to target primary " + primary)); + } + /** * checks whether we can perform a write based on the write consistency setting * returns **null* if OK to proceed, or a string describing the reason to stop From fe2a29211b34af143d90ce5d6926b60d7d1f3a89 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 12 Feb 2016 23:22:35 -0500 Subject: [PATCH 19/31] Inline ReroutePhase#doRun Relates #16725 --- .../TransportReplicationAction.java | 106 +++++++++++------- 1 file changed, 68 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 16147cc5b3c..894b19fedb7 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -461,60 +461,90 @@ public abstract class TransportReplicationAction Date: Sat, 13 Feb 2016 00:12:35 -0500 Subject: [PATCH 20/31] Inline MetaData#resolveIndexRouting Relates #16725 --- .../cluster/metadata/MetaData.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 8ba78979f3f..d7dddb15984 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -399,27 +399,16 @@ public class MetaData implements Iterable, Diffable, Fr // in the index,bulk,update and delete apis. public String resolveIndexRouting(@Nullable String parent, @Nullable String routing, String aliasOrIndex) { if (aliasOrIndex == null) { - if (routing == null) { - return parent; - } - return routing; + return routingOrParent(parent, routing); } AliasOrIndex result = getAliasAndIndexLookup().get(aliasOrIndex); if (result == null || result.isAlias() == false) { - if (routing == null) { - return parent; - } - return routing; + return routingOrParent(parent, routing); } AliasOrIndex.Alias alias = (AliasOrIndex.Alias) result; if (result.getIndices().size() > 1) { - String[] indexNames = new String[result.getIndices().size()]; - int i = 0; - for (IndexMetaData indexMetaData : result.getIndices()) { - indexNames[i++] = indexMetaData.getIndex().getName(); - } - throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op"); + rejectSingleIndexOperation(aliasOrIndex, result); } AliasMetaData aliasMd = alias.getFirstAliasMetaData(); if (aliasMd.indexRouting() != null) { @@ -434,6 +423,19 @@ public class MetaData implements Iterable, Diffable, Fr // Alias routing overrides the parent routing (if any). return aliasMd.indexRouting(); } + return routingOrParent(parent, routing); + } + + private void rejectSingleIndexOperation(String aliasOrIndex, AliasOrIndex result) { + String[] indexNames = new String[result.getIndices().size()]; + int i = 0; + for (IndexMetaData indexMetaData : result.getIndices()) { + indexNames[i++] = indexMetaData.getIndex().getName(); + } + throw new IllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one index associated with it [" + Arrays.toString(indexNames) + "], can't execute a single index op"); + } + + private String routingOrParent(@Nullable String parent, @Nullable String routing) { if (routing == null) { return parent; } From e39280b5a6f4cb5d3933a818bbe8f2858fae969d Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 00:14:03 -0500 Subject: [PATCH 21/31] Inline CMCB#addEstimateBytesAndMaybeBreak Relates #16725 --- .../breaker/ChildMemoryCircuitBreaker.java | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java index 9284b7b50a2..0a0be5eb163 100644 --- a/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java +++ b/core/src/main/java/org/elasticsearch/common/breaker/ChildMemoryCircuitBreaker.java @@ -118,37 +118,9 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker { // .addAndGet() instead of looping (because we don't have to check a // limit), which makes the RamAccountingTermsEnum case faster. if (this.memoryBytesLimit == -1) { - newUsed = this.used.addAndGet(bytes); - if (logger.isTraceEnabled()) { - logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: [-1b]]", - this.name, new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed)); - } + newUsed = noLimit(bytes, label); } else { - // Otherwise, check the addition and commit the addition, looping if - // there are conflicts. May result in additional logging, but it's - // trace logging and shouldn't be counted on for additions. - long currentUsed; - do { - currentUsed = this.used.get(); - newUsed = currentUsed + bytes; - long newUsedWithOverhead = (long) (newUsed * overheadConstant); - if (logger.isTraceEnabled()) { - logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]", - this.name, - new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed), - memoryBytesLimit, new ByteSizeValue(memoryBytesLimit), - newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead)); - } - if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) { - logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking", - this.name, - newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label, - memoryBytesLimit, new ByteSizeValue(memoryBytesLimit)); - circuitBreak(label, newUsedWithOverhead); - } - // Attempt to set the new used value, but make sure it hasn't changed - // underneath us, if it has, keep trying until we are able to set it - } while (!this.used.compareAndSet(currentUsed, newUsed)); + newUsed = limit(bytes, label); } // Additionally, we need to check that we haven't exceeded the parent's limit @@ -164,6 +136,45 @@ public class ChildMemoryCircuitBreaker implements CircuitBreaker { return newUsed; } + private long noLimit(long bytes, String label) { + long newUsed; + newUsed = this.used.addAndGet(bytes); + if (logger.isTraceEnabled()) { + logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: [-1b]]", + this.name, new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed)); + } + return newUsed; + } + + private long limit(long bytes, String label) { + long newUsed;// Otherwise, check the addition and commit the addition, looping if + // there are conflicts. May result in additional logging, but it's + // trace logging and shouldn't be counted on for additions. + long currentUsed; + do { + currentUsed = this.used.get(); + newUsed = currentUsed + bytes; + long newUsedWithOverhead = (long) (newUsed * overheadConstant); + if (logger.isTraceEnabled()) { + logger.trace("[{}] Adding [{}][{}] to used bytes [new used: [{}], limit: {} [{}], estimate: {} [{}]]", + this.name, + new ByteSizeValue(bytes), label, new ByteSizeValue(newUsed), + memoryBytesLimit, new ByteSizeValue(memoryBytesLimit), + newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead)); + } + if (memoryBytesLimit > 0 && newUsedWithOverhead > memoryBytesLimit) { + logger.warn("[{}] New used memory {} [{}] for data of [{}] would be larger than configured breaker: {} [{}], breaking", + this.name, + newUsedWithOverhead, new ByteSizeValue(newUsedWithOverhead), label, + memoryBytesLimit, new ByteSizeValue(memoryBytesLimit)); + circuitBreak(label, newUsedWithOverhead); + } + // Attempt to set the new used value, but make sure it hasn't changed + // underneath us, if it has, keep trying until we are able to set it + } while (!this.used.compareAndSet(currentUsed, newUsed)); + return newUsed; + } + /** * Add an exact number of bytes, not checking for tripping the * circuit breaker. This bypasses the overheadConstant multiplication. From b9a7db554a703c841ecab8df2cdc1ab2d4c0613c Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 07:56:31 -0500 Subject: [PATCH 22/31] Inline RestUtils#decodeComponent Relates #16725 --- .../elasticsearch/rest/support/RestUtils.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java index 57d4c8e01d9..7a976fe1f1d 100644 --- a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java +++ b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java @@ -142,13 +142,22 @@ public class RestUtils { * @throws IllegalArgumentException if the string contains a malformed * escape sequence. */ - @SuppressWarnings("fallthrough") public static String decodeComponent(final String s, final Charset charset) { if (s == null) { return ""; } final int size = s.length(); - boolean modified = false; + if (!decodingNeeded(s, size)) { + return s; + } + final byte[] buf = new byte[size]; + int pos = decode(s, size, buf); + return new String(buf, 0, pos, charset); + } + + @SuppressWarnings("fallthrough") + private static boolean decodingNeeded(String s, int size) { + boolean decodingNeeded = false; for (int i = 0; i < size; i++) { final char c = s.charAt(i); switch (c) { @@ -156,14 +165,15 @@ public class RestUtils { i++; // We can skip at least one char, e.g. `%%'. // Fall through. case '+': - modified = true; + decodingNeeded = true; break; } } - if (!modified) { - return s; - } - final byte[] buf = new byte[size]; + return decodingNeeded; + } + + @SuppressWarnings("fallthrough") + private static int decode(String s, int size, byte[] buf) { int pos = 0; // position in `buf'. for (int i = 0; i < size; i++) { char c = s.charAt(i); @@ -173,24 +183,22 @@ public class RestUtils { break; case '%': if (i == size - 1) { - throw new IllegalArgumentException("unterminated escape" - + " sequence at end of string: " + s); + throw new IllegalArgumentException("unterminated escape sequence at end of string: " + s); } c = s.charAt(++i); if (c == '%') { buf[pos++] = '%'; // "%%" -> "%" break; } else if (i == size - 1) { - throw new IllegalArgumentException("partial escape" - + " sequence at end of string: " + s); + throw new IllegalArgumentException("partial escape sequence at end of string: " + s); } c = decodeHexNibble(c); final char c2 = decodeHexNibble(s.charAt(++i)); if (c == Character.MAX_VALUE || c2 == Character.MAX_VALUE) { throw new IllegalArgumentException( - "invalid escape sequence `%" + s.charAt(i - 1) - + s.charAt(i) + "' at index " + (i - 2) - + " of: " + s); + "invalid escape sequence `%" + s.charAt(i - 1) + + s.charAt(i) + "' at index " + (i - 2) + + " of: " + s); } c = (char) (c * 16 + c2); // Fall through. @@ -199,7 +207,7 @@ public class RestUtils { break; } } - return new String(buf, 0, pos, charset); + return pos; } /** From 6a66882e460fc9260541ab32995ea52b05fc21fb Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 08:55:59 -0500 Subject: [PATCH 23/31] Inline DocumentParser#parseDocument Relates #16725 --- .../index/mapper/DocumentParser.java | 200 +++++++++++------- 1 file changed, 124 insertions(+), 76 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index b0cdb993b78..98721b4358a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -75,93 +75,118 @@ class DocumentParser implements Closeable { this.docMapper = docMapper; } - public ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException { + final ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException { + validateType(source); + + source.type(docMapper.type()); + final Mapping mapping = docMapper.mapping(); + final ParseContext.InternalParseContext context = cache.get(); + XContentParser parser = null; + try { + parser = parser(source); + context.reset(parser, new ParseContext.Document(), source); + validateStart(parser); + internalParseDocument(mapping, context, parser); + validateEnd(source, parser); + } catch (Throwable t) { + throw wrapInMapperParsingException(source, t); + } finally { + // only close the parser when its not provided externally + if (internalParser(source, parser)) { + parser.close(); + } + } + + reverseOrder(context); + applyDocBoost(context); + + ParsedDocument doc = parsedDocument(source, context, update(context, mapping)); + // reset the context to free up memory + context.reset(null, null, null); + return doc; + } + + private static void internalParseDocument(Mapping mapping, ParseContext.InternalParseContext context, XContentParser parser) throws IOException { + final boolean emptyDoc = isEmptyDoc(mapping, parser); + + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + metadataMapper.preParse(context); + } + + if (mapping.root.isEnabled() == false) { + // entire type is disabled + parser.skipChildren(); + } else if (emptyDoc == false) { + Mapper update = parseObject(context, mapping.root, true); + if (update != null) { + context.addDynamicMappingsUpdate(update); + } + } + + for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { + metadataMapper.postParse(context); + } + } + + private void validateType(SourceToParse source) { if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) { throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); } - ParseContext.InternalParseContext context = cache.get(); - - final Mapping mapping = docMapper.mapping(); if (source.type() != null && !source.type().equals(docMapper.type())) { throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); } - source.type(docMapper.type()); + } - XContentParser parser = source.parser(); - try { - if (parser == null) { - parser = XContentHelper.createParser(source.source()); - } - context.reset(parser, new ParseContext.Document(), source); + private static XContentParser parser(SourceToParse source) throws IOException { + return source.parser() == null ? XContentHelper.createParser(source.source()) : source.parser(); + } - // will result in START_OBJECT - XContentParser.Token token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new MapperParsingException("Malformed content, must start with an object"); - } + private static boolean internalParser(SourceToParse source, XContentParser parser) { + return source.parser() == null && parser != null; + } - boolean emptyDoc = false; - if (mapping.root.isEnabled()) { - token = parser.nextToken(); - if (token == XContentParser.Token.END_OBJECT) { - // empty doc, we can handle it... - emptyDoc = true; - } else if (token != XContentParser.Token.FIELD_NAME) { - throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist"); - } - } - - for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { - metadataMapper.preParse(context); - } - - if (mapping.root.isEnabled() == false) { - // entire type is disabled - parser.skipChildren(); - } else if (emptyDoc == false) { - Mapper update = parseObject(context, mapping.root, true); - if (update != null) { - context.addDynamicMappingsUpdate(update); - } - } - - for (MetadataFieldMapper metadataMapper : mapping.metadataMappers) { - metadataMapper.postParse(context); - } + private static void validateStart(XContentParser parser) throws IOException { + // will result in START_OBJECT + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new MapperParsingException("Malformed content, must start with an object"); + } + } + private static void validateEnd(SourceToParse source, XContentParser parser) throws IOException { + XContentParser.Token token;// only check for end of tokens if we created the parser here + if (internalParser(source, parser)) { // try to parse the next token, this should be null if the object is ended properly // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled by the catch) - if (source.parser() == null && parser != null) { - // only check for end of tokens if we created the parser here - token = parser.nextToken(); - if (token != null) { - throw new IllegalArgumentException("Malformed content, found extra data after parsing: " + token); - } - } - - } catch (Throwable e) { - // if its already a mapper parsing exception, no need to wrap it... - if (e instanceof MapperParsingException) { - throw (MapperParsingException) e; - } - - // Throw a more meaningful message if the document is empty. - if (source.source() != null && source.source().length() == 0) { - throw new MapperParsingException("failed to parse, document is empty"); - } - - throw new MapperParsingException("failed to parse", e); - } finally { - // only close the parser when its not provided externally - if (source.parser() == null && parser != null) { - parser.close(); + token = parser.nextToken(); + if (token != null) { + throw new IllegalArgumentException("Malformed content, found extra data after parsing: " + token); } } + } + + private static boolean isEmptyDoc(Mapping mapping, XContentParser parser) throws IOException { + if (mapping.root.isEnabled()) { + final XContentParser.Token token = parser.nextToken(); + if (token == XContentParser.Token.END_OBJECT) { + // empty doc, we can handle it... + return true; + } else if (token != XContentParser.Token.FIELD_NAME) { + throw new MapperParsingException("Malformed content, after first object, either the type field or the actual properties should exist"); + } + } + return false; + } + + private static void reverseOrder(ParseContext.InternalParseContext context) { // reverse the order of docs for nested docs support, parent should be last if (context.docs().size() > 1) { Collections.reverse(context.docs()); } + } + + private static void applyDocBoost(ParseContext.InternalParseContext context) { // apply doc boost if (context.docBoost() != 1.0f) { Set encounteredFields = new HashSet<>(); @@ -177,18 +202,41 @@ class DocumentParser implements Closeable { } } } + } + private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) { + return new ParsedDocument( + context.uid(), + context.version(), + context.id(), + context.type(), + source.routing(), + source.timestamp(), + source.ttl(), + context.docs(), + context.source(), + update + ).parent(source.parent()); + } + + + private static Mapping update(ParseContext.InternalParseContext context, Mapping mapping) { Mapper rootDynamicUpdate = context.dynamicMappingsUpdate(); - Mapping update = null; - if (rootDynamicUpdate != null) { - update = mapping.mappingUpdate(rootDynamicUpdate); + return rootDynamicUpdate != null ? mapping.mappingUpdate(rootDynamicUpdate) : null; + } + + private static MapperParsingException wrapInMapperParsingException(SourceToParse source, Throwable e) { + // if its already a mapper parsing exception, no need to wrap it... + if (e instanceof MapperParsingException) { + return (MapperParsingException) e; } - ParsedDocument doc = new ParsedDocument(context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), - context.source(), update).parent(source.parent()); - // reset the context to free up memory - context.reset(null, null, null); - return doc; + // Throw a more meaningful message if the document is empty. + if (source.source() != null && source.source().length() == 0) { + return new MapperParsingException("failed to parse, document is empty"); + } + + return new MapperParsingException("failed to parse", e); } static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException { From 9e52bcd166d7f1a1a8aca906ca93f3fe90f0da5c Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 09:15:29 -0500 Subject: [PATCH 24/31] Inline OrdinalsStore#addOrdinal Relates #16725 --- .../fielddata/ordinals/OrdinalsBuilder.java | 81 ++++++++++--------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java index a36af886364..bdc121b134b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java @@ -190,48 +190,55 @@ public final class OrdinalsBuilder implements Closeable { public int addOrdinal(int docID, long ordinal) { final long position = positions.get(docID); - if (position == 0L) { // on the first level - // 0 or 1 ordinal - if (firstOrdinals.get(docID) == 0L) { - firstOrdinals.set(docID, ordinal + 1); - return 1; - } else { - final long newSlice = newSlice(1); - if (firstNextLevelSlices == null) { - firstNextLevelSlices = new PagedGrowableWriter(firstOrdinals.size(), PAGE_SIZE, 3, acceptableOverheadRatio); - } - firstNextLevelSlices.set(docID, newSlice); - final long offset = startOffset(1, newSlice); - ordinals[1].set(offset, ordinal + 1); - positions.set(docID, position(1, offset)); // current position is on the 1st level and not allocated yet - return 2; - } + return firstLevel(docID, ordinal); } else { - int level = level(position); - long offset = offset(position, level); - assert offset != 0L; - if (((offset + 1) & slotsMask(level)) == 0L) { - // reached the end of the slice, allocate a new one on the next level - final long newSlice = newSlice(level + 1); - if (nextLevelSlices[level] == null) { - nextLevelSlices[level] = new PagedGrowableWriter(sizes[level], PAGE_SIZE, 1, acceptableOverheadRatio); - } - nextLevelSlices[level].set(sliceID(level, offset), newSlice); - ++level; - offset = startOffset(level, newSlice); - assert (offset & slotsMask(level)) == 0L; - } else { - // just go to the next slot - ++offset; - } - ordinals[level].set(offset, ordinal + 1); - final long newPosition = position(level, offset); - positions.set(docID, newPosition); - return numOrdinals(level, offset); + return nonFirstLevel(docID, ordinal, position); } } + private int firstLevel(int docID, long ordinal) { + // 0 or 1 ordinal + if (firstOrdinals.get(docID) == 0L) { + firstOrdinals.set(docID, ordinal + 1); + return 1; + } else { + final long newSlice = newSlice(1); + if (firstNextLevelSlices == null) { + firstNextLevelSlices = new PagedGrowableWriter(firstOrdinals.size(), PAGE_SIZE, 3, acceptableOverheadRatio); + } + firstNextLevelSlices.set(docID, newSlice); + final long offset = startOffset(1, newSlice); + ordinals[1].set(offset, ordinal + 1); + positions.set(docID, position(1, offset)); // current position is on the 1st level and not allocated yet + return 2; + } + } + + private int nonFirstLevel(int docID, long ordinal, long position) { + int level = level(position); + long offset = offset(position, level); + assert offset != 0L; + if (((offset + 1) & slotsMask(level)) == 0L) { + // reached the end of the slice, allocate a new one on the next level + final long newSlice = newSlice(level + 1); + if (nextLevelSlices[level] == null) { + nextLevelSlices[level] = new PagedGrowableWriter(sizes[level], PAGE_SIZE, 1, acceptableOverheadRatio); + } + nextLevelSlices[level].set(sliceID(level, offset), newSlice); + ++level; + offset = startOffset(level, newSlice); + assert (offset & slotsMask(level)) == 0L; + } else { + // just go to the next slot + ++offset; + } + ordinals[level].set(offset, ordinal + 1); + final long newPosition = position(level, offset); + positions.set(docID, newPosition); + return numOrdinals(level, offset); + } + public void appendOrdinals(int docID, LongsRef ords) { // First level final long firstOrd = firstOrdinals.get(docID); From 26c1bb36a2642f7ae7657d9b7f49edfb7c5988aa Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 13:07:34 -0500 Subject: [PATCH 25/31] Inline TSBA#shardOperationOnPrimary Relates #16725 --- .../action/bulk/TransportShardBulkAction.java | 373 +++++++++--------- 1 file changed, 194 insertions(+), 179 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index d7d40426a6e..30f6b03a116 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.action.update.UpdateHelper; import org.elasticsearch.action.update.UpdateRequest; @@ -111,185 +112,7 @@ public class TransportShardBulkAction extends TransportReplicationAction result = shardIndexOperation(request, indexRequest, metaData, indexShard, true); - location = locationToSync(location, result.location); - // add the response - IndexResponse indexResponse = result.response(); - setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse)); - } catch (Throwable e) { - // rethrow the failure if we are going to retry on primary and let parent failure to handle it - if (retryPrimaryException(e)) { - // restore updated versions... - for (int j = 0; j < requestIndex; j++) { - applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); - } - throw (ElasticsearchException) e; - } - if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { - logger.trace("{} failed to execute bulk item (index) {}", e, request.shardId(), indexRequest); - } else { - logger.debug("{} failed to execute bulk item (index) {}", e, request.shardId(), indexRequest); - } - // if its a conflict failure, and we already executed the request on a primary (and we execute it - // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) - // then just use the response we got from the successful execution - if (item.getPrimaryResponse() != null && isConflictException(e)) { - setResponse(item, item.getPrimaryResponse()); - } else { - setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), - new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e))); - } - } - } else if (item.request() instanceof DeleteRequest) { - DeleteRequest deleteRequest = (DeleteRequest) item.request(); - preVersions[requestIndex] = deleteRequest.version(); - preVersionTypes[requestIndex] = deleteRequest.versionType(); - - try { - // add the response - final WriteResult writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard); - DeleteResponse deleteResponse = writeResult.response(); - location = locationToSync(location, writeResult.location); - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); - } catch (Throwable e) { - // rethrow the failure if we are going to retry on primary and let parent failure to handle it - if (retryPrimaryException(e)) { - // restore updated versions... - for (int j = 0; j < requestIndex; j++) { - applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); - } - throw (ElasticsearchException) e; - } - if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { - logger.trace("{} failed to execute bulk item (delete) {}", e, request.shardId(), deleteRequest); - } else { - logger.debug("{} failed to execute bulk item (delete) {}", e, request.shardId(), deleteRequest); - } - // if its a conflict failure, and we already executed the request on a primary (and we execute it - // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) - // then just use the response we got from the successful execution - if (item.getPrimaryResponse() != null && isConflictException(e)) { - setResponse(item, item.getPrimaryResponse()); - } else { - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, - new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), e))); - } - } - } else if (item.request() instanceof UpdateRequest) { - UpdateRequest updateRequest = (UpdateRequest) item.request(); - preVersions[requestIndex] = updateRequest.version(); - preVersionTypes[requestIndex] = updateRequest.versionType(); - // We need to do the requested retries plus the initial attempt. We don't do < 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE - for (int updateAttemptsCount = 0; updateAttemptsCount <= updateRequest.retryOnConflict(); updateAttemptsCount++) { - UpdateResult updateResult; - try { - updateResult = shardUpdateOperation(metaData, request, updateRequest, indexShard); - } catch (Throwable t) { - updateResult = new UpdateResult(null, null, false, t, null); - } - if (updateResult.success()) { - if (updateResult.writeResult != null) { - location = locationToSync(location, updateResult.writeResult.location); - } - switch (updateResult.result.operation()) { - case UPSERT: - case INDEX: - WriteResult result = updateResult.writeResult; - IndexRequest indexRequest = updateResult.request(); - BytesReference indexSourceAsBytes = indexRequest.source(); - // add the response - IndexResponse indexResponse = result.response(); - UpdateResponse updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated()); - if (updateRequest.fields() != null && updateRequest.fields().length > 0) { - Tuple> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); - updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); - } - item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest); - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); - break; - case DELETE: - WriteResult writeResult = updateResult.writeResult; - DeleteResponse response = writeResult.response(); - DeleteRequest deleteRequest = updateResult.request(); - updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); - updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); - // Replace the update request to the translated delete request to execute on the replica. - item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest); - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); - break; - case NONE: - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult)); - item.setIgnoreOnReplica(); // no need to go to the replica - break; - } - // NOTE: Breaking out of the retry_on_conflict loop! - break; - } else if (updateResult.failure()) { - Throwable t = updateResult.error; - if (updateResult.retry) { - // updateAttemptCount is 0 based and marks current attempt, if it's equal to retryOnConflict we are going out of the iteration - if (updateAttemptsCount >= updateRequest.retryOnConflict()) { - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, - new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t))); - } - } else { - // rethrow the failure if we are going to retry on primary and let parent failure to handle it - if (retryPrimaryException(t)) { - // restore updated versions... - for (int j = 0; j < requestIndex; j++) { - applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); - } - throw (ElasticsearchException) t; - } - // if its a conflict failure, and we already executed the request on a primary (and we execute it - // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) - // then just use the response we got from the successful execution - if (item.getPrimaryResponse() != null && isConflictException(t)) { - setResponse(item, item.getPrimaryResponse()); - } else if (updateResult.result == null) { - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t))); - } else { - switch (updateResult.result.operation()) { - case UPSERT: - case INDEX: - IndexRequest indexRequest = updateResult.request(); - if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { - logger.trace("{} failed to execute bulk item (index) {}", t, request.shardId(), indexRequest); - } else { - logger.debug("{} failed to execute bulk item (index) {}", t, request.shardId(), indexRequest); - } - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, - new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), t))); - break; - case DELETE: - DeleteRequest deleteRequest = updateResult.request(); - if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { - logger.trace("{} failed to execute bulk item (delete) {}", t, request.shardId(), deleteRequest); - } else { - logger.debug("{} failed to execute bulk item (delete) {}", t, request.shardId(), deleteRequest); - } - setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, - new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), t))); - break; - } - } - // NOTE: Breaking out of the retry_on_conflict loop! - break; - } - - } - } - } else { - throw new IllegalStateException("Unexpected index operation: " + item.request()); - } - - assert item.getPrimaryResponse() != null; - assert preVersionTypes[requestIndex] != null; + location = handleItem(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item); } processAfterWrite(request.refresh(), indexShard, location); @@ -301,6 +124,198 @@ public class TransportShardBulkAction extends TransportReplicationAction(new BulkShardResponse(request.shardId(), responses), request); } + private Translog.Location handleItem(MetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) { + if (item.request() instanceof IndexRequest) { + location = index(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item); + } else if (item.request() instanceof DeleteRequest) { + location = delete(request, indexShard, preVersions, preVersionTypes, location, requestIndex, item); + } else if (item.request() instanceof UpdateRequest) { + Tuple tuple = update(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item); + location = tuple.v1(); + item = tuple.v2(); + } else { + throw new IllegalStateException("Unexpected index operation: " + item.request()); + } + + assert item.getPrimaryResponse() != null; + assert preVersionTypes[requestIndex] != null; + return location; + } + + private Translog.Location index(MetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) { + IndexRequest indexRequest = (IndexRequest) item.request(); + preVersions[requestIndex] = indexRequest.version(); + preVersionTypes[requestIndex] = indexRequest.versionType(); + try { + WriteResult result = shardIndexOperation(request, indexRequest, metaData, indexShard, true); + location = locationToSync(location, result.location); + // add the response + IndexResponse indexResponse = result.response(); + setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse)); + } catch (Throwable e) { + // rethrow the failure if we are going to retry on primary and let parent failure to handle it + if (retryPrimaryException(e)) { + // restore updated versions... + for (int j = 0; j < requestIndex; j++) { + applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); + } + throw (ElasticsearchException) e; + } + logFailure(e, "index", request.shardId(), indexRequest); + // if its a conflict failure, and we already executed the request on a primary (and we execute it + // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) + // then just use the response we got from the successful execution + if (item.getPrimaryResponse() != null && isConflictException(e)) { + setResponse(item, item.getPrimaryResponse()); + } else { + setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), + new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e))); + } + } + return location; + } + + private > void logFailure(Throwable e, String operation, ShardId shardId, ReplicationRequest request) { + if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { + logger.trace("{} failed to execute bulk item ({}) {}", e, shardId, operation, request); + } else { + logger.debug("{} failed to execute bulk item ({}) {}", e, shardId, operation, request); + } + } + + private Translog.Location delete(BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) { + DeleteRequest deleteRequest = (DeleteRequest) item.request(); + preVersions[requestIndex] = deleteRequest.version(); + preVersionTypes[requestIndex] = deleteRequest.versionType(); + + try { + // add the response + final WriteResult writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard); + DeleteResponse deleteResponse = writeResult.response(); + location = locationToSync(location, writeResult.location); + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); + } catch (Throwable e) { + // rethrow the failure if we are going to retry on primary and let parent failure to handle it + if (retryPrimaryException(e)) { + // restore updated versions... + for (int j = 0; j < requestIndex; j++) { + applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); + } + throw (ElasticsearchException) e; + } + logFailure(e, "delete", request.shardId(), deleteRequest); + // if its a conflict failure, and we already executed the request on a primary (and we execute it + // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) + // then just use the response we got from the successful execution + if (item.getPrimaryResponse() != null && isConflictException(e)) { + setResponse(item, item.getPrimaryResponse()); + } else { + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, + new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), e))); + } + } + return location; + } + + private Tuple update(MetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) { + UpdateRequest updateRequest = (UpdateRequest) item.request(); + preVersions[requestIndex] = updateRequest.version(); + preVersionTypes[requestIndex] = updateRequest.versionType(); + // We need to do the requested retries plus the initial attempt. We don't do < 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE + for (int updateAttemptsCount = 0; updateAttemptsCount <= updateRequest.retryOnConflict(); updateAttemptsCount++) { + UpdateResult updateResult; + try { + updateResult = shardUpdateOperation(metaData, request, updateRequest, indexShard); + } catch (Throwable t) { + updateResult = new UpdateResult(null, null, false, t, null); + } + if (updateResult.success()) { + if (updateResult.writeResult != null) { + location = locationToSync(location, updateResult.writeResult.location); + } + switch (updateResult.result.operation()) { + case UPSERT: + case INDEX: + WriteResult result = updateResult.writeResult; + IndexRequest indexRequest = updateResult.request(); + BytesReference indexSourceAsBytes = indexRequest.source(); + // add the response + IndexResponse indexResponse = result.response(); + UpdateResponse updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated()); + if (updateRequest.fields() != null && updateRequest.fields().length > 0) { + Tuple> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); + updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); + } + item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest); + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); + break; + case DELETE: + WriteResult writeResult = updateResult.writeResult; + DeleteResponse response = writeResult.response(); + DeleteRequest deleteRequest = updateResult.request(); + updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); + updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); + // Replace the update request to the translated delete request to execute on the replica. + item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest); + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); + break; + case NONE: + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult)); + item.setIgnoreOnReplica(); // no need to go to the replica + break; + } + // NOTE: Breaking out of the retry_on_conflict loop! + break; + } else if (updateResult.failure()) { + Throwable t = updateResult.error; + if (updateResult.retry) { + // updateAttemptCount is 0 based and marks current attempt, if it's equal to retryOnConflict we are going out of the iteration + if (updateAttemptsCount >= updateRequest.retryOnConflict()) { + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, + new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t))); + } + } else { + // rethrow the failure if we are going to retry on primary and let parent failure to handle it + if (retryPrimaryException(t)) { + // restore updated versions... + for (int j = 0; j < requestIndex; j++) { + applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); + } + throw (ElasticsearchException) t; + } + // if its a conflict failure, and we already executed the request on a primary (and we execute it + // again, due to primary relocation and only processing up to N bulk items when the shard gets closed) + // then just use the response we got from the successful execution + if (item.getPrimaryResponse() != null && isConflictException(t)) { + setResponse(item, item.getPrimaryResponse()); + } else if (updateResult.result == null) { + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t))); + } else { + switch (updateResult.result.operation()) { + case UPSERT: + case INDEX: + IndexRequest indexRequest = updateResult.request(); + logFailure(t, "index", request.shardId(), indexRequest); + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, + new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), t))); + break; + case DELETE: + DeleteRequest deleteRequest = updateResult.request(); + logFailure(t, "delete", request.shardId(), deleteRequest); + setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, + new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), t))); + break; + } + } + // NOTE: Breaking out of the retry_on_conflict loop! + break; + } + + } + } + return Tuple.tuple(location, item); + } + private void setResponse(BulkItemRequest request, BulkItemResponse response) { request.setPrimaryResponse(response); if (response.isFailed()) { From 5bbb1312b1b752a87d8ab1721042fad3f2133a7e Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 13 Feb 2016 14:17:26 -0500 Subject: [PATCH 26/31] Inline WildcardExpressionResolver#resolve Relates #16725 --- .../metadata/IndexNameExpressionResolver.java | 212 ++++++++++-------- 1 file changed, 123 insertions(+), 89 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 0661f6c4362..cca633a7651 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.joda.DateMathParser; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.indices.IndexClosedException; import org.joda.time.DateTimeZone; @@ -527,29 +526,35 @@ public class IndexNameExpressionResolver extends AbstractComponent { return expressions; } - if (expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0))) || Regex.isMatchAllPattern(expressions.get(0)))) { - if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { - return Arrays.asList(metaData.concreteAllIndices()); - } else if (options.expandWildcardsOpen()) { - return Arrays.asList(metaData.concreteAllOpenIndices()); - } else if (options.expandWildcardsClosed()) { - return Arrays.asList(metaData.concreteAllClosedIndices()); - } else { - return Collections.emptyList(); - } + if (isEmptyOrTrivialWildcard(expressions)) { + return resolveEmptyOrTrivialWildcard(options, metaData, true); } + Set result = innerResolve(context, expressions, options, metaData); + + if (result == null) { + return expressions; + } + if (result.isEmpty() && !options.allowNoIndices()) { + IndexNotFoundException infe = new IndexNotFoundException((String)null); + infe.setResources("index_or_alias", expressions.toArray(new String[0])); + throw infe; + } + return new ArrayList<>(result); + } + + private Set innerResolve(Context context, List expressions, IndicesOptions options, MetaData metaData) { Set result = null; for (int i = 0; i < expressions.size(); i++) { String expression = expressions.get(i); - if (metaData.getAliasAndIndexLookup().containsKey(expression)) { + if (aliasOrIndexExists(metaData, expression)) { if (result != null) { result.add(expression); } continue; } if (Strings.isEmpty(expression)) { - throw new IndexNotFoundException(expression); + throw infe(expression); } boolean add = true; if (expression.charAt(0) == '+') { @@ -557,32 +562,19 @@ public class IndexNameExpressionResolver extends AbstractComponent { if (i == 0) { result = new HashSet<>(); } - add = true; expression = expression.substring(1); } else if (expression.charAt(0) == '-') { // if its the first, fill it with all the indices... if (i == 0) { - String[] concreteIndices; - if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { - concreteIndices = metaData.concreteAllIndices(); - } else if (options.expandWildcardsOpen()) { - concreteIndices = metaData.concreteAllOpenIndices(); - } else if (options.expandWildcardsClosed()) { - concreteIndices = metaData.concreteAllClosedIndices(); - } else { - assert false : "Shouldn't end up here"; - concreteIndices = Strings.EMPTY_ARRAY; - } - result = new HashSet<>(Arrays.asList(concreteIndices)); + List concreteIndices = resolveEmptyOrTrivialWildcard(options, metaData, false); + result = new HashSet<>(concreteIndices); } add = false; expression = expression.substring(1); } if (!Regex.isSimpleMatchPattern(expression)) { - if (!options.ignoreUnavailable() && !metaData.getAliasAndIndexLookup().containsKey(expression)) { - IndexNotFoundException infe = new IndexNotFoundException(expression); - infe.setResources("index_or_alias", expression); - throw infe; + if (!unavailableIgnoredOrExists(options, metaData, expression)) { + throw infe(expression); } if (result != null) { if (add) { @@ -595,77 +587,119 @@ public class IndexNameExpressionResolver extends AbstractComponent { } if (result == null) { // add all the previous ones... - result = new HashSet<>(); - result.addAll(expressions.subList(0, i)); + result = new HashSet<>(expressions.subList(0, i)); } - final IndexMetaData.State excludeState; - if (options.expandWildcardsOpen() && options.expandWildcardsClosed()){ - excludeState = null; - } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed() == false) { - excludeState = IndexMetaData.State.CLOSE; - } else if (options.expandWildcardsClosed() && options.expandWildcardsOpen() == false) { - excludeState = IndexMetaData.State.OPEN; - } else { - assert false : "this shouldn't get called if wildcards expand to none"; - excludeState = null; - } - - final Map matches; - if (Regex.isMatchAllPattern(expression)) { - // Can only happen if the expressions was initially: '-*' - matches = metaData.getAliasAndIndexLookup(); - } else if (expression.indexOf("*") == expression.length() - 1) { - // Suffix wildcard: - assert expression.length() >= 2 : "expression [" + expression + "] should have at least a length of 2"; - String fromPrefix = expression.substring(0, expression.length() - 1); - char[] toPrefixCharArr = fromPrefix.toCharArray(); - toPrefixCharArr[toPrefixCharArr.length - 1]++; - String toPrefix = new String(toPrefixCharArr); - matches = metaData.getAliasAndIndexLookup().subMap(fromPrefix, toPrefix); - } else { - // Other wildcard expressions: - final String pattern = expression; - matches = metaData.getAliasAndIndexLookup() - .entrySet() - .stream() - .filter(e -> Regex.simpleMatch(pattern, e.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - Set expand = new HashSet<>(); - for (Map.Entry entry : matches.entrySet()) { - AliasOrIndex aliasOrIndex = entry.getValue(); - if (context.isPreserveAliases() && aliasOrIndex.isAlias()) { - expand.add(entry.getKey()); - } else { - for (IndexMetaData meta : aliasOrIndex.getIndices()) { - if (excludeState == null || meta.getState() != excludeState) { - expand.add(meta.getIndex().getName()); - } - } - } - } + final IndexMetaData.State excludeState = excludeState(options); + final Map matches = matches(metaData, expression); + Set expand = expand(context, excludeState, matches); if (add) { result.addAll(expand); } else { result.removeAll(expand); } - if (matches.isEmpty() && options.allowNoIndices() == false) { - IndexNotFoundException infe = new IndexNotFoundException(expression); - infe.setResources("index_or_alias", expression); - throw infe; + if (!noIndicesAllowedOrMatches(options, matches)) { + throw infe(expression); } } - if (result == null) { - return expressions; + return result; + } + + private boolean noIndicesAllowedOrMatches(IndicesOptions options, Map matches) { + return options.allowNoIndices() || !matches.isEmpty(); + } + + private boolean unavailableIgnoredOrExists(IndicesOptions options, MetaData metaData, String expression) { + return options.ignoreUnavailable() || aliasOrIndexExists(metaData, expression); + } + + private boolean aliasOrIndexExists(MetaData metaData, String expression) { + return metaData.getAliasAndIndexLookup().containsKey(expression); + } + + private static IndexNotFoundException infe(String expression) { + IndexNotFoundException infe = new IndexNotFoundException(expression); + infe.setResources("index_or_alias", expression); + return infe; + } + + private static IndexMetaData.State excludeState(IndicesOptions options) { + final IndexMetaData.State excludeState; + if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { + excludeState = null; + } else if (options.expandWildcardsOpen() && options.expandWildcardsClosed() == false) { + excludeState = IndexMetaData.State.CLOSE; + } else if (options.expandWildcardsClosed() && options.expandWildcardsOpen() == false) { + excludeState = IndexMetaData.State.OPEN; + } else { + assert false : "this shouldn't get called if wildcards expand to none"; + excludeState = null; } - if (result.isEmpty() && !options.allowNoIndices()) { - IndexNotFoundException infe = new IndexNotFoundException((String)null); - infe.setResources("index_or_alias", expressions.toArray(new String[0])); - throw infe; + return excludeState; + } + + private static Map matches(MetaData metaData, String expression) { + if (Regex.isMatchAllPattern(expression)) { + // Can only happen if the expressions was initially: '-*' + return metaData.getAliasAndIndexLookup(); + } else if (expression.indexOf("*") == expression.length() - 1) { + return suffixWildcard(metaData, expression); + } else { + return otherWildcard(metaData, expression); + } + } + + private static Map suffixWildcard(MetaData metaData, String expression) { + assert expression.length() >= 2 : "expression [" + expression + "] should have at least a length of 2"; + String fromPrefix = expression.substring(0, expression.length() - 1); + char[] toPrefixCharArr = fromPrefix.toCharArray(); + toPrefixCharArr[toPrefixCharArr.length - 1]++; + String toPrefix = new String(toPrefixCharArr); + return metaData.getAliasAndIndexLookup().subMap(fromPrefix, toPrefix); + } + + private static Map otherWildcard(MetaData metaData, String expression) { + final String pattern = expression; + return metaData.getAliasAndIndexLookup() + .entrySet() + .stream() + .filter(e -> Regex.simpleMatch(pattern, e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private static Set expand(Context context, IndexMetaData.State excludeState, Map matches) { + Set expand = new HashSet<>(); + for (Map.Entry entry : matches.entrySet()) { + AliasOrIndex aliasOrIndex = entry.getValue(); + if (context.isPreserveAliases() && aliasOrIndex.isAlias()) { + expand.add(entry.getKey()); + } else { + for (IndexMetaData meta : aliasOrIndex.getIndices()) { + if (excludeState == null || meta.getState() != excludeState) { + expand.add(meta.getIndex().getName()); + } + } + } + } + return expand; + } + + private boolean isEmptyOrTrivialWildcard(List expressions) { + return expressions.isEmpty() || (expressions.size() == 1 && (MetaData.ALL.equals(expressions.get(0))) || Regex.isMatchAllPattern(expressions.get(0))); + } + + private List resolveEmptyOrTrivialWildcard(IndicesOptions options, MetaData metaData, boolean assertEmpty) { + if (options.expandWildcardsOpen() && options.expandWildcardsClosed()) { + return Arrays.asList(metaData.concreteAllIndices()); + } else if (options.expandWildcardsOpen()) { + return Arrays.asList(metaData.concreteAllOpenIndices()); + } else if (options.expandWildcardsClosed()) { + return Arrays.asList(metaData.concreteAllClosedIndices()); + } else { + assert assertEmpty : "Shouldn't end up here"; + return Collections.emptyList(); } - return new ArrayList<>(result); } } From 7273948188b0a5edc4d62648a3ec7c28b03f1976 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 14 Feb 2016 00:10:56 -0500 Subject: [PATCH 27/31] Inline DocumentParser#parseObject Relates #16725 --- .../index/mapper/DocumentParser.java | 94 ++++++++++--------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 98721b4358a..6e0865fcc0b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -262,22 +262,7 @@ class DocumentParser implements Closeable { ObjectMapper.Nested nested = mapper.nested(); if (nested.isNested()) { - context = context.createNestedContext(mapper.fullPath()); - ParseContext.Document nestedDoc = context.doc(); - ParseContext.Document parentDoc = nestedDoc.getParent(); - // pre add the uid field if possible (id was already provided) - IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME); - if (uidField != null) { - // we don't need to add it as a full uid field in nested docs, since we don't need versioning - // we also rely on this for UidField#loadVersion - - // this is a deeply nested field - nestedDoc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); - } - // the type of the nested doc starts with __, so we can identify that its a nested one in filters - // note, we don't prefix it with the type of the doc since it allows us to execute a nested query - // across types (for example, with similar nested objects) - nestedDoc.add(new Field(TypeFieldMapper.NAME, mapper.nestedTypePathAsString(), TypeFieldMapper.Defaults.FIELD_TYPE)); + context = nestedContext(context, mapper); } // if we are at the end of the previous object, advance @@ -290,6 +275,15 @@ class DocumentParser implements Closeable { } ObjectMapper update = null; + update = innerParseObject(context, mapper, parser, currentFieldName, token, update); + // restore the enable path flag + if (nested.isNested()) { + nested(context, nested); + } + return update; + } + + private static ObjectMapper innerParseObject(ParseContext context, ObjectMapper mapper, XContentParser parser, String currentFieldName, XContentParser.Token token, ObjectMapper update) throws IOException { while (token != XContentParser.Token.END_OBJECT) { ObjectMapper newUpdate = null; if (token == XContentParser.Token.START_OBJECT) { @@ -314,34 +308,50 @@ class DocumentParser implements Closeable { } } } - // restore the enable path flag - if (nested.isNested()) { - ParseContext.Document nestedDoc = context.doc(); - ParseContext.Document parentDoc = nestedDoc.getParent(); - if (nested.isIncludeInParent()) { - for (IndexableField field : nestedDoc.getFields()) { - if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { - continue; - } else { - parentDoc.add(field); - } - } - } - if (nested.isIncludeInRoot()) { - ParseContext.Document rootDoc = context.rootDoc(); - // don't add it twice, if its included in parent, and we are handling the master doc... - if (!nested.isIncludeInParent() || parentDoc != rootDoc) { - for (IndexableField field : nestedDoc.getFields()) { - if (field.name().equals(UidFieldMapper.NAME) || field.name().equals(TypeFieldMapper.NAME)) { - continue; - } else { - rootDoc.add(field); - } - } - } + return update; + } + + private static void nested(ParseContext context, ObjectMapper.Nested nested) { + ParseContext.Document nestedDoc = context.doc(); + ParseContext.Document parentDoc = nestedDoc.getParent(); + if (nested.isIncludeInParent()) { + addFields(nestedDoc, parentDoc); + } + if (nested.isIncludeInRoot()) { + ParseContext.Document rootDoc = context.rootDoc(); + // don't add it twice, if its included in parent, and we are handling the master doc... + if (!nested.isIncludeInParent() || parentDoc != rootDoc) { + addFields(nestedDoc, rootDoc); } } - return update; + } + + private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { + for (IndexableField field : nestedDoc.getFields()) { + if (!field.name().equals(UidFieldMapper.NAME) && !field.name().equals(TypeFieldMapper.NAME)) { + rootDoc.add(field); + } + } + } + + private static ParseContext nestedContext(ParseContext context, ObjectMapper mapper) { + context = context.createNestedContext(mapper.fullPath()); + ParseContext.Document nestedDoc = context.doc(); + ParseContext.Document parentDoc = nestedDoc.getParent(); + // pre add the uid field if possible (id was already provided) + IndexableField uidField = parentDoc.getField(UidFieldMapper.NAME); + if (uidField != null) { + // we don't need to add it as a full uid field in nested docs, since we don't need versioning + // we also rely on this for UidField#loadVersion + + // this is a deeply nested field + nestedDoc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), UidFieldMapper.Defaults.NESTED_FIELD_TYPE)); + } + // the type of the nested doc starts with __, so we can identify that its a nested one in filters + // note, we don't prefix it with the type of the doc since it allows us to execute a nested query + // across types (for example, with similar nested objects) + nestedDoc.add(new Field(TypeFieldMapper.NAME, mapper.nestedTypePathAsString(), TypeFieldMapper.Defaults.FIELD_TYPE)); + return context; } private static Mapper parseObjectOrField(ParseContext context, Mapper mapper) throws IOException { From 7a7f6055dc26929de2ae3f37f779f9592025a636 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 14 Feb 2016 00:34:07 -0500 Subject: [PATCH 28/31] Inline InternalEngine#innerIndex Relates #16725 --- .../index/engine/InternalEngine.java | 53 ++++++++++++------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index ff738c0140b..dd023363984 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -362,13 +362,12 @@ public class InternalEngine extends Engine { } long expectedVersion = index.version(); - if (index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) { - if (index.origin() == Operation.Origin.RECOVERY) { - return false; - } else { + if (isVersionConflictForWrites(index, currentVersion, deleted, expectedVersion)) { + if (index.origin() != Operation.Origin.RECOVERY) { throw new VersionConflictEngineException(shardId, index.type(), index.id(), - index.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted)); + index.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted)); } + return false; } long updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion); @@ -378,22 +377,9 @@ public class InternalEngine extends Engine { if (currentVersion == Versions.NOT_FOUND) { // document does not exists, we can optimize for create created = true; - if (index.docs().size() > 1) { - indexWriter.addDocuments(index.docs()); - } else { - indexWriter.addDocument(index.docs().get(0)); - } + index(index, indexWriter); } else { - if (versionValue != null) { - created = versionValue.delete(); // we have a delete which is not GC'ed... - } else { - created = false; - } - if (index.docs().size() > 1) { - indexWriter.updateDocuments(index.uid(), index.docs()); - } else { - indexWriter.updateDocument(index.uid(), index.docs().get(0)); - } + created = update(index, versionValue, indexWriter); } Translog.Location translogLocation = translog.add(new Translog.Index(index)); @@ -403,6 +389,33 @@ public class InternalEngine extends Engine { } } + private static boolean update(Index index, VersionValue versionValue, IndexWriter indexWriter) throws IOException { + boolean created; + if (versionValue != null) { + created = versionValue.delete(); // we have a delete which is not GC'ed... + } else { + created = false; + } + if (index.docs().size() > 1) { + indexWriter.updateDocuments(index.uid(), index.docs()); + } else { + indexWriter.updateDocument(index.uid(), index.docs().get(0)); + } + return created; + } + + private static void index(Index index, IndexWriter indexWriter) throws IOException { + if (index.docs().size() > 1) { + indexWriter.addDocuments(index.docs()); + } else { + indexWriter.addDocument(index.docs().get(0)); + } + } + + private boolean isVersionConflictForWrites(Index index, long currentVersion, boolean deleted, long expectedVersion) { + return index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted); + } + @Override public void delete(Delete delete) throws EngineException { try (ReleasableLock lock = readLock.acquire()) { From 7101ecea94757463a66b38d1cbb006a1beff37f1 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 14 Feb 2016 09:07:02 -0500 Subject: [PATCH 29/31] Inline ReplicationPhase# Relates #16725 --- .../TransportReplicationAction.java | 95 ++++++++++--------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 894b19fedb7..7fc18266816 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -83,6 +83,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.function.Supplier; /** @@ -874,23 +875,48 @@ public abstract class TransportReplicationAction shards = shards(shardRoutingTable); + boolean executeOnReplica = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings()); + DiscoveryNodes nodes = state.getNodes(); if (shards.isEmpty()) { logger.debug("replication phase for request [{}] on [{}] is skipped due to index deletion after primary operation", replicaRequest, shardId); } // we calculate number of target nodes to send replication operations, including nodes with relocating shards + AtomicInteger numberOfPendingShardInstances = new AtomicInteger(); + this.totalShards = countTotalAndPending(shards, executeOnReplica, nodes, numberOfPendingShardInstances); + this.pending = numberOfPendingShardInstances; + this.shards = shards; + this.executeOnReplica = executeOnReplica; + this.nodes = nodes; + if (logger.isTraceEnabled()) { + logger.trace("replication phase started. pending [{}], action [{}], request [{}], cluster state version used [{}]", pending.get(), + transportReplicaAction, replicaRequest, state.version()); + } + } + + private int countTotalAndPending(List shards, boolean executeOnReplica, DiscoveryNodes nodes, AtomicInteger pending) { + assert pending.get() == 0; + int numberOfIgnoredShardInstances = performOnShards(shards, executeOnReplica, nodes, shard -> pending.incrementAndGet(), shard -> pending.incrementAndGet()); + // one for the local primary copy + return 1 + numberOfIgnoredShardInstances + pending.get(); + } + + private int performOnShards(List shards, boolean executeOnReplica, DiscoveryNodes nodes, Consumer onLocalShard, Consumer onRelocatingShard) { int numberOfIgnoredShardInstances = 0; - int numberOfPendingShardInstances = 0; for (ShardRouting shard : shards) { - // the following logic to select the shards to replicate to is mirrored and explained in the doRun method below if (shard.primary() == false && executeOnReplica == false) { + // If the replicas use shadow replicas, there is no reason to + // perform the action on the replica, so skip it and + // immediately return + + // this delays mapping updates on replicas because they have + // to wait until they get the new mapping through the cluster + // state, which is why we recommend pre-defined mappings for + // indices using shadow replicas numberOfIgnoredShardInstances++; continue; } @@ -898,20 +924,26 @@ public abstract class TransportReplicationAction shards(IndexShardRoutingTable shardRoutingTable) { + return (shardRoutingTable != null) ? shardRoutingTable.shards() : Collections.emptyList(); } /** @@ -951,36 +983,7 @@ public abstract class TransportReplicationAction performOnReplica(shard), shard -> performOnReplica(shard.buildTargetRelocatingShard())); } /** From bd5c7f088974bbde574ee00fbe9d2b4b2e687c8c Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 14 Feb 2016 09:37:10 -0500 Subject: [PATCH 30/31] Inline TimeValue#parseTimeValue Relates #16725 --- .../org/elasticsearch/common/unit/TimeValue.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java index bbe1fbbf055..c467a0c18a8 100644 --- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java +++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java @@ -265,17 +265,17 @@ public class TimeValue implements Streamable { long millis; String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim(); if (lowerSValue.endsWith("ms")) { - millis = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 2))); + millis = parse(lowerSValue, 2, 1); } else if (lowerSValue.endsWith("s")) { - millis = (long) Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * 1000; + millis = parse(lowerSValue, 1, 1000); } else if (lowerSValue.endsWith("m")) { - millis = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * 60 * 1000); + millis = parse(lowerSValue, 1, 60 * 1000); } else if (lowerSValue.endsWith("h")) { - millis = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * 60 * 60 * 1000); + millis = parse(lowerSValue, 1, 60 * 60 * 1000); } else if (lowerSValue.endsWith("d")) { - millis = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * 24 * 60 * 60 * 1000); + millis = parse(lowerSValue, 1, 24 * 60 * 60 * 1000); } else if (lowerSValue.endsWith("w")) { - millis = (long) (Double.parseDouble(lowerSValue.substring(0, lowerSValue.length() - 1)) * 7 * 24 * 60 * 60 * 1000); + millis = parse(lowerSValue, 1, 7 * 24 * 60 * 60 * 1000); } else if (lowerSValue.equals("-1")) { // Allow this special value to be unit-less: millis = -1; @@ -292,6 +292,10 @@ public class TimeValue implements Streamable { } } + private static long parse(String s, int suffixLength, long scale) { + return (long) (Double.parseDouble(s.substring(0, s.length() - suffixLength)) * scale); + } + static final long C0 = 1L; static final long C1 = C0 * 1000L; static final long C2 = C1 * 1000L; From a996e218590134694638efc9807e5e40c6043daf Mon Sep 17 00:00:00 2001 From: Henrik Nordvik Date: Mon, 22 Feb 2016 13:30:37 -0800 Subject: [PATCH 31/31] Document cpu usage in _cat/nodes Closes #16775 --- docs/reference/cat/nodes.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index 3dc174bce72..1690baa55f3 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -102,6 +102,7 @@ descriptors percentage |1 |`file_desc.max` |`fdm`, `fileDescriptorMax` |No |Maximum number of file descriptors |1024 |`load` |`l` |No |Most recent load average |0.22 +|`cpu` | |No |Recent system CPU usage as percent |12 |`uptime` |`u` |No |Node uptime |17.3m |`node.role` |`r`, `role`, `dc`, `nodeRole` |Yes |Data node (d); Client node (c) |d