Merge pull request #17176 from jasontedor/its-not-easy-being-green

Vagrant tests should be green
This commit is contained in:
Jason Tedor 2016-03-18 15:28:37 -04:00
commit 96c6ac2d3f
12 changed files with 219 additions and 61 deletions

View File

@ -46,7 +46,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFileAttributeView;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@ -97,8 +99,11 @@ class InstallPluginCommand extends Command {
// TODO: make this a resource file generated by gradle
static final Set<String> MODULES = unmodifiableSet(newHashSet(
"ingest-grok",
"lang-expression",
"lang-groovy"));
"lang-groovy",
"lang-painless",
"reindex"));
// TODO: make this a resource file generated by gradle
static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(new LinkedHashSet<>(Arrays.asList(
@ -111,8 +116,9 @@ class InstallPluginCommand extends Command {
"discovery-azure",
"discovery-ec2",
"discovery-gce",
"ingest-attachment",
"ingest-geoip",
"lang-javascript",
"lang-painless",
"lang-python",
"mapper-attachments",
"mapper-murmur3",
@ -235,7 +241,15 @@ class InstallPluginCommand extends Command {
private Path unzip(Path zip, Path pluginsDir) throws IOException, UserError {
// unzip plugin to a staging temp dir
Path target = Files.createTempDirectory(pluginsDir, ".installing-");
Set<PosixFilePermission> perms = new HashSet<>();
perms.add(PosixFilePermission.OWNER_EXECUTE);
perms.add(PosixFilePermission.OWNER_READ);
perms.add(PosixFilePermission.OWNER_WRITE);
perms.add(PosixFilePermission.GROUP_READ);
perms.add(PosixFilePermission.GROUP_EXECUTE);
perms.add(PosixFilePermission.OTHERS_READ);
perms.add(PosixFilePermission.OTHERS_EXECUTE);
Path target = Files.createTempDirectory(pluginsDir, ".installing-", PosixFilePermissions.asFileAttribute(perms));
Files.createDirectories(target);
boolean hasEsDir = false;
@ -424,6 +438,10 @@ class InstallPluginCommand extends Command {
// create the plugin's config dir "if necessary"
Files.createDirectories(destConfigDir);
final PosixFileAttributes destConfigDirAttributes =
Files.getFileAttributeView(destConfigDir.getParent(), PosixFileAttributeView.class).readAttributes();
setOwnerGroup(destConfigDir, destConfigDirAttributes);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpConfigDir)) {
for (Path srcFile : stream) {
if (Files.isDirectory(srcFile)) {
@ -433,9 +451,17 @@ class InstallPluginCommand extends Command {
Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
if (Files.exists(destFile) == false) {
Files.copy(srcFile, destFile);
setOwnerGroup(destFile, destConfigDirAttributes);
}
}
}
IOUtils.rm(tmpConfigDir); // clean up what we just copied
}
private static void setOwnerGroup(Path path, PosixFileAttributes attributes) throws IOException {
PosixFileAttributeView fileAttributeView = Files.getFileAttributeView(path, PosixFileAttributeView.class);
fileAttributeView.setOwner(attributes.owner());
fileAttributeView.setGroup(attributes.group());
}
}

View File

@ -19,10 +19,9 @@
package org.elasticsearch.plugins;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.varia.NullAppender;
import org.elasticsearch.cli.MultiCommand;
import org.elasticsearch.cli.Terminal;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
@ -40,7 +39,20 @@ public class PluginCli extends MultiCommand {
}
public static void main(String[] args) throws Exception {
BasicConfigurator.configure(new NullAppender());
// initialize default for es.logger.level because we will not read the logging.yml
String loggerLevel = System.getProperty("es.logger.level", "INFO");
// Set the appender for all potential log files to terminal so that other components that use the logger print out the
// same terminal.
// The reason for this is that the plugin cli cannot be configured with a file appender because when the plugin command is
// executed there is no way of knowing where the logfiles should be placed. For example, if elasticsearch
// is run as service then the logs should be at /var/log/elasticsearch but when started from the tar they should be at es.home/logs.
// Therefore we print to Terminal.
Environment loggingEnvironment = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
.put("appender.terminal.type", "terminal")
.put("rootLogger", "${es.logger.level}, terminal")
.put("es.logger.level", loggerLevel)
.build(), Terminal.DEFAULT);
LogConfigurator.configure(loggingEnvironment.settings(), false);
Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
exit(new PluginCli(env).main(args, Terminal.DEFAULT));
}

View File

@ -350,21 +350,19 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
/**
* Suck up all the empty directories that we need to install into the path.
*/
Closure suckUpEmptyDirectories = { path ->
Closure suckUpEmptyDirectories = { path, u, g ->
into(path) {
from "${packagingFiles}/${path}"
includeEmptyDirs true
createDirectoryEntry true
/* All of these empty directories have this ownership. We're just
lucky! */
user 'elasticsearch'
permissionGroup 'elasticsearch'
user u
permissionGroup g
}
}
suckUpEmptyDirectories('/var/run')
suckUpEmptyDirectories('/var/log')
suckUpEmptyDirectories('/var/lib')
suckUpEmptyDirectories('/usr/share/elasticsearch')
suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch')
suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch')
suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch')
suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root')
}
}

View File

@ -1 +1 @@
plugin_test_cases.bash
module_and_plugin_test_cases.bash

View File

@ -0,0 +1 @@
module_and_plugin_test_cases.bash

View File

@ -1 +0,0 @@
plugin_test_cases.bash

View File

@ -33,13 +33,14 @@
##################################
# Common test cases for both tar and rpm/deb based plugin tests
##################################
# This file is symlinked to both 25_tar_plugins.bats and 50_plugins.bats so its
# This file is symlinked to both 25_tar_plugins.bats and 50_modules_and_plugins.bats so its
# executed twice - once to test plugins using the tar distribution and once to
# test files using the rpm distribution or the deb distribution, whichever the
# system uses.
# Load test utilities
load packaging_test_utils
load modules
load plugins
setup() {
@ -219,16 +220,41 @@ fi
install_and_check_plugin discovery ec2 aws-java-sdk-core-*.jar
}
@test "[$GROUP] install lang-expression plugin" {
install_and_check_plugin lang expression
@test "[$GROUP] install ingest-attachment plugin" {
# we specify the version on the poi-3.13.jar so that the test does
# not spuriously pass if the jar is missing but the other poi jars
# are present
install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.13.jar
}
@test "[$GROUP] install lang-groovy plugin" {
install_and_check_plugin lang groovy
@test "[$GROUP] install ingest-geoip plugin" {
install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
}
@test "[$GROUP] install lang-painless plugin" {
install_and_check_plugin lang painless
@test "[$GROUP] check ingest-grok module" {
check_module ingest-grok jcodings-*.jar joni-*.jar
}
@test "[$GROUP] check lang-expression module" {
# we specify the version on the asm-5.0.4.jar so that the test does
# not spuriously pass if the jar is missing but the other asm jars
# are present
check_secure_module lang-expression antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar lucene-expressions-*.jar
}
@test "[$GROUP] check lang-groovy module" {
check_secure_module lang-groovy groovy-*-indy.jar
}
@test "[$GROUP] check lang-mustache module" {
check_secure_module lang-mustache compiler-*.jar
}
@test "[$GROUP] check lang-painless module" {
# we specify the version on the asm-5.0.4.jar so that the test does
# not spuriously pass if the jar is missing but the other asm jars
# are present
check_secure_module lang-painless antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar
}
@test "[$GROUP] install javascript plugin" {
@ -247,6 +273,14 @@ fi
install_and_check_plugin mapper murmur3
}
@test "[$GROUP] check reindex module" {
check_module reindex
}
@test "[$GROUP] install repository-hdfs plugin" {
install_and_check_plugin repository hdfs hadoop-client-*.jar hadoop-common-*.jar hadoop-annotations-*.jar hadoop-auth-*.jar hadoop-hdfs-*.jar htrace-core-*.jar guava-*.jar protobuf-java-*.jar commons-logging-*.jar commons-cli-*.jar commons-collections-*.jar commons-configuration-*.jar commons-io-*.jar commons-lang-*.jar servlet-api-*.jar slf4j-api-*.jar
}
@test "[$GROUP] install size mapper plugin" {
install_and_check_plugin mapper size
}
@ -264,7 +298,7 @@ fi
}
@test "[$GROUP] check the installed plugins can be listed with 'plugins list' and result matches the list of plugins in plugins pom" {
"$ESHOME/bin/elasticsearch-plugin" list | tail -n +2 | sed 's/^......//' > /tmp/installed
"$ESHOME/bin/elasticsearch-plugin" list > /tmp/installed
compare_plugins_list "/tmp/installed" "'plugins list'"
}
@ -321,16 +355,12 @@ fi
remove_plugin discovery-ec2
}
@test "[$GROUP] remove lang-expression plugin" {
remove_plugin lang-expression
@test "[$GROUP] remove ingest-attachment plugin" {
remove_plugin ingest-attachment
}
@test "[$GROUP] remove lang-groovy plugin" {
remove_plugin lang-groovy
}
@test "[$GROUP] remove lang-painless plugin" {
remove_plugin lang-painless
@test "[$GROUP] remove ingest-geoip plugin" {
remove_plugin ingest-geoip
}
@test "[$GROUP] remove javascript plugin" {
@ -357,12 +387,12 @@ fi
remove_plugin repository-azure
}
@test "[$GROUP] remove repository-s3 plugin" {
remove_plugin repository-s3
@test "[$GROUP] remove repository-hdfs plugin" {
remove_plugin repository-hdfs
}
@test "[$GROUP] remove site example plugin" {
remove_plugin site-example
@test "[$GROUP] remove repository-s3 plugin" {
remove_plugin repository-s3
}
@test "[$GROUP] remove store-smb plugin" {
@ -393,14 +423,14 @@ fi
local loglines=$(cat /tmp/plugin-cli-output | wc -l)
if [ "$GROUP" == "TAR PLUGINS" ]; then
# tar extraction does not create the plugins directory so the plugin tool will print an additional line that the directory will be created
[ "$loglines" -eq "7" ] || {
echo "Expected 7 lines but the output was:"
[ "$loglines" -eq "3" ] || {
echo "Expected 3 lines but the output was:"
cat /tmp/plugin-cli-output
false
}
else
[ "$loglines" -eq "6" ] || {
echo "Expected 6 lines but the output was:"
[ "$loglines" -eq "2" ] || {
echo "Expected 2 lines but the output was:"
cat /tmp/plugin-cli-output
false
}
@ -408,17 +438,17 @@ fi
remove_jvm_example
local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" -Ees.logger.level=DEBUG > /tmp/plugin-cli-output
sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" -Des.logger.level=DEBUG > /tmp/plugin-cli-output
local loglines=$(cat /tmp/plugin-cli-output | wc -l)
if [ "$GROUP" == "TAR PLUGINS" ]; then
[ "$loglines" -gt "7" ] || {
echo "Expected more than 7 lines but the output was:"
[ "$loglines" -gt "3" ] || {
echo "Expected more than 3 lines but the output was:"
cat /tmp/plugin-cli-output
false
}
else
[ "$loglines" -gt "6" ] || {
echo "Expected more than 6 lines but the output was:"
[ "$loglines" -gt "2" ] || {
echo "Expected more than 2 lines but the output was:"
cat /tmp/plugin-cli-output
false
}

View File

@ -0,0 +1,44 @@
#!/bin/sh
# This file contains some utilities to test the elasticsearch scripts,
# the .deb/.rpm packages and the SysV/Systemd scripts.
# WARNING: This testing file must be executed as root and can
# dramatically change your system. It removes the 'elasticsearch'
# user/group and also many directories. Do not execute this file
# unless you know exactly what you are doing.
# Licensed to Elasticsearch under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
check_module() {
local name=$1
shift
assert_module_or_plugin_directory "$ESMODULES/$name"
for file in "$@"; do
assert_module_or_plugin_file "$ESMODULES/$name/$file"
done
assert_module_or_plugin_file "$ESMODULES/$name/$name-*.jar"
assert_module_or_plugin_file "$ESMODULES/$name/plugin-descriptor.properties"
}
check_secure_module() {
check_module "$@" plugin-security.policy
}

View File

@ -30,6 +30,7 @@
export_elasticsearch_paths() {
export ESHOME="/usr/share/elasticsearch"
export ESPLUGINS="$ESHOME/plugins"
export ESMODULES="$ESHOME/modules"
export ESCONFIG="/etc/elasticsearch"
export ESSCRIPTS="$ESCONFIG/scripts"
export ESDATA="/var/lib/elasticsearch"
@ -81,7 +82,8 @@ verify_package_installation() {
assert_file "$ESSCRIPTS" d root elasticsearch 750
assert_file "$ESDATA" d elasticsearch elasticsearch 755
assert_file "$ESLOG" d elasticsearch elasticsearch 755
assert_file "$ESPLUGINS" d elasticsearch elasticsearch 755
assert_file "$ESPLUGINS" d root root 755
assert_file "$ESMODULES" d root root 755
assert_file "$ESPIDDIR" d elasticsearch elasticsearch 755
assert_file "$ESHOME/NOTICE.txt" f root root 644
assert_file "$ESHOME/README.textile" f root root 644

View File

@ -179,6 +179,32 @@ assert_file() {
fi
}
assert_module_or_plugin_directory() {
local directory=$1
shift
#owner group and permissions vary depending on how es was installed
#just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
config_user=$(find "$ESHOME" -maxdepth 0 -printf "%u")
config_owner=$(find "$ESHOME" -maxdepth 0 -printf "%g")
# directories should use the user file-creation mask
config_privileges=$(executable_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
assert_file $directory d $config_user $config_owner $(printf "%o" $config_privileges)
}
assert_module_or_plugin_file() {
local file=$1
shift
assert_file_exist "$(readlink -m $file)"
# config files should not be executable and otherwise use the user
# file-creation mask
expected_file_privileges=$(file_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
assert_file $file f $config_user $config_owner $(printf "%o" $expected_file_privileges)
}
assert_output() {
echo "$output" | grep -E "$1"
}
@ -285,6 +311,9 @@ run_elasticsearch_service() {
if [ -f "/tmp/elasticsearch/bin/elasticsearch" ]; then
if [ -z "$CONF_DIR" ]; then
local CONF_DIR=""
local ES_PATH_CONF=""
else
local ES_PATH_CONF="-Ees.path.conf=$CONF_DIR"
fi
# we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
local background=""
@ -303,7 +332,7 @@ run_elasticsearch_service() {
# This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf
[ -f /usr/share/java/jayatanaag.jar ] && export JAVA_TOOL_OPTIONS="-javaagent:/usr/share/java/jayatanaag.jar"
# And now we can start Elasticsearch normally, in the background (-d) and with a pidfile (-p).
$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid -Ees.path.conf=$CONF_DIR $commandLineArgs
$timeoutCommand/tmp/elasticsearch/bin/elasticsearch $background -p /tmp/elasticsearch/elasticsearch.pid $ES_PATH_CONF $commandLineArgs
BASH
[ "$status" -eq "$expectedStatus" ]
elif is_systemd; then
@ -444,3 +473,19 @@ install_script() {
echo "Installing $script to $ESSCRIPTS"
cp $script $ESSCRIPTS
}
# permissions from the user umask with the executable bit set
executable_privileges_for_user_from_umask() {
local user=$1
shift
echo $((0777 & ~$(sudo -E -u $user sh -c umask) | 0111))
}
# permissions from the user umask without the executable bit set
file_privileges_for_user_from_umask() {
local user=$1
shift
echo $((0777 & ~$(sudo -E -u $user sh -c umask) & ~0111))
}

View File

@ -32,7 +32,7 @@ install_plugin() {
assert_file_exist "$path"
sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$path"
sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install -batch "file://$path"
assert_file_exist "$ESPLUGINS/$name"
assert_file_exist "$ESPLUGINS/$name/plugin-descriptor.properties"
@ -91,16 +91,13 @@ install_jvm_example() {
#just make sure that everything is the same as $CONFIG_DIR, which was properly set up during install
config_user=$(find "$ESCONFIG" -maxdepth 0 -printf "%u")
config_owner=$(find "$ESCONFIG" -maxdepth 0 -printf "%g")
config_privileges=$(find "$ESCONFIG" -maxdepth 0 -printf "%m")
assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner $config_privileges
#the original file has no execute permissions and that must not change, but all other permissions
#need to be inherited from the parent config dir. We check this by applying the 111 mask to the config dir privileges.
for i in `seq 0 2`; do
current_perm_dir=${config_privileges:$i:1}
final_perm=$(($current_perm_dir & ~1))
expected_file_privileges+=$final_perm
done
assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner $expected_file_privileges
# directories should user the user file-creation mask
config_privileges=$(executable_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
assert_file "$ESCONFIG/jvm-example" d $config_user $config_owner $(printf "%o" $config_privileges)
# config files should not be executable and otherwise use the user
# file-creation mask
expected_file_privileges=$(file_privileges_for_user_from_umask $ESPLUGIN_COMMAND_USER)
assert_file "$ESCONFIG/jvm-example/example.yaml" f $config_user $config_owner $(printf "%o" $expected_file_privileges)
echo "Running jvm-example's bin script...."
"$ESHOME/bin/jvm-example/test" | grep test
@ -136,11 +133,14 @@ install_and_check_plugin() {
fi
install_jvm_plugin $fullName "$(readlink -m $fullName-*.zip)"
assert_module_or_plugin_directory "$ESPLUGINS/$fullName"
if [ $prefix == 'analysis' ]; then
assert_file_exist "$(readlink -m $ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar)"
assert_module_or_plugin_file "$ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar"
fi
for file in "$@"; do
assert_file_exist "$(readlink -m $ESPLUGINS/$fullName/$file)"
assert_module_or_plugin_file "$ESPLUGINS/$fullName/$file"
done
}

View File

@ -68,6 +68,7 @@ move_elasticsearch() {
# Export some useful paths.
export_elasticsearch_paths() {
export ESMODULES="$ESHOME/modules"
export ESPLUGINS="$ESHOME/plugins"
export ESCONFIG="$ESHOME/config"
export ESSCRIPTS="$ESCONFIG/scripts"