Enables plugins to define default logging configuration for their needs.

- adds support for multiple logging configurations under the config dir (will pick up any logging.xxx in the config folder tree)
- plugins can now define a top level config directory that will be copied under es config dir and will be renamed after the plugin name (same as the support we have the plugin "bin" dirs)

 Closes #6802

src/main/java/org/elasticsearch/common/logging/log4j/Log4jESLogger.java

@@ -35,6 +35,10 @@ public class Log4jESLogger extends AbstractESLogger {
         this.logger = logger;
     }
 
+    public Logger logger() {
+        return logger;
+    }
+
     public void setLevel(String level) {
         if (level == null) {
             logger.setLevel(null);

src/main/java/org/elasticsearch/common/logging/log4j/LogConfigurator.java

@@ -21,12 +21,18 @@ package org.elasticsearch.common.logging.log4j;
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.log4j.PropertyConfigurator;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.FailedToResolveConfigException;
 
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.EnumSet;
 import java.util.Map;
 import java.util.Properties;
 
@@ -72,23 +78,7 @@ public class LogConfigurator {
         loaded = true;
         Environment environment = new Environment(settings);
         ImmutableSettings.Builder settingsBuilder = settingsBuilder().put(settings);
-        try {
-            settingsBuilder.loadFromUrl(environment.resolveConfig("logging.yml"));
-        } catch (FailedToResolveConfigException e) {
-            // ignore
-        } catch (NoClassDefFoundError e) {
-            // ignore, no yaml
-        }
-        try {
-            settingsBuilder.loadFromUrl(environment.resolveConfig("logging.json"));
-        } catch (FailedToResolveConfigException e) {
-            // ignore
-        }
-        try {
-            settingsBuilder.loadFromUrl(environment.resolveConfig("logging.properties"));
-        } catch (FailedToResolveConfigException e) {
-            // ignore
-        }
+        resolveConfig(environment, settingsBuilder);
         settingsBuilder
                 .putProperties("elasticsearch.", System.getProperties())
                 .putProperties("es.", System.getProperties())
@@ -110,4 +100,29 @@ public class LogConfigurator {
         }
         PropertyConfigurator.configure(props);
     }
+
+    public static void resolveConfig(Environment env, final ImmutableSettings.Builder settingsBuilder) {
+
+        try {
+            Files.walkFileTree(env.configFile().toPath(), EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    if (file.getFileName().toString().startsWith("logging.")) {
+                        loadConfig(file, settingsBuilder);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException ioe) {
+            throw new ElasticsearchException("Failed to load logging configuration", ioe);
+        }
+    }
+
+    public static void loadConfig(Path file, ImmutableSettings.Builder settingsBuilder) {
+        try {
+            settingsBuilder.loadFromUrl(file.toUri().toURL());
+        } catch (FailedToResolveConfigException | NoClassDefFoundError | MalformedURLException e) {
+            // ignore
+        }
+    }
 }

src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -215,6 +215,15 @@ public class PluginManager {
             debug("Installed " + name + " into " + toLocation.getAbsolutePath());
         }
 
+        File configFile = new File(extractLocation, "config");
+        if (configFile.exists() && configFile.isDirectory()) {
+            File toLocation = pluginHandle.configDir(environment);
+            debug("Found config, moving to " + toLocation.getAbsolutePath());
+            FileSystemUtils.deleteRecursively(toLocation);
+            configFile.renameTo(toLocation);
+            debug("Installed " + name + " into " + toLocation.getAbsolutePath());
+        }
+
         // try and identify the plugin type, see if it has no .class or .jar files in it
         // so its probably a _site, and it it does not have a _site in it, move everything to _site
         if (!new File(extractLocation, "_site").exists()) {
@@ -272,6 +281,15 @@ public class PluginManager {
             }
             removed = true;
         }
+        File configLocation = pluginHandle.configDir(environment);
+        if (configLocation.exists()) {
+            debug("Removing: " + configLocation.getPath());
+            if (!FileSystemUtils.deleteRecursively(configLocation)) {
+                throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " +
+                        configLocation.toString());
+            }
+            removed = true;
+        }
         if (removed) {
             log("Removed " + name);
         } else {
@@ -585,6 +603,10 @@ public class PluginManager {
             return new File(new File(env.homeFile(), "bin"), name);
         }
 
+        File configDir(Environment env) {
+            return new File(new File(env.homeFile(), "config"), name);
+        }
+
         static PluginHandle parse(String name) {
             String[] elements = name.split("/");
             // We first consider the simplest form: pluginname

src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.logging;
+
+import org.apache.log4j.Appender;
+import org.apache.log4j.Logger;
+import org.elasticsearch.common.logging.log4j.Log4jESLogger;
+import org.elasticsearch.common.logging.log4j.Log4jESLoggerFactory;
+import org.elasticsearch.common.logging.log4j.LogConfigurator;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URL;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ *
+ */
+public class LoggingConfigurationTests {
+
+    @Test
+    public void testMultipleConfigs() throws Exception {
+        File configDir = resolveConfigDir();
+        Settings settings = ImmutableSettings.builder()
+                .put("path.conf", configDir.getAbsolutePath())
+                .build();
+        LogConfigurator.configure(settings);
+
+        ESLogger esLogger = Log4jESLoggerFactory.getLogger("first");
+        Logger logger = ((Log4jESLogger) esLogger).logger();
+        Appender appender = logger.getAppender("console1");
+        assertThat(appender, notNullValue());
+
+        esLogger = Log4jESLoggerFactory.getLogger("second");
+        logger = ((Log4jESLogger) esLogger).logger();
+        appender = logger.getAppender("console2");
+        assertThat(appender, notNullValue());
+
+        esLogger = Log4jESLoggerFactory.getLogger("third");
+        logger = ((Log4jESLogger) esLogger).logger();
+        appender = logger.getAppender("console3");
+        assertThat(appender, notNullValue());
+    }
+
+    private static File resolveConfigDir() throws Exception {
+        URL url = LoggingConfigurationTests.class.getResource("config");
+        return new File(url.toURI());
+    }
+}

src/test/java/org/elasticsearch/plugin/PluginManagerTests.java

@@ -45,6 +45,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.net.URI;
+import java.nio.file.Files;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope;
@@ -88,6 +89,33 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
         assertPluginAvailable(pluginName);
     }
 
+    @Test
+    public void testLocalPluginInstallWithBinAndConfig() throws Exception {
+        String pluginName = "plugin-test";
+        Tuple<Settings, Environment> initialSettings = InternalSettingsPreparer.prepareSettings(
+                ImmutableSettings.settingsBuilder().build(), false);
+        Environment env = initialSettings.v2();
+        File pluginBinDir = new File(new File(env.homeFile(), "bin"), pluginName);
+        File pluginConfigDir = new File(env.configFile(), pluginName);
+        try {
+
+            PluginManager pluginManager = pluginManager(getPluginUrlForResource("plugin_with_bin_and_config.zip"), initialSettings);
+
+            pluginManager.downloadAndExtract(pluginName);
+
+            File[] plugins = pluginManager.getListInstalledPlugins();
+
+            assertThat(plugins.length, is(1));
+            assertTrue(pluginBinDir.exists());
+            assertTrue(pluginConfigDir.exists());
+
+        } finally {
+            // we need to clean up the copied dirs
+            FileSystemUtils.deleteRecursively(pluginBinDir);
+            FileSystemUtils.deleteRecursively(pluginConfigDir);
+        }
+    }
+
     @Test
     public void testLocalPluginInstallSiteFolder() throws Exception {
         //When we have only a folder in top-level (no files either) but it's called _site, we make it work
@@ -131,13 +159,17 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
         downloadAndExtract(pluginName, getPluginUrlForResource("plugin_with_sourcefiles.zip"));
     }
 
+    private static PluginManager pluginManager(String pluginUrl) {
+        Tuple<Settings, Environment> initialSettings = InternalSettingsPreparer.prepareSettings(
+                ImmutableSettings.settingsBuilder().build(), false);
+        return pluginManager(pluginUrl, initialSettings);
+    }
+
     /**
      * We build a plugin manager instance which wait only for 30 seconds before
      * raising an ElasticsearchTimeoutException
      */
-    private static PluginManager pluginManager(String pluginUrl) {
-        Tuple<Settings, Environment> initialSettings = InternalSettingsPreparer.prepareSettings(
-                ImmutableSettings.settingsBuilder().build(), false);
+    private static PluginManager pluginManager(String pluginUrl, Tuple<Settings, Environment> initialSettings) {
         if (!initialSettings.v2().pluginsFile().exists()) {
             FileSystemUtils.mkdirs(initialSettings.v2().pluginsFile());
         }

src/test/resources/org/elasticsearch/common/logging/config/logging.yml

@@ -0,0 +1,12 @@
+# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
+es.logger.level: INFO
+rootLogger: ${es.logger.level}, console1
+logger:
+  first: DEBUG, console1
+
+appender:
+  console1:
+    type: console
+    layout:
+      type: consolePattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

src/test/resources/org/elasticsearch/common/logging/config/test2/logging.yml

@@ -0,0 +1,10 @@
+logger:
+  # log action execution errors for easier debugging
+  second: DEBUG, console2
+
+appender:
+  console2:
+    type: console
+    layout:
+      type: consolePattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

src/test/resources/org/elasticsearch/common/logging/config/test2/test3/logging.yml

@@ -0,0 +1,10 @@
+logger:
+  # log action execution errors for easier debugging
+  third: DEBUG, console3
+
+appender:
+  console3:
+    type: console
+    layout:
+      type: consolePattern
+      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"