Merge pull request #10717 from rmuir/put_me_in_coach

enable securitymanager

config/elasticsearch.yml

@@ -231,7 +231,6 @@
 #
 #http.enabled: false
 
-
 ################################### Gateway ###################################
 
 # The gateway allows for persisting the cluster state between full cluster

pom.xml

@@ -630,7 +630,7 @@
                                 <tests.compatibility>${tests.compatibility}</tests.compatibility>
                                 <java.awt.headless>true</java.awt.headless>
                                 <!-- security manager / test.policy -->
-                                <java.security.policy>${basedir}/dev-tools/tests.policy</java.security.policy>
+                                <java.security.policy>${basedir}/src/main/resources/org/elasticsearch/bootstrap/security.policy</java.security.policy>
                             </systemProperties>
                         </configuration>
                     </execution>

src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -26,7 +26,6 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.inject.spi.Message;
-import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.jna.Natives;
 import org.elasticsearch.common.logging.ESLogger;
@@ -40,7 +39,6 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeBuilder;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
-import java.nio.file.Paths;
 import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
@@ -61,6 +59,7 @@ public class Bootstrap {
     private static Bootstrap bootstrap;
 
     private void setup(boolean addShutdownHook, Tuple<Settings, Environment> tuple) throws Exception {
+        setupSecurity(tuple.v1(), tuple.v2());
         if (tuple.v1().getAsBoolean("bootstrap.mlockall", false)) {
             Natives.tryMlockall();
         }
@@ -92,6 +91,18 @@ public class Bootstrap {
             });
         }
     }
+    
+    /** 
+     * option for elasticsearch.yml etc to turn off our security manager completely,
+     * for example if you want to have your own configuration or just disable.
+     */
+    static final String SECURITY_SETTING = "security.manager.enabled";
+
+    private void setupSecurity(Settings settings, Environment environment) throws Exception {
+        if (settings.getAsBoolean(SECURITY_SETTING, true)) {
+            Security.configure(environment);
+        }
+    }
 
     @SuppressForbidden(reason = "Exception#printStackTrace()")
     private static void setupLogging(Tuple<Settings, Environment> tuple) {

src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -0,0 +1,143 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import com.google.common.io.ByteStreams;
+
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.env.Environment;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+
+/** 
+ * Initializes securitymanager with necessary permissions.
+ * <p>
+ * We use a template file (the one we test with), and add additional 
+ * permissions based on the environment (data paths, etc)
+ */
+class Security {
+    
+    /** template policy file, the one used in tests */
+    static final String POLICY_RESOURCE = "security.policy";
+    
+    /** 
+     * Initializes securitymanager for the environment
+     * Can only happen once!
+     */
+    static void configure(Environment environment) throws IOException {
+        // init lucene random seed. it will use /dev/urandom where available.
+        StringHelper.randomId();
+        InputStream config = Security.class.getResourceAsStream(POLICY_RESOURCE);
+        if (config == null) {
+            throw new NoSuchFileException(POLICY_RESOURCE);
+        }
+        Path newConfig = processTemplate(config, environment);
+        System.setProperty("java.security.policy", newConfig.toString());
+        System.setSecurityManager(new SecurityManager());
+        IOUtils.deleteFilesIgnoringExceptions(newConfig); // TODO: maybe log something if it fails?
+    }
+   
+    // package-private for testing
+    static Path processTemplate(InputStream template, Environment environment) throws IOException {
+        Path processed = Files.createTempFile(null, null);
+        try (OutputStream output = new BufferedOutputStream(Files.newOutputStream(processed))) {
+            // copy the template as-is.
+            try (InputStream in = new BufferedInputStream(template)) {
+                ByteStreams.copy(in, output);
+            }
+            
+            // add permissions for all configured paths.
+            Set<Path> paths = new HashSet<>();
+            paths.add(environment.homeFile());
+            paths.add(environment.configFile());
+            paths.add(environment.logsFile());
+            paths.add(environment.pluginsFile());
+            paths.add(environment.workFile());
+            paths.add(environment.workWithClusterFile());
+            for (Path path : environment.dataFiles()) {
+                paths.add(path);
+            }
+            for (Path path : environment.dataWithClusterFiles()) {
+                paths.add(path);
+            }
+            output.write(createPermissions(paths));
+        }
+        return processed;
+    }
+    
+    // package private for testing
+    static byte[] createPermissions(Set<Path> paths) throws IOException {
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        
+        // all policy files are UTF-8:
+        //  https://docs.oracle.com/javase/7/docs/technotes/guides/security/PolicyFiles.html
+        try (Writer writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) {
+            writer.write(System.lineSeparator());
+            writer.write("grant {");
+            writer.write(System.lineSeparator());
+            for (Path path : paths) {
+                // data paths actually may not exist yet.
+                Files.createDirectories(path);
+                // add each path twice: once for itself, again for files underneath it
+                addPath(writer, encode(path), "read,readlink,write,delete");
+                addRecursivePath(writer, encode(path), "read,readlink,write,delete");
+            }
+            writer.write("};");
+            writer.write(System.lineSeparator());
+        }
+        
+        return stream.toByteArray();
+    }
+    
+    static void addPath(Writer writer, String path, String permissions) throws IOException {
+        writer.write("permission java.io.FilePermission \"" + path + "\", \"" + permissions + "\";");
+        writer.write(System.lineSeparator());
+    }
+    
+    static void addRecursivePath(Writer writer, String path, String permissions) throws IOException {
+        writer.write("permission java.io.FilePermission \"" + path + "${/}-\", \"" + permissions + "\";");
+        writer.write(System.lineSeparator());
+    }
+    
+    // Any backslashes in paths must be escaped, because it is the escape character when parsing.
+    // See "Note Regarding File Path Specifications on Windows Systems".
+    // https://docs.oracle.com/javase/7/docs/technotes/guides/security/PolicyFiles.html
+    static String encode(Path path) {
+        return encode(path.toString());
+    }
+    
+    static String encode(String path) {
+        return path.replace("\\", "\\\\");
+    }
+}

src/main/java/org/elasticsearch/env/Environment.java

@@ -58,7 +58,20 @@ public class Environment {
     private final Path logsFile;
 
     /** List of filestores on the system */
-    private final FileStore[] fileStores;
+    private static final FileStore[] fileStores;
+
+    /**
+     * We have to do this in clinit instead of init, because ES code is pretty messy,
+     * and makes these environments, throws them away, makes them again, etc.
+     */
+    static {
+        // gather information about filesystems
+        ArrayList<FileStore> allStores = new ArrayList<>();
+        for (FileStore store : PathUtils.getDefaultFileSystem().getFileStores()) {
+            allStores.add(new ESFileStore(store));
+        }
+        fileStores = allStores.toArray(new ESFileStore[allStores.size()]);
+    }
 
     public Environment() {
         this(EMPTY_SETTINGS);
@@ -109,13 +122,6 @@ public class Environment {
         } else {
             logsFile = homeFile.resolve("logs");
         }
-
-        // gather information about filesystems
-        ArrayList<FileStore> allStores = new ArrayList<>();
-        for (FileStore store : PathUtils.getDefaultFileSystem().getFileStores()) {
-            allStores.add(new ESFileStore(store));
-        }
-        fileStores = allStores.toArray(new ESFileStore[allStores.size()]);
     }
 
     /**

src/main/resources/org/elasticsearch/bootstrap/security.policy

@@ -17,21 +17,26 @@
  * under the License.
  */
 
-// Policy file to prevent tests from writing outside the test sandbox directory
+// Default security policy file.
-// PLEASE NOTE: You may need to enable other permissions when new tests are added,
+// On startup, BootStrap reads environment and adds additional permissions
-// everything not allowed here is forbidden!
+// for configured paths to these.
 
 grant {
 
-  // contain read access to only what we need:
+  // system jar resources
+  permission java.io.FilePermission "${java.home}${/}-", "read";
+
+  // temporary files
+  permission java.io.FilePermission "${java.io.tmpdir}", "read,write";
+  permission java.io.FilePermission "${java.io.tmpdir}${/}-", "read,write,delete";
+
+  // paths used for running tests
   // project base directory
   permission java.io.FilePermission "${project.basedir}${/}target${/}-", "read";
   // read permission for lib sigar
   permission java.io.FilePermission "${project.basedir}${/}lib/sigar{/}-", "read";
   // mvn custom ./m2/repository for dependency jars
   permission java.io.FilePermission "${m2.repository}${/}-", "read";
-  // system jar resources
-  permission java.io.FilePermission "${java.home}${/}-", "read";
   // per-jvm directory
   permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,write";
   permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,write,delete";

src/test/java/org/elasticsearch/bootstrap/SecurityTests.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.test.ElasticsearchTestCase;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Collections;
+
+public class SecurityTests extends ElasticsearchTestCase {
+    
+    /** backslash escaping (e.g. windows paths) */
+    public void testEncode() {
+        assertEquals("c:\\\\foobar", Security.encode("c:\\foobar"));
+    }
+    
+    /** test template processing */
+    public void testTemplateProcessing() throws Exception {
+        Path path = createTempDir();
+        
+        byte results[] = Security.createPermissions(Collections.singleton(path));
+        String unicode = new String(results, StandardCharsets.UTF_8);
+        // try not to make this test too fragile or useless
+        assertTrue(unicode.contains("grant {"));
+        assertTrue(unicode.contains(Security.encode(path)));
+        assertTrue(unicode.contains("read"));
+        assertTrue(unicode.contains("write"));
+    }
+    
+}