diff --git a/src/main/java/org/elasticsearch/common/Names.java b/src/main/java/org/elasticsearch/common/Names.java index c641f22393f..3210245d9fa 100644 --- a/src/main/java/org/elasticsearch/common/Names.java +++ b/src/main/java/org/elasticsearch/common/Names.java @@ -20,6 +20,7 @@ package org.elasticsearch.common; import com.google.common.base.Charsets; +import org.elasticsearch.common.io.FileSystemUtils; import java.io.BufferedReader; import java.io.IOException; @@ -34,15 +35,15 @@ import java.util.concurrent.ThreadLocalRandom; */ public abstract class Names { - public static String randomNodeName(Path nodeNames) { + public static String randomNodeName(URL nodeNames) { try { int numberOfNames = 0; - try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(nodeNames), Charsets.UTF_8))) { + try (BufferedReader reader = FileSystemUtils.newBufferedReader(nodeNames, Charsets.UTF_8)) { while (reader.readLine() != null) { numberOfNames++; } } - try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(nodeNames), Charsets.UTF_8))) { + try (BufferedReader reader = FileSystemUtils.newBufferedReader(nodeNames, Charsets.UTF_8)) { int number = ((ThreadLocalRandom.current().nextInt(numberOfNames)) % numberOfNames); for (int i = 0; i < number; i++) { reader.readLine(); diff --git a/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index b901ce7d831..53d0f4b7a16 100644 --- a/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -24,7 +24,13 @@ import com.google.common.collect.Sets; import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.logging.ESLogger; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.List; @@ -131,6 +137,17 @@ public final class FileSystemUtils { return true; } + /** + * Opens the given url for reading returning a {@code BufferedReader} that may be + * used to read text from the URL in an efficient manner. Bytes from the + * file are decoded into characters using the specified charset. + */ + public static BufferedReader newBufferedReader(URL url, Charset cs) throws IOException { + CharsetDecoder decoder = cs.newDecoder(); + Reader reader = new InputStreamReader(url.openStream(), decoder); + return new BufferedReader(reader); + } + /** * This utility copy a full directory content (excluded) under * a new directory but without overwriting existing files. diff --git a/src/main/java/org/elasticsearch/env/Environment.java b/src/main/java/org/elasticsearch/env/Environment.java index 3f144d958a7..62f09d72d04 100644 --- a/src/main/java/org/elasticsearch/env/Environment.java +++ b/src/main/java/org/elasticsearch/env/Environment.java @@ -32,9 +32,8 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; +import java.util.Collections; import static org.elasticsearch.common.Strings.cleanPath; import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS; @@ -176,16 +175,16 @@ public class Environment { return logsFile; } - public String resolveConfigAndLoadToString(String path) throws FailedToResolveConfigException, IOException { - return Streams.copyToString(Files.newBufferedReader(resolveConfig(path), Charsets.UTF_8)); - } - - public Path resolveConfig(String path) throws FailedToResolveConfigException { + public URL resolveConfig(String path) throws FailedToResolveConfigException { String origPath = path; // first, try it as a path on the file system Path f1 = Paths.get(path); if (Files.exists(f1)) { - return f1; + try { + return f1.toUri().toURL(); + } catch (MalformedURLException e) { + throw new FailedToResolveConfigException("Failed to resolve path [" + f1 + "]", e); + } } if (path.startsWith("/")) { path = path.substring(1); @@ -193,23 +192,23 @@ public class Environment { // next, try it relative to the config location Path f2 = configFile.resolve(path); if (Files.exists(f2)) { - return f2; + try { + return f2.toUri().toURL(); + } catch (MalformedURLException e) { + throw new FailedToResolveConfigException("Failed to resolve path [" + f1 + "]", e); + } } - try { - // try and load it from the classpath directly - URL resource = settings.getClassLoader().getResource(path); + // try and load it from the classpath directly + URL resource = settings.getClassLoader().getResource(path); + if (resource != null) { + return resource; + } + // try and load it from the classpath with config/ prefix + if (!path.startsWith("config/")) { + resource = settings.getClassLoader().getResource("config/" + path); if (resource != null) { - return Paths.get(resource.toURI()); + return resource; } - // try and load it from the classpath with config/ prefix - if (!path.startsWith("config/")) { - resource = settings.getClassLoader().getResource("config/" + path); - if (resource != null) { - return Paths.get(resource.toURI()); - } - } - } catch (URISyntaxException ex) { - throw new FailedToResolveConfigException("Failed to resolve config path [" + origPath + "], tried file path [" + f1 + "], path file [" + f2 + "], and classpath", ex); } throw new FailedToResolveConfigException("Failed to resolve config path [" + origPath + "], tried file path [" + f1 + "], path file [" + f2 + "], and classpath"); } diff --git a/src/main/java/org/elasticsearch/index/analysis/Analysis.java b/src/main/java/org/elasticsearch/index/analysis/Analysis.java index d1d51fc5142..7285840f83c 100644 --- a/src/main/java/org/elasticsearch/index/analysis/Analysis.java +++ b/src/main/java/org/elasticsearch/index/analysis/Analysis.java @@ -62,6 +62,7 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -73,6 +74,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -229,10 +231,10 @@ public class Analysis { } } - Path wordListFile = env.resolveConfig(wordListPath); + final URL wordListFile = env.resolveConfig(wordListPath); - try { - return loadWordList(Files.newBufferedReader(wordListFile, Charsets.UTF_8), "#"); + try (BufferedReader reader = FileSystemUtils.newBufferedReader(wordListFile, Charsets.UTF_8)) { + return loadWordList(reader, "#"); } catch (IOException ioe) { String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage()); throw new ElasticsearchIllegalArgumentException(message); @@ -276,17 +278,14 @@ public class Analysis { return null; } - Path fileUrl = env.resolveConfig(filePath); + final URL fileUrl = env.resolveConfig(filePath); - Reader reader = null; try { - reader = new InputStreamReader(Files.newInputStream(fileUrl), Charsets.UTF_8); + return FileSystemUtils.newBufferedReader(fileUrl, Charsets.UTF_8); } catch (IOException ioe) { String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage()); throw new ElasticsearchIllegalArgumentException(message); } - - return reader; } /** diff --git a/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java index fc00ac62c0a..acea0c2abc8 100644 --- a/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java +++ b/src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java @@ -58,10 +58,10 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW throw new ElasticsearchIllegalArgumentException("hyphenation_patterns_path is a required setting."); } - Path hyphenationPatternsFile = env.resolveConfig(hyphenationPatternsPath); + URL hyphenationPatternsFile = env.resolveConfig(hyphenationPatternsPath); try { - hyphenationTree = HyphenationCompoundWordTokenFilter.getHyphenationTree(new InputSource(Files.newInputStream(hyphenationPatternsFile))); + hyphenationTree = HyphenationCompoundWordTokenFilter.getHyphenationTree(new InputSource(hyphenationPatternsFile.toExternalForm())); } catch (Exception e) { throw new ElasticsearchIllegalArgumentException("Exception while reading hyphenation_patterns_path: " + e.getMessage()); } diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 7a3b8f3ea90..0d719975e83 100755 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedString; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.lucene.search.AndFilter; import org.elasticsearch.common.lucene.search.NotFilter; @@ -138,11 +139,11 @@ public class MapperService extends AbstractIndexComponent { this.dynamic = componentSettings.getAsBoolean("dynamic", true); String defaultMappingLocation = componentSettings.get("default_mapping_location"); - final Path defaultMappingUrl; + final URL defaultMappingUrl; if (index.getName().equals(ScriptService.SCRIPT_INDEX)){ - defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation,"script-mapping.json","org/elasticsearch/index/mapper/script-mapping.json"); + defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation, "script-mapping.json", "org/elasticsearch/index/mapper/script-mapping.json"); } else { - defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation,"default-mapping.json","org/elasticsearch/index/mapper/default-mapping.json"); + defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation, "default-mapping.json", "org/elasticsearch/index/mapper/default-mapping.json"); } if (defaultMappingUrl == null) { @@ -164,25 +165,29 @@ public class MapperService extends AbstractIndexComponent { } } else { try { - defaultMappingSource = Streams.copyToString(new InputStreamReader(Files.newInputStream(defaultMappingUrl), Charsets.UTF_8)); + defaultMappingSource = Streams.copyToString(FileSystemUtils.newBufferedReader(defaultMappingUrl, Charsets.UTF_8)); } catch (IOException e) { throw new MapperException("Failed to load default mapping source from [" + defaultMappingLocation + "]", e); } } String percolatorMappingLocation = componentSettings.get("default_percolator_mapping_location"); - Path percolatorMappingUrl = null; + URL percolatorMappingUrl = null; if (percolatorMappingLocation != null) { try { percolatorMappingUrl = environment.resolveConfig(percolatorMappingLocation); } catch (FailedToResolveConfigException e) { // not there, default to the built in one - percolatorMappingUrl = Paths.get(percolatorMappingLocation); + try { + percolatorMappingUrl = Paths.get(percolatorMappingLocation).toUri().toURL(); + } catch (MalformedURLException e1) { + throw new FailedToResolveConfigException("Failed to resolve default percolator mapping location [" + percolatorMappingLocation + "]"); + } } } if (percolatorMappingUrl != null) { try { - defaultPercolatorMappingSource = Streams.copyToString(new InputStreamReader(Files.newInputStream(percolatorMappingUrl), Charsets.UTF_8)); + defaultPercolatorMappingSource = Streams.copyToString(FileSystemUtils.newBufferedReader(percolatorMappingUrl, Charsets.UTF_8)); } catch (IOException e) { throw new MapperException("Failed to load default percolator mapping source from [" + percolatorMappingUrl + "]", e); } @@ -208,20 +213,16 @@ public class MapperService extends AbstractIndexComponent { } } - private Path getMappingUrl(Settings indexSettings, Environment environment, String mappingLocation, String configString, String resourceLocation) { - Path mappingUrl; + private URL getMappingUrl(Settings indexSettings, Environment environment, String mappingLocation, String configString, String resourceLocation) { + URL mappingUrl; if (mappingLocation == null) { try { mappingUrl = environment.resolveConfig(configString); } catch (FailedToResolveConfigException e) { - try { - // not there, default to the built in one - mappingUrl = Paths.get(indexSettings.getClassLoader().getResource(resourceLocation).toURI()); - if (mappingUrl == null) { - mappingUrl = Paths.get(MapperService.class.getClassLoader().getResource(resourceLocation).toURI()); - } - } catch (URISyntaxException e1) { - throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + mappingLocation + "]"); + // not there, default to the built in one + mappingUrl = indexSettings.getClassLoader().getResource(resourceLocation); + if (mappingUrl == null) { + mappingUrl = MapperService.class.getClassLoader().getResource(resourceLocation); } } } else { @@ -229,8 +230,11 @@ public class MapperService extends AbstractIndexComponent { mappingUrl = environment.resolveConfig(mappingLocation); } catch (FailedToResolveConfigException e) { // not there, default to the built in one - mappingUrl = Paths.get(mappingLocation); - + try { + mappingUrl = Paths.get(mappingLocation).toUri().toURL(); + } catch (MalformedURLException e1) { + throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + mappingLocation + "]"); + } } } return mappingUrl; diff --git a/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java index 00b09a89d27..fcb6f3919c6 100644 --- a/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java +++ b/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java @@ -60,33 +60,33 @@ public class InternalSettingsPreparer { // if its default, then load it, but also load form env if (Strings.hasText(System.getProperty("es.default.config"))) { loadFromEnv = true; - settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("es.default.config"))); + settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("es.default.config"))); } // if explicit, just load it and don't load from env if (Strings.hasText(System.getProperty("es.config"))) { loadFromEnv = false; - settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("es.config"))); + settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("es.config"))); } if (Strings.hasText(System.getProperty("elasticsearch.config"))) { loadFromEnv = false; - settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("elasticsearch.config"))); + settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("elasticsearch.config"))); } } if (loadFromEnv) { try { - settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.yml")); + settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.yml")); } catch (FailedToResolveConfigException e) { // ignore } catch (NoClassDefFoundError e) { // ignore, no yaml } try { - settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.json")); + settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.json")); } catch (FailedToResolveConfigException e) { // ignore } try { - settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.properties")); + settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.properties")); } catch (FailedToResolveConfigException e) { // ignore } diff --git a/src/main/java/org/elasticsearch/plugins/PluginManager.java b/src/main/java/org/elasticsearch/plugins/PluginManager.java index d8ef6725045..4b3037afa4f 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -50,8 +50,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.util.*; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import static org.elasticsearch.common.Strings.hasLength; import static org.elasticsearch.common.io.FileSystemUtils.moveFilesWithoutOverwriting; diff --git a/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/src/test/java/org/elasticsearch/env/EnvironmentTests.java new file mode 100644 index 00000000000..897d8e72905 --- /dev/null +++ b/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.env; + +import com.google.common.base.Charsets; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.ElasticsearchIllegalStateException; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Simple unit-tests for Environment.java + */ +public class EnvironmentTests extends ElasticsearchTestCase { + + public Environment newEnvironment() throws IOException { + return newEnvironment(ImmutableSettings.EMPTY); + } + + public Environment newEnvironment(Settings settings) throws IOException { + Settings build = ImmutableSettings.builder() + .put(settings) + .put("path.home", newTempDir().getAbsolutePath()) + .putArray("path.data", tmpPaths()).build(); + return new Environment(build); + } + + @Test + public void testResolveJaredResource() throws IOException { + Environment environment = newEnvironment(); + URL url = environment.resolveConfig("META-INF/MANIFEST.MF"); // this works because there is one jar having this file in the classpath + assertNotNull(url); + try (BufferedReader reader = FileSystemUtils.newBufferedReader(url, Charsets.UTF_8)) { + String string = Streams.copyToString(reader); + assertTrue(string, string.contains("Manifest-Version")); + } + } + + @Test + public void testResolveFileResource() throws IOException { + Environment environment = newEnvironment(); + URL url = environment.resolveConfig("org/elasticsearch/common/cli/tool.help"); + assertNotNull(url); + try (BufferedReader reader = FileSystemUtils.newBufferedReader(url, Charsets.UTF_8)) { + String string = Streams.copyToString(reader); + assertEquals(string, "tool help"); + } + } +} diff --git a/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 7e100cbc74f..9884e203499 100644 --- a/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -284,15 +284,6 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase { env.close(); } - private String[] tmpPaths() { - final int numPaths = randomIntBetween(1, 3); - final String[] absPaths = new String[numPaths]; - for (int i = 0; i < numPaths; i++) { - absPaths[i] = newTempDir().getAbsolutePath(); - } - return absPaths; - } - public NodeEnvironment newNodeEnvironment() throws IOException { return newNodeEnvironment(ImmutableSettings.EMPTY); } @@ -304,4 +295,6 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase { .putArray("path.data", tmpPaths()).build(); return new NodeEnvironment(build, new Environment(build)); } + + } diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java index 4db83cf3e9a..bf103bd0673 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java @@ -560,4 +560,16 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest { return ThreadPool.terminate(service, 10, TimeUnit.SECONDS); } + /** + * Returns a random number of temporary paths. + */ + public String[] tmpPaths() { + final int numPaths = randomIntBetween(1, 3); + final String[] absPaths = new String[numPaths]; + for (int i = 0; i < numPaths; i++) { + absPaths[i] = newTempDir().getAbsolutePath(); + } + return absPaths; + } + }