honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert back APIs that resolve files from classpath to java.net.URL 

The conversion to the Path API doesn't work if the path points
to a file inside a JAR like a config. These path must be read
while the ZIP filesystem is opened which can't be guaranteed across
the board. This commit reverts back the relevant changes to java.net.URL
and adds a util method to read UTF-8 Encoded files from URLs correctly.
This commit is contained in:
Simon Willnauer 2014-12-02 23:09:07 +01:00
parent a6510f9245
commit 3dfff84043
11 changed files with 179 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/elasticsearch/common/Names.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.common;
import com.google.common.base.Charsets;
import org.elasticsearch.common.io.FileSystemUtils;
import java.io.BufferedReader;
import java.io.IOException;
@ -34,15 +35,15 @@ import java.util.concurrent.ThreadLocalRandom;
*/
public abstract class Names {
public static String randomNodeName(Path nodeNames) {
public static String randomNodeName(URL nodeNames) {
try {
int numberOfNames = 0;
try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(nodeNames), Charsets.UTF_8))) {
try (BufferedReader reader = FileSystemUtils.newBufferedReader(nodeNames, Charsets.UTF_8)) {
while (reader.readLine() != null) {
numberOfNames++;
}
}
try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(nodeNames), Charsets.UTF_8))) {
try (BufferedReader reader = FileSystemUtils.newBufferedReader(nodeNames, Charsets.UTF_8)) {
int number = ((ThreadLocalRandom.current().nextInt(numberOfNames)) % numberOfNames);
for (int i = 0; i < number; i++) {
reader.readLine();

17
src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
View File

@ -24,7 +24,13 @@ import com.google.common.collect.Sets;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.logging.ESLogger;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
@ -131,6 +137,17 @@ public final class FileSystemUtils {
return true;
}
/**
* Opens the given url for reading returning a {@code BufferedReader} that may be
* used to read text from the URL in an efficient manner. Bytes from the
* file are decoded into characters using the specified charset.
*/
public static BufferedReader newBufferedReader(URL url, Charset cs) throws IOException {
CharsetDecoder decoder = cs.newDecoder();
Reader reader = new InputStreamReader(url.openStream(), decoder);
return new BufferedReader(reader);
}
/**
* This utility copy a full directory content (excluded) under
* a new directory but without overwriting existing files.

45
src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -32,9 +32,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.*;
import java.util.Collections;
import static org.elasticsearch.common.Strings.cleanPath;
import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
@ -176,16 +175,16 @@ public class Environment {
return logsFile;
}
public String resolveConfigAndLoadToString(String path) throws FailedToResolveConfigException, IOException {
return Streams.copyToString(Files.newBufferedReader(resolveConfig(path), Charsets.UTF_8));
}
public Path resolveConfig(String path) throws FailedToResolveConfigException {
public URL resolveConfig(String path) throws FailedToResolveConfigException {
String origPath = path;
// first, try it as a path on the file system
Path f1 = Paths.get(path);
if (Files.exists(f1)) {
return f1;
try {
return f1.toUri().toURL();
} catch (MalformedURLException e) {
throw new FailedToResolveConfigException("Failed to resolve path [" + f1 + "]", e);
}
}
if (path.startsWith("/")) {
path = path.substring(1);
@ -193,23 +192,23 @@ public class Environment {
// next, try it relative to the config location
Path f2 = configFile.resolve(path);
if (Files.exists(f2)) {
return f2;
try {
return f2.toUri().toURL();
} catch (MalformedURLException e) {
throw new FailedToResolveConfigException("Failed to resolve path [" + f1 + "]", e);
}
}
try {
// try and load it from the classpath directly
URL resource = settings.getClassLoader().getResource(path);
// try and load it from the classpath directly
URL resource = settings.getClassLoader().getResource(path);
if (resource != null) {
return resource;
}
// try and load it from the classpath with config/ prefix
if (!path.startsWith("config/")) {
resource = settings.getClassLoader().getResource("config/" + path);
if (resource != null) {
return Paths.get(resource.toURI());
return resource;
}
// try and load it from the classpath with config/ prefix
if (!path.startsWith("config/")) {
resource = settings.getClassLoader().getResource("config/" + path);
if (resource != null) {
return Paths.get(resource.toURI());
}
}
} catch (URISyntaxException ex) {
throw new FailedToResolveConfigException("Failed to resolve config path [" + origPath + "], tried file path [" + f1 + "], path file [" + f2 + "], and classpath", ex);
}
throw new FailedToResolveConfigException("Failed to resolve config path [" + origPath + "], tried file path [" + f1 + "], path file [" + f2 + "], and classpath");
}

15
src/main/java/org/elasticsearch/index/analysis/Analysis.java
View File

@ -62,6 +62,7 @@ import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
@ -73,6 +74,7 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
@ -229,10 +231,10 @@ public class Analysis {
}
}
Path wordListFile = env.resolveConfig(wordListPath);
final URL wordListFile = env.resolveConfig(wordListPath);
try {
return loadWordList(Files.newBufferedReader(wordListFile, Charsets.UTF_8), "#");
try (BufferedReader reader = FileSystemUtils.newBufferedReader(wordListFile, Charsets.UTF_8)) {
return loadWordList(reader, "#");
} catch (IOException ioe) {
String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage());
throw new ElasticsearchIllegalArgumentException(message);
@ -276,17 +278,14 @@ public class Analysis {
return null;
}
Path fileUrl = env.resolveConfig(filePath);
final URL fileUrl = env.resolveConfig(filePath);
Reader reader = null;
try {
reader = new InputStreamReader(Files.newInputStream(fileUrl), Charsets.UTF_8);
return FileSystemUtils.newBufferedReader(fileUrl, Charsets.UTF_8);
} catch (IOException ioe) {
String message = String.format(Locale.ROOT, "IOException while reading %s_path: %s", settingPrefix, ioe.getMessage());
throw new ElasticsearchIllegalArgumentException(message);
}
return reader;
}
/**

4
src/main/java/org/elasticsearch/index/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java
View File

@ -58,10 +58,10 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW
throw new ElasticsearchIllegalArgumentException("hyphenation_patterns_path is a required setting.");
}
Path hyphenationPatternsFile = env.resolveConfig(hyphenationPatternsPath);
URL hyphenationPatternsFile = env.resolveConfig(hyphenationPatternsPath);
try {
hyphenationTree = HyphenationCompoundWordTokenFilter.getHyphenationTree(new InputSource(Files.newInputStream(hyphenationPatternsFile)));
hyphenationTree = HyphenationCompoundWordTokenFilter.getHyphenationTree(new InputSource(hyphenationPatternsFile.toExternalForm()));
} catch (Exception e) {
throw new ElasticsearchIllegalArgumentException("Exception while reading hyphenation_patterns_path: " + e.getMessage());
}

42
src/main/java/org/elasticsearch/index/mapper/MapperService.java
View File

@ -40,6 +40,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.lucene.search.AndFilter;
import org.elasticsearch.common.lucene.search.NotFilter;
@ -138,11 +139,11 @@ public class MapperService extends AbstractIndexComponent {
this.dynamic = componentSettings.getAsBoolean("dynamic", true);
String defaultMappingLocation = componentSettings.get("default_mapping_location");
final Path defaultMappingUrl;
final URL defaultMappingUrl;
if (index.getName().equals(ScriptService.SCRIPT_INDEX)){
defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation,"script-mapping.json","org/elasticsearch/index/mapper/script-mapping.json");
defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation, "script-mapping.json", "org/elasticsearch/index/mapper/script-mapping.json");
} else {
defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation,"default-mapping.json","org/elasticsearch/index/mapper/default-mapping.json");
defaultMappingUrl = getMappingUrl(indexSettings, environment, defaultMappingLocation, "default-mapping.json", "org/elasticsearch/index/mapper/default-mapping.json");
}
if (defaultMappingUrl == null) {
@ -164,25 +165,29 @@ public class MapperService extends AbstractIndexComponent {
}
} else {
try {
defaultMappingSource = Streams.copyToString(new InputStreamReader(Files.newInputStream(defaultMappingUrl), Charsets.UTF_8));
defaultMappingSource = Streams.copyToString(FileSystemUtils.newBufferedReader(defaultMappingUrl, Charsets.UTF_8));
} catch (IOException e) {
throw new MapperException("Failed to load default mapping source from [" + defaultMappingLocation + "]", e);
}
}
String percolatorMappingLocation = componentSettings.get("default_percolator_mapping_location");
Path percolatorMappingUrl = null;
URL percolatorMappingUrl = null;
if (percolatorMappingLocation != null) {
try {
percolatorMappingUrl = environment.resolveConfig(percolatorMappingLocation);
} catch (FailedToResolveConfigException e) {
// not there, default to the built in one
percolatorMappingUrl = Paths.get(percolatorMappingLocation);
try {
percolatorMappingUrl = Paths.get(percolatorMappingLocation).toUri().toURL();
} catch (MalformedURLException e1) {
throw new FailedToResolveConfigException("Failed to resolve default percolator mapping location [" + percolatorMappingLocation + "]");
}
}
}
if (percolatorMappingUrl != null) {
try {
defaultPercolatorMappingSource = Streams.copyToString(new InputStreamReader(Files.newInputStream(percolatorMappingUrl), Charsets.UTF_8));
defaultPercolatorMappingSource = Streams.copyToString(FileSystemUtils.newBufferedReader(percolatorMappingUrl, Charsets.UTF_8));
} catch (IOException e) {
throw new MapperException("Failed to load default percolator mapping source from [" + percolatorMappingUrl + "]", e);
}
@ -208,20 +213,16 @@ public class MapperService extends AbstractIndexComponent {
}
}
private Path getMappingUrl(Settings indexSettings, Environment environment, String mappingLocation, String configString, String resourceLocation) {
Path mappingUrl;
private URL getMappingUrl(Settings indexSettings, Environment environment, String mappingLocation, String configString, String resourceLocation) {
URL mappingUrl;
if (mappingLocation == null) {
try {
mappingUrl = environment.resolveConfig(configString);
} catch (FailedToResolveConfigException e) {
try {
// not there, default to the built in one
mappingUrl = Paths.get(indexSettings.getClassLoader().getResource(resourceLocation).toURI());
if (mappingUrl == null) {
mappingUrl = Paths.get(MapperService.class.getClassLoader().getResource(resourceLocation).toURI());
}
} catch (URISyntaxException e1) {
throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + mappingLocation + "]");
// not there, default to the built in one
mappingUrl = indexSettings.getClassLoader().getResource(resourceLocation);
if (mappingUrl == null) {
mappingUrl = MapperService.class.getClassLoader().getResource(resourceLocation);
}
}
} else {
@ -229,8 +230,11 @@ public class MapperService extends AbstractIndexComponent {
mappingUrl = environment.resolveConfig(mappingLocation);
} catch (FailedToResolveConfigException e) {
// not there, default to the built in one
mappingUrl = Paths.get(mappingLocation);
try {
mappingUrl = Paths.get(mappingLocation).toUri().toURL();
} catch (MalformedURLException e1) {
throw new FailedToResolveConfigException("Failed to resolve dynamic mapping location [" + mappingLocation + "]");
}
}
}
return mappingUrl;

12
src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
View File

@ -60,33 +60,33 @@ public class InternalSettingsPreparer {
// if its default, then load it, but also load form env
if (Strings.hasText(System.getProperty("es.default.config"))) {
loadFromEnv = true;
settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("es.default.config")));
settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("es.default.config")));
}
// if explicit, just load it and don't load from env
if (Strings.hasText(System.getProperty("es.config"))) {
loadFromEnv = false;
settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("es.config")));
settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("es.config")));
}
if (Strings.hasText(System.getProperty("elasticsearch.config"))) {
loadFromEnv = false;
settingsBuilder.loadFromPath(environment.resolveConfig(System.getProperty("elasticsearch.config")));
settingsBuilder.loadFromUrl(environment.resolveConfig(System.getProperty("elasticsearch.config")));
}
}
if (loadFromEnv) {
try {
settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.yml"));
settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.yml"));
} catch (FailedToResolveConfigException e) {
// ignore
} catch (NoClassDefFoundError e) {
// ignore, no yaml
}
try {
settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.json"));
settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.json"));
} catch (FailedToResolveConfigException e) {
// ignore
}
try {
settingsBuilder.loadFromPath(environment.resolveConfig("elasticsearch.properties"));
settingsBuilder.loadFromUrl(environment.resolveConfig("elasticsearch.properties"));
} catch (FailedToResolveConfigException e) {
// ignore
}

2
src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -50,8 +50,6 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import static org.elasticsearch.common.Strings.hasLength;
import static org.elasticsearch.common.io.FileSystemUtils.moveFilesWithoutOverwriting;

84
src/test/java/org/elasticsearch/env/EnvironmentTests.java vendored Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.env;
import com.google.common.base.Charsets;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import java.io.BufferedReader;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Simple unit-tests for Environment.java
*/
public class EnvironmentTests extends ElasticsearchTestCase {
public Environment newEnvironment() throws IOException {
return newEnvironment(ImmutableSettings.EMPTY);
}
public Environment newEnvironment(Settings settings) throws IOException {
Settings build = ImmutableSettings.builder()
.put(settings)
.put("path.home", newTempDir().getAbsolutePath())
.putArray("path.data", tmpPaths()).build();
return new Environment(build);
}
@Test
public void testResolveJaredResource() throws IOException {
Environment environment = newEnvironment();
URL url = environment.resolveConfig("META-INF/MANIFEST.MF"); // this works because there is one jar having this file in the classpath
assertNotNull(url);
try (BufferedReader reader = FileSystemUtils.newBufferedReader(url, Charsets.UTF_8)) {
String string = Streams.copyToString(reader);
assertTrue(string, string.contains("Manifest-Version"));
}
}
@Test
public void testResolveFileResource() throws IOException {
Environment environment = newEnvironment();
URL url = environment.resolveConfig("org/elasticsearch/common/cli/tool.help");
assertNotNull(url);
try (BufferedReader reader = FileSystemUtils.newBufferedReader(url, Charsets.UTF_8)) {
String string = Streams.copyToString(reader);
assertEquals(string, "tool help");
}
}
}

11
src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java vendored
View File

@ -284,15 +284,6 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
env.close();
}
private String[] tmpPaths() {
final int numPaths = randomIntBetween(1, 3);
final String[] absPaths = new String[numPaths];
for (int i = 0; i < numPaths; i++) {
absPaths[i] = newTempDir().getAbsolutePath();
}
return absPaths;
}
public NodeEnvironment newNodeEnvironment() throws IOException {
return newNodeEnvironment(ImmutableSettings.EMPTY);
}
@ -304,4 +295,6 @@ public class NodeEnvironmentTests extends ElasticsearchTestCase {
.putArray("path.data", tmpPaths()).build();
return new NodeEnvironment(build, new Environment(build));
}
}

12
src/test/java/org/elasticsearch/test/ElasticsearchTestCase.java
View File

@ -560,4 +560,16 @@ public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
return ThreadPool.terminate(service, 10, TimeUnit.SECONDS);
}
/**
* Returns a random number of temporary paths.
*/
public String[] tmpPaths() {
final int numPaths = randomIntBetween(1, 3);
final String[] absPaths = new String[numPaths];
for (int i = 0; i < numPaths; i++) {
absPaths[i] = newTempDir().getAbsolutePath();
}
return absPaths;
}
}