honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10755 from rmuir/filestores 

refactor SSD/FileStore logic out of NodeEnvironment
This commit is contained in:
Robert Muir 2015-04-23 12:50:10 -04:00
parent adc0807c68 a66cf85f37
commit 05138151a2
3 changed files with 210 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
src/main/java/org/elasticsearch/env/ESFileStore.java vendored Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.env;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.io.PathUtils;
import java.io.IOException;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.attribute.FileStoreAttributeView;
/**
* Implementation of FileStore that supports
* additional features, such as SSD detection and better
* filesystem information for the root filesystem.
* @see Environment#getFileStore(Path)
*/
class ESFileStore extends FileStore {
/** Underlying filestore */
final FileStore in;
/** Cached result of Lucene's {@code IOUtils.spins} on path. */
final Boolean spins;
ESFileStore(FileStore in) {
this.in = in;
Boolean spins;
// Lucene's IOUtils.spins only works on Linux today:
if (Constants.LINUX) {
try {
spins = IOUtils.spins(PathUtils.get(getMountPointLinux(in)));
} catch (Exception e) {
spins = null;
}
} else {
spins = null;
}
this.spins = spins;
}
// these are hacks that are not guaranteed
private static String getMountPointLinux(FileStore store) {
String desc = store.toString();
int index = desc.lastIndexOf(" (");
if (index != -1) {
return desc.substring(0, index);
} else {
return desc;
}
}
/** Files.getFileStore(Path) useless here! Don't complain, just try it yourself. */
static FileStore getMatchingFileStore(Path path, FileStore fileStores[]) throws IOException {
FileStore store = Files.getFileStore(path);
if (Constants.WINDOWS) {
return store; // be defensive, don't even try to do anything fancy.
}
try {
String mount = getMountPointLinux(store);
FileStore sameMountPoint = null;
for (FileStore fs : fileStores) {
if (mount.equals(getMountPointLinux(fs))) {
if (sameMountPoint == null) {
sameMountPoint = fs;
} else {
// more than one filesystem has the same mount point; something is wrong!
// fall back to crappy one we got from Files.getFileStore
return store;
}
}
}
if (sameMountPoint != null) {
// ok, we found only one, use it:
return sameMountPoint;
} else {
// fall back to crappy one we got from Files.getFileStore
return store;
}
} catch (Exception e) {
// ignore
}
// fall back to crappy one we got from Files.getFileStore
return store;
}
@Override
public String name() {
return in.name();
}
@Override
public String type() {
return in.type();
}
@Override
public boolean isReadOnly() {
return in.isReadOnly();
}
@Override
public long getTotalSpace() throws IOException {
return in.getTotalSpace();
}
@Override
public long getUsableSpace() throws IOException {
return in.getUsableSpace();
}
@Override
public long getUnallocatedSpace() throws IOException {
return in.getUnallocatedSpace();
}
@Override
public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type) {
return in.supportsFileAttributeView(type);
}
@Override
public boolean supportsFileAttributeView(String name) {
if ("lucene".equals(name)) {
return true;
} else {
return in.supportsFileAttributeView(name);
}
}
@Override
public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type) {
return in.getFileStoreAttributeView(type);
}
@Override
public Object getAttribute(String attribute) throws IOException {
if ("lucene:spins".equals(attribute)) {
return spins;
} else {
return in.getAttribute(attribute);
}
}
@Override
public String toString() {
return in.toString();
}
}

42
src/main/java/org/elasticsearch/env/Environment.java vendored
View File

@ -20,22 +20,16 @@
package org.elasticsearch.env;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import com.google.common.base.Charsets;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.*;
import java.util.Collections;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import static org.elasticsearch.common.Strings.cleanPath;
import static org.elasticsearch.common.settings.ImmutableSettings.Builder.EMPTY_SETTINGS;
@ -63,6 +57,9 @@ public class Environment {
private final Path logsFile;
/** List of filestores on the system */
private final FileStore[] fileStores;
public Environment() {
this(EMPTY_SETTINGS);
}
@ -112,6 +109,13 @@ public class Environment {
} else {
logsFile = homeFile.resolve("logs");
}
// gather information about filesystems
ArrayList<FileStore> allStores = new ArrayList<>();
for (FileStore store : PathUtils.getDefaultFileSystem().getFileStores()) {
allStores.add(new ESFileStore(store));
}
fileStores = allStores.toArray(new ESFileStore[allStores.size()]);
}
/**
@ -177,6 +181,24 @@ public class Environment {
return logsFile;
}
/**
* Looks up the filestore associated with a Path.
* <p>
* This is an enhanced version of {@link Files#getFileStore(Path)}:
* <ul>
* <li>On *nix systems, the store returned for the root filesystem will contain
* the actual filesystem type (e.g. {@code ext4}) instead of {@code rootfs}.
* <li>On some systems, the custom attribute {@code lucene:spins} is supported
* via the {@link FileStore#getAttribute(String)} method.
* <li>Only requires the security permissions of {@link Files#getFileStore(Path)},
* no permissions to the actual mount point are required.
* <li>Exception handling has the same semantics as {@link Files#getFileStore(Path)}.
* </ul>
*/
public FileStore getFileStore(Path path) throws IOException {
return ESFileStore.getMatchingFileStore(path, fileStores);
}
public URL resolveConfig(String path) throws FailedToResolveConfigException {
String origPath = path;
// first, try it as a path on the file system

73
src/main/java/org/elasticsearch/env/NodeEnvironment.java vendored
View File

@ -22,7 +22,6 @@ package org.elasticsearch.env;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
@ -64,23 +63,15 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
* not running on Linux, or we hit an exception trying), True means the device possibly spins and False means it does not. */
public final Boolean spins;
public NodePath(Path path) throws IOException {
public NodePath(Path path, Environment environment) throws IOException {
this.path = path;
this.indicesPath = path.resolve(INDICES_FOLDER);
this.fileStore = getFileStore(path);
Boolean spins;
// Lucene's IOUtils.spins only works on Linux today:
if (Constants.LINUX) {
try {
spins = IOUtils.spins(path);
} catch (Exception e) {
spins = null;
}
this.fileStore = environment.getFileStore(path);
if (fileStore.supportsFileAttributeView("lucene")) {
this.spins = (Boolean) fileStore.getAttribute("lucene:spins");
} else {
spins = null;
this.spins = null;
}
this.spins = spins;
}
/**
@ -157,7 +148,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
Lock tmpLock = luceneDir.makeLock(NODE_LOCK_FILENAME);
boolean obtained = tmpLock.obtain();
if (obtained) {
nodePaths[dirIndex] = new NodePath(dir);
nodePaths[dirIndex] = new NodePath(dir, environment);
locks[dirIndex] = tmpLock;
localNodeId = possibleLockId;
} else {
@ -289,58 +280,6 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
return b.toString();
}
// TODO: move somewhere more "util"? But, this is somewhat hacky code ... not great to publicize it any more:
// NOTE: poached from Lucene's IOUtils:
/** Files.getFileStore(Path) useless here! Don't complain, just try it yourself. */
private static FileStore getFileStore(Path path) throws IOException {
FileStore store = Files.getFileStore(path);
try {
String mount = getMountPoint(store);
FileStore sameMountPoint = null;
for (FileStore fs : path.getFileSystem().getFileStores()) {
if (mount.equals(getMountPoint(fs))) {
if (sameMountPoint == null) {
sameMountPoint = fs;
} else {
// more than one filesystem has the same mount point; something is wrong!
// fall back to crappy one we got from Files.getFileStore
return store;
}
}
}
if (sameMountPoint != null) {
// ok, we found only one, use it:
return sameMountPoint;
} else {
// fall back to crappy one we got from Files.getFileStore
return store;
}
} catch (Exception e) {
// ignore
}
// fall back to crappy one we got from Files.getFileStore
return store;
}
// NOTE: poached from Lucene's IOUtils:
// these are hacks that are not guaranteed
private static String getMountPoint(FileStore store) {
String desc = store.toString();
int index = desc.lastIndexOf(" (");
if (index != -1) {
return desc.substring(0, index);
} else {
return desc;
}
}
/**
* Deletes a shard data directory iff the shards locks were successfully acquired.
*