SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config if it is not found in solrconfig.xml.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1651373 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2015-01-13 15:12:46 +00:00
parent 8c86381eab
commit 1696206de6
6 changed files with 219 additions and 61 deletions

View File

@ -691,6 +691,9 @@ Other Changes
* SOLR-6904: Removed deprecated Circle & rect syntax. See upgrading notes. (David Smiley)
* SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config
if it is not found in solrconfig.xml. (Mark Miller, Mike Drob)
================== 4.10.3 ==================
Bug Fixes

View File

@ -61,7 +61,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled";
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
@ -96,13 +96,13 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
public void init(NamedList args) {
params = SolrParams.toSolrParams(args);
this.hdfsDataDir = params.get(HDFS_HOME);
this.hdfsDataDir = getConfig(HDFS_HOME, null);
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
this.hdfsDataDir = null;
} else {
LOG.info(HDFS_HOME + "=" + this.hdfsDataDir);
}
boolean kerberosEnabled = params.getBool(KERBEROS_ENABLED, false);
boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
LOG.info("Solr Kerberos Authentication "
+ (kerberosEnabled ? "enabled" : "disabled"));
if (kerberosEnabled) {
@ -132,6 +132,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
assert params != null : "init must be called before create";
LOG.info("creating directory factory for path {}", path);
Configuration conf = getConf();
@ -139,25 +140,19 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
metrics = MetricsHolder.metrics;
}
boolean blockCacheEnabled = params.getBool(BLOCKCACHE_ENABLED, true);
boolean blockCacheGlobal = params.getBool(BLOCKCACHE_GLOBAL, false); // default to false for back compat
boolean blockCacheReadEnabled = params.getBool(BLOCKCACHE_READ_ENABLED, true);
boolean blockCacheWriteEnabled = params.getBool(BLOCKCACHE_WRITE_ENABLED, false);
if (blockCacheWriteEnabled) {
LOG.warn("Using " + BLOCKCACHE_WRITE_ENABLED + " is currently buggy and can result in readers seeing a corrupted view of the index.");
}
boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, false); // default to false for back compat
boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
final Directory dir;
if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
int numberOfBlocksPerBank = params.getInt(NUMBEROFBLOCKSPERBANK, 16384);
int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
int blockSize = BlockDirectory.BLOCK_SIZE;
int bankCount = params.getInt(BLOCKCACHE_SLAB_COUNT, 1);
int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
boolean directAllocation = params.getBool(
BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
int slabSize = numberOfBlocksPerBank * blockSize;
LOG.info(
@ -168,8 +163,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
new Object[] {slabSize, bankCount,
((long) bankCount * (long) slabSize)});
int bufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", 128);
int bufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
int bufferSize = getConfig("solr.hdfs.blockcache.bufferstore.buffersize", 128);
int bufferCount = getConfig("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank,
blockSize, bankCount, directAllocation, slabSize,
@ -177,25 +172,51 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
HdfsDirectory hdfsDirectory = new HdfsDirectory(new Path(path), lockFactory, conf);
dir = new BlockDirectory(path, hdfsDirectory, cache, null,
blockCacheReadEnabled, blockCacheWriteEnabled);
dir = new BlockDirectory(path, hdfsDirectory, cache, null, blockCacheReadEnabled, false);
} else {
dir = new HdfsDirectory(new Path(path), lockFactory, conf);
}
boolean nrtCachingDirectory = params.getBool(NRTCACHINGDIRECTORY_ENABLE, true);
boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
if (nrtCachingDirectory) {
double nrtCacheMaxMergeSizeMB = params.getInt(
NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
double nrtCacheMaxCacheMB = params.getInt(NRTCACHINGDIRECTORY_MAXCACHEMB,
192);
double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB,
nrtCacheMaxCacheMB);
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
}
return dir;
}
boolean getConfig(String name, boolean defaultValue) {
Boolean value = params.getBool(name);
if (value == null) {
String sysValue = System.getProperty(name);
if (sysValue != null) {
value = Boolean.valueOf(sysValue);
}
}
return value == null ? defaultValue : value;
}
int getConfig(String name, int defaultValue) {
Integer value = params.getInt(name);
if (value == null) {
String sysValue = System.getProperty(name);
if (sysValue != null) {
value = Integer.parseInt(sysValue);
}
}
return value == null ? defaultValue : value;
}
String getConfig(String name, String defaultValue) {
String value = params.get(name);
if (value == null) {
value = System.getProperty(name);
}
return value == null ? defaultValue : value;
}
private BlockCache getBlockDirectoryCache(int numberOfBlocksPerBank, int blockSize, int bankCount,
boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
if (!staticBlockCache) {
@ -252,7 +273,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
private Configuration getConf() {
Configuration conf = new Configuration();
confDir = params.get(CONFIG_DIRECTORY, null);
confDir = getConfig(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
return conf;
}
@ -329,12 +350,12 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
}
private void initKerberos() {
String keytabFile = params.get(KERBEROS_KEYTAB, "").trim();
String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
if (keytabFile.length() == 0) {
throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
+ KERBEROS_ENABLED + " set to true");
}
String principal = params.get(KERBEROS_PRINCIPAL, "");
String principal = getConfig(KERBEROS_PRINCIPAL, "");
if (principal.length() == 0) {
throw new IllegalArgumentException(KERBEROS_PRINCIPAL
+ " required because " + KERBEROS_ENABLED + " set to true");

View File

@ -39,6 +39,10 @@ public class HdfsTestUtil {
private static Map<MiniDFSCluster,Timer> timers = new ConcurrentHashMap<>();
public static MiniDFSCluster setupClass(String dir) throws Exception {
return setupClass(dir, true);
}
public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting) throws Exception {
LuceneTestCase.assumeFalse("HDFS tests were disabled by -Dtests.disableHdfs",
Boolean.parseBoolean(System.getProperty("tests.disableHdfs", "false")));
@ -68,20 +72,21 @@ public class HdfsTestUtil {
System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
int rnd = LuceneTestCase.random().nextInt(10000);
Timer timer = new Timer();
timer.schedule(new TimerTask() {
if (safeModeTesting) {
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
@Override
public void run() {
NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
}
}, rnd);
timers.put(dfsCluster, timer);
int rnd = LuceneTestCase.random().nextInt(10000);
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
}
}, rnd);
timers.put(dfsCluster, timer);
}
SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");

View File

@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.nio.file.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.util.MockCoreContainer.MockCoreDescriptor;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
System.clearProperty("solr.hdfs.home");
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
dfsCluster = null;
}
@Test
public void testInitArgsOrSysPropConfig() throws Exception {
HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
// test sys prop config
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
hdfsFactory.init(new NamedList<>());
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr1/mock/data"));
System.clearProperty("solr.hdfs.home");
// test init args config
NamedList<Object> nl = new NamedList<>();
nl.add("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr2");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr2/mock/data"));
// test sys prop and init args config - init args wins
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
hdfsFactory.init(nl);
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
assertTrue(dataHome.endsWith("/solr2/mock/data"));
System.clearProperty("solr.hdfs.home");
// set conf dir by sys prop
Path confDir = createTempDir();
System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
Directory dir = hdfsFactory.create(dfsCluster.getURI().toString() + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT);
try {
assertEquals(confDir.toString(), hdfsFactory.getConfDir());
} finally {
dir.close();
}
// check bool and int getConf impls
nl = new NamedList<>();
nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
hdfsFactory.init(nl);
assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
nl = new NamedList<>();
hdfsFactory.init(nl);
System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
assertEquals(false, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
hdfsFactory.close();
}
}

View File

@ -1,7 +1,5 @@
package org.apache.solr.util;
import org.apache.solr.core.CoreContainer;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -19,6 +17,22 @@ import org.apache.solr.core.CoreContainer;
* limitations under the License.
*/
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
public class MockCoreContainer extends CoreContainer {
public static class MockCoreDescriptor extends CoreDescriptor {
public MockCoreDescriptor() {
super(new MockCoreContainer(), "mock", "path");
}
}
public MockCoreContainer() {
super(new Object());
}
public String getCoreRootDirectory() {
return "coreroot";
}
}

View File

@ -117,23 +117,7 @@
persistent, and doesn't work with replication.
-->
<directoryFactory name="DirectoryFactory"
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}">
<!-- These will be used if you are using the solr.HdfsDirectoryFactory,
otherwise they will be ignored. If you don't plan on using hdfs,
you can safely remove this section. -->
<!-- The root directory that collection data should be written to. -->
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
<!-- The hadoop configuration files to use for the hdfs client. -->
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
<!-- Enable/Disable the hdfs cache. -->
<str name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</str>
<!-- Enable/Disable using one global cache for all SolrCores.
The settings used will be from the first HdfsDirectoryFactory created. -->
<str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
</directoryFactory>
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
<!-- The CodecFactory for defining the format of the inverted index.
The default implementation is SchemaCodecFactory, which is the official Lucene