mirror of https://github.com/apache/lucene.git
SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config if it is not found in solrconfig.xml.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1651373 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller
parent
8c86381eab
commit
1696206de6
|
|
@ -691,6 +691,9 @@ Other Changes
|
|||
|
||||
* SOLR-6904: Removed deprecated Circle & rect syntax. See upgrading notes. (David Smiley)
|
||||
|
||||
* SOLR-6943: HdfsDirectoryFactory should fall back to system props for most of it's config
|
||||
if it is not found in solrconfig.xml. (Mark Miller, Mike Drob)
|
||||
|
||||
================== 4.10.3 ==================
|
||||
|
||||
Bug Fixes
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
|
||||
public static final String BLOCKCACHE_GLOBAL = "solr.hdfs.blockcache.global";
|
||||
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
|
||||
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled";
|
||||
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled"; // currently buggy and disabled
|
||||
|
||||
public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
|
||||
public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
|
||||
|
|
@ -96,13 +96,13 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
@Override
|
||||
public void init(NamedList args) {
|
||||
params = SolrParams.toSolrParams(args);
|
||||
this.hdfsDataDir = params.get(HDFS_HOME);
|
||||
this.hdfsDataDir = getConfig(HDFS_HOME, null);
|
||||
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
|
||||
this.hdfsDataDir = null;
|
||||
} else {
|
||||
LOG.info(HDFS_HOME + "=" + this.hdfsDataDir);
|
||||
}
|
||||
boolean kerberosEnabled = params.getBool(KERBEROS_ENABLED, false);
|
||||
boolean kerberosEnabled = getConfig(KERBEROS_ENABLED, false);
|
||||
LOG.info("Solr Kerberos Authentication "
|
||||
+ (kerberosEnabled ? "enabled" : "disabled"));
|
||||
if (kerberosEnabled) {
|
||||
|
|
@ -132,6 +132,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
|
||||
@Override
|
||||
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
|
||||
assert params != null : "init must be called before create";
|
||||
LOG.info("creating directory factory for path {}", path);
|
||||
Configuration conf = getConf();
|
||||
|
||||
|
|
@ -139,25 +140,19 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
metrics = MetricsHolder.metrics;
|
||||
}
|
||||
|
||||
boolean blockCacheEnabled = params.getBool(BLOCKCACHE_ENABLED, true);
|
||||
boolean blockCacheGlobal = params.getBool(BLOCKCACHE_GLOBAL, false); // default to false for back compat
|
||||
boolean blockCacheReadEnabled = params.getBool(BLOCKCACHE_READ_ENABLED, true);
|
||||
boolean blockCacheWriteEnabled = params.getBool(BLOCKCACHE_WRITE_ENABLED, false);
|
||||
|
||||
if (blockCacheWriteEnabled) {
|
||||
LOG.warn("Using " + BLOCKCACHE_WRITE_ENABLED + " is currently buggy and can result in readers seeing a corrupted view of the index.");
|
||||
}
|
||||
boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
|
||||
boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, false); // default to false for back compat
|
||||
boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
|
||||
|
||||
final Directory dir;
|
||||
if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
|
||||
int numberOfBlocksPerBank = params.getInt(NUMBEROFBLOCKSPERBANK, 16384);
|
||||
int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
|
||||
|
||||
int blockSize = BlockDirectory.BLOCK_SIZE;
|
||||
|
||||
int bankCount = params.getInt(BLOCKCACHE_SLAB_COUNT, 1);
|
||||
int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
|
||||
|
||||
boolean directAllocation = params.getBool(
|
||||
BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
|
||||
boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
|
||||
|
||||
int slabSize = numberOfBlocksPerBank * blockSize;
|
||||
LOG.info(
|
||||
|
|
@ -168,8 +163,8 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
new Object[] {slabSize, bankCount,
|
||||
((long) bankCount * (long) slabSize)});
|
||||
|
||||
int bufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", 128);
|
||||
int bufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
|
||||
int bufferSize = getConfig("solr.hdfs.blockcache.bufferstore.buffersize", 128);
|
||||
int bufferCount = getConfig("solr.hdfs.blockcache.bufferstore.buffercount", 128 * 128);
|
||||
|
||||
BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank,
|
||||
blockSize, bankCount, directAllocation, slabSize,
|
||||
|
|
@ -177,25 +172,51 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
|
||||
Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
|
||||
HdfsDirectory hdfsDirectory = new HdfsDirectory(new Path(path), lockFactory, conf);
|
||||
dir = new BlockDirectory(path, hdfsDirectory, cache, null,
|
||||
blockCacheReadEnabled, blockCacheWriteEnabled);
|
||||
dir = new BlockDirectory(path, hdfsDirectory, cache, null, blockCacheReadEnabled, false);
|
||||
} else {
|
||||
dir = new HdfsDirectory(new Path(path), lockFactory, conf);
|
||||
}
|
||||
|
||||
boolean nrtCachingDirectory = params.getBool(NRTCACHINGDIRECTORY_ENABLE, true);
|
||||
boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
|
||||
if (nrtCachingDirectory) {
|
||||
double nrtCacheMaxMergeSizeMB = params.getInt(
|
||||
NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
|
||||
double nrtCacheMaxCacheMB = params.getInt(NRTCACHINGDIRECTORY_MAXCACHEMB,
|
||||
192);
|
||||
double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
|
||||
double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
|
||||
|
||||
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB,
|
||||
nrtCacheMaxCacheMB);
|
||||
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
boolean getConfig(String name, boolean defaultValue) {
|
||||
Boolean value = params.getBool(name);
|
||||
if (value == null) {
|
||||
String sysValue = System.getProperty(name);
|
||||
if (sysValue != null) {
|
||||
value = Boolean.valueOf(sysValue);
|
||||
}
|
||||
}
|
||||
return value == null ? defaultValue : value;
|
||||
}
|
||||
|
||||
int getConfig(String name, int defaultValue) {
|
||||
Integer value = params.getInt(name);
|
||||
if (value == null) {
|
||||
String sysValue = System.getProperty(name);
|
||||
if (sysValue != null) {
|
||||
value = Integer.parseInt(sysValue);
|
||||
}
|
||||
}
|
||||
return value == null ? defaultValue : value;
|
||||
}
|
||||
|
||||
String getConfig(String name, String defaultValue) {
|
||||
String value = params.get(name);
|
||||
if (value == null) {
|
||||
value = System.getProperty(name);
|
||||
}
|
||||
return value == null ? defaultValue : value;
|
||||
}
|
||||
|
||||
private BlockCache getBlockDirectoryCache(int numberOfBlocksPerBank, int blockSize, int bankCount,
|
||||
boolean directAllocation, int slabSize, int bufferSize, int bufferCount, boolean staticBlockCache) {
|
||||
if (!staticBlockCache) {
|
||||
|
|
@ -252,7 +273,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
|
||||
private Configuration getConf() {
|
||||
Configuration conf = new Configuration();
|
||||
confDir = params.get(CONFIG_DIRECTORY, null);
|
||||
confDir = getConfig(CONFIG_DIRECTORY, null);
|
||||
HdfsUtil.addHdfsResources(conf, confDir);
|
||||
return conf;
|
||||
}
|
||||
|
|
@ -329,12 +350,12 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
|||
}
|
||||
|
||||
private void initKerberos() {
|
||||
String keytabFile = params.get(KERBEROS_KEYTAB, "").trim();
|
||||
String keytabFile = getConfig(KERBEROS_KEYTAB, "").trim();
|
||||
if (keytabFile.length() == 0) {
|
||||
throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
|
||||
+ KERBEROS_ENABLED + " set to true");
|
||||
}
|
||||
String principal = params.get(KERBEROS_PRINCIPAL, "");
|
||||
String principal = getConfig(KERBEROS_PRINCIPAL, "");
|
||||
if (principal.length() == 0) {
|
||||
throw new IllegalArgumentException(KERBEROS_PRINCIPAL
|
||||
+ " required because " + KERBEROS_ENABLED + " set to true");
|
||||
|
|
|
|||
|
|
@ -39,6 +39,10 @@ public class HdfsTestUtil {
|
|||
private static Map<MiniDFSCluster,Timer> timers = new ConcurrentHashMap<>();
|
||||
|
||||
public static MiniDFSCluster setupClass(String dir) throws Exception {
|
||||
return setupClass(dir, true);
|
||||
}
|
||||
|
||||
public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting) throws Exception {
|
||||
LuceneTestCase.assumeFalse("HDFS tests were disabled by -Dtests.disableHdfs",
|
||||
Boolean.parseBoolean(System.getProperty("tests.disableHdfs", "false")));
|
||||
|
||||
|
|
@ -68,20 +72,21 @@ public class HdfsTestUtil {
|
|||
|
||||
System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home"));
|
||||
|
||||
|
||||
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
|
||||
|
||||
int rnd = LuceneTestCase.random().nextInt(10000);
|
||||
Timer timer = new Timer();
|
||||
timer.schedule(new TimerTask() {
|
||||
if (safeModeTesting) {
|
||||
NameNodeAdapter.enterSafeMode(dfsCluster.getNameNode(), false);
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
|
||||
}
|
||||
}, rnd);
|
||||
|
||||
timers.put(dfsCluster, timer);
|
||||
int rnd = LuceneTestCase.random().nextInt(10000);
|
||||
Timer timer = new Timer();
|
||||
timer.schedule(new TimerTask() {
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
NameNodeAdapter.leaveSafeMode(dfsCluster.getNameNode());
|
||||
}
|
||||
}, rnd);
|
||||
|
||||
timers.put(dfsCluster, timer);
|
||||
}
|
||||
|
||||
SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.NoLockFactory;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.DirectoryFactory.DirContext;
|
||||
import org.apache.solr.util.MockCoreContainer.MockCoreDescriptor;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
|
||||
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
|
||||
|
||||
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
|
||||
public class HdfsDirectoryFactoryTest extends SolrTestCaseJ4 {
|
||||
|
||||
private static MiniDFSCluster dfsCluster;
|
||||
|
||||
@BeforeClass
|
||||
public static void setupClass() throws Exception {
|
||||
dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath(), false);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardownClass() throws Exception {
|
||||
HdfsTestUtil.teardownClass(dfsCluster);
|
||||
System.clearProperty("solr.hdfs.home");
|
||||
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
|
||||
dfsCluster = null;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInitArgsOrSysPropConfig() throws Exception {
|
||||
|
||||
HdfsDirectoryFactory hdfsFactory = new HdfsDirectoryFactory();
|
||||
|
||||
// test sys prop config
|
||||
|
||||
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
|
||||
hdfsFactory.init(new NamedList<>());
|
||||
String dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
|
||||
|
||||
assertTrue(dataHome.endsWith("/solr1/mock/data"));
|
||||
|
||||
System.clearProperty("solr.hdfs.home");
|
||||
|
||||
// test init args config
|
||||
|
||||
NamedList<Object> nl = new NamedList<>();
|
||||
nl.add("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr2");
|
||||
hdfsFactory.init(nl);
|
||||
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
|
||||
|
||||
assertTrue(dataHome.endsWith("/solr2/mock/data"));
|
||||
|
||||
// test sys prop and init args config - init args wins
|
||||
|
||||
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr1");
|
||||
hdfsFactory.init(nl);
|
||||
dataHome = hdfsFactory.getDataHome(new MockCoreDescriptor());
|
||||
|
||||
assertTrue(dataHome.endsWith("/solr2/mock/data"));
|
||||
|
||||
System.clearProperty("solr.hdfs.home");
|
||||
|
||||
|
||||
// set conf dir by sys prop
|
||||
|
||||
Path confDir = createTempDir();
|
||||
|
||||
System.setProperty(HdfsDirectoryFactory.CONFIG_DIRECTORY, confDir.toString());
|
||||
|
||||
Directory dir = hdfsFactory.create(dfsCluster.getURI().toString() + "/solr", NoLockFactory.INSTANCE, DirContext.DEFAULT);
|
||||
try {
|
||||
assertEquals(confDir.toString(), hdfsFactory.getConfDir());
|
||||
} finally {
|
||||
dir.close();
|
||||
}
|
||||
|
||||
// check bool and int getConf impls
|
||||
nl = new NamedList<>();
|
||||
nl.add(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 4);
|
||||
System.setProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, "3");
|
||||
nl.add(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, true);
|
||||
System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "false");
|
||||
|
||||
hdfsFactory.init(nl);
|
||||
|
||||
assertEquals(4, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
|
||||
assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
|
||||
|
||||
nl = new NamedList<>();
|
||||
hdfsFactory.init(nl);
|
||||
System.setProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, "true");
|
||||
|
||||
assertEquals(3, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
|
||||
assertEquals(true, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
|
||||
|
||||
System.clearProperty(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB);
|
||||
System.clearProperty(HdfsDirectoryFactory.BLOCKCACHE_ENABLED);
|
||||
|
||||
assertEquals(0, hdfsFactory.getConfig(HdfsDirectoryFactory.NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 0));
|
||||
assertEquals(false, hdfsFactory.getConfig(HdfsDirectoryFactory.BLOCKCACHE_ENABLED, false));
|
||||
|
||||
hdfsFactory.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,7 +1,5 @@
|
|||
package org.apache.solr.util;
|
||||
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
|
|
@ -19,6 +17,22 @@ import org.apache.solr.core.CoreContainer;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.core.CoreDescriptor;
|
||||
|
||||
|
||||
public class MockCoreContainer extends CoreContainer {
|
||||
public static class MockCoreDescriptor extends CoreDescriptor {
|
||||
public MockCoreDescriptor() {
|
||||
super(new MockCoreContainer(), "mock", "path");
|
||||
}
|
||||
}
|
||||
|
||||
public MockCoreContainer() {
|
||||
super(new Object());
|
||||
}
|
||||
|
||||
public String getCoreRootDirectory() {
|
||||
return "coreroot";
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,23 +117,7 @@
|
|||
persistent, and doesn't work with replication.
|
||||
-->
|
||||
<directoryFactory name="DirectoryFactory"
|
||||
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}">
|
||||
|
||||
|
||||
<!-- These will be used if you are using the solr.HdfsDirectoryFactory,
|
||||
otherwise they will be ignored. If you don't plan on using hdfs,
|
||||
you can safely remove this section. -->
|
||||
<!-- The root directory that collection data should be written to. -->
|
||||
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
|
||||
<!-- The hadoop configuration files to use for the hdfs client. -->
|
||||
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
|
||||
<!-- Enable/Disable the hdfs cache. -->
|
||||
<str name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</str>
|
||||
<!-- Enable/Disable using one global cache for all SolrCores.
|
||||
The settings used will be from the first HdfsDirectoryFactory created. -->
|
||||
<str name="solr.hdfs.blockcache.global">${solr.hdfs.blockcache.global:true}</str>
|
||||
|
||||
</directoryFactory>
|
||||
class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
|
||||
|
||||
<!-- The CodecFactory for defining the format of the inverted index.
|
||||
The default implementation is SchemaCodecFactory, which is the official Lucene
|
||||
|
|
|
|||
Loading…
Reference in New Issue