From d362439e277c63a0e3be539179d3b18cf96df617 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Mon, 29 Oct 2018 08:23:14 -0400 Subject: [PATCH] SOLR-12861: Add Solr factory for new ByteBuffersDirectory --- solr/CHANGES.txt | 5 ++ .../core/ByteBuffersDirectoryFactory.java | 46 ++++++++++++ .../core/ByteBuffersDirectoryFactoryTest.java | 72 +++++++++++++++++++ .../core/CachingDirectoryFactoryTest.java | 7 +- .../solr/core/DirectoryFactoryTest.java | 45 +++++++----- 5 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java create mode 100644 solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2c79aaabe33..ef91ba3ec03 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -129,6 +129,9 @@ Upgrade Notes be set to -1 during collection creation to fall back to the old behavior of unlimited maxShardsPerNode when using autoscaling policy. +* SOLR-12861: Added a Solr factory for ByteBuffersDirectory, which will replace deprecated RAMDirectory in + a future version of Solr. + New Features ---------------------- @@ -190,6 +193,8 @@ Other Changes * SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package (Varun Thacker) +* SOLR-12861: Add Solr factory for ByteBuffersDirectory. + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java new file mode 100644 index 00000000000..ba27650d403 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/ByteBuffersDirectoryFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.SingleInstanceLockFactory; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; + +/** + * Factory to instantiate {@link org.apache.lucene.store.ByteBuffersDirectory} + */ +public class ByteBuffersDirectoryFactory extends EphemeralDirectoryFactory { + + @Override + protected LockFactory createLockFactory(String rawLockType) throws IOException { + if (!(rawLockType == null || DirectoryFactory.LOCK_TYPE_SINGLE.equalsIgnoreCase(rawLockType.trim()))) { + throw new SolrException(ErrorCode.FORBIDDEN, + "ByteBuffersDirectory can only be used with the '"+DirectoryFactory.LOCK_TYPE_SINGLE+"' lock factory type."); + } + return new SingleInstanceLockFactory(); + } + + @Override + protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException { + return new ByteBuffersDirectory(lockFactory); + } +} diff --git a/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java new file mode 100644 index 00000000000..45c7a152c00 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/ByteBuffersDirectoryFactoryTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.core.DirectoryFactory.DirContext; + +/** + * Test-case for ByteBuffersDirectoryFactory + */ +public class ByteBuffersDirectoryFactoryTest extends SolrTestCaseJ4 { + + public void testOpenReturnsTheSameForSamePath() throws IOException { + final Directory directory = new ByteBuffersDirectory(); + ByteBuffersDirectoryFactory factory = new ByteBuffersDirectoryFactory() { + @Override + protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) { + return directory; + } + }; + String path = "/fake/path"; + Directory dir1 = factory.get(path, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE); + Directory dir2 = factory.get(path, DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE); + assertEquals("ByteBuffersDirectoryFactory should not create new instance of ByteBuffersDirectory " + + "every time open() is called for the same path", dir1, dir2); + + factory.release(dir1); + factory.release(dir2); + factory.close(); + } + + public void testOpenSucceedForEmptyDir() throws IOException { + ByteBuffersDirectoryFactory factory = new ByteBuffersDirectoryFactory(); + Directory dir = factory.get("/fake/path", DirContext.DEFAULT, DirectoryFactory.LOCK_TYPE_SINGLE); + assertNotNull("ByteBuffersDirectoryFactory should create ByteBuffersDirectory even if the path doesn't lead " + + "to index directory on the file system", dir); + factory.release(dir); + factory.close(); + } + + public void testIndexRetrieve() throws Exception { + System.setProperty("solr.directoryFactory", "solr.ByteBuffersDirectoryFactory"); + initCore("solrconfig-minimal.xml","schema-minimal.xml"); + DirectoryFactory factory = h.getCore().getDirectoryFactory(); + assertTrue("Found: " + factory.getClass().getName(), factory instanceof ByteBuffersDirectoryFactory); + for (int i = 0 ; i < 5 ; ++i) { + assertU(adoc("id", "" + i, "a_s", "_" + i + "_")); + } + assertU(commit()); + assertQ(req("q", "a_s:_0_"), "//result[@numFound = '1']"); + deleteCore(); + } +} diff --git a/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java index 8714054eada..5c6bce79028 100644 --- a/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/CachingDirectoryFactoryTest.java @@ -49,8 +49,11 @@ public class CachingDirectoryFactoryTest extends SolrTestCaseJ4 { @Test public void stressTest() throws Exception { - final CachingDirectoryFactory df = new RAMDirectoryFactory(); - + doStressTest(new RAMDirectoryFactory()); + doStressTest(new ByteBuffersDirectoryFactory()); + } + + private void doStressTest(final CachingDirectoryFactory df) throws Exception { List threads = new ArrayList<>(); int threadCount = 11; for (int i = 0; i < threadCount; i++) { diff --git a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java index 869a4d2c2e2..dfe50bd26c6 100755 --- a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java @@ -67,45 +67,54 @@ public class DirectoryFactoryTest extends LuceneTestCase { } @Test - public void testGetDataHome() throws Exception { + public void testGetDataHomeRAMDirectory() throws Exception { + doTestGetDataHome(RAMDirectoryFactory.class); + } + + @Test + public void testGetDataHomeByteBuffersDirectory() throws Exception { + doTestGetDataHome(ByteBuffersDirectoryFactory.class); + } + + private void doTestGetDataHome(Class directoryFactoryClass) throws Exception { NodeConfig config = loadNodeConfig("/solr/solr-solrDataHome.xml"); CoreContainer cc = new CoreContainer(config); Properties cp = cc.getContainerProperties(); - RAMDirectoryFactory rdf = new RAMDirectoryFactory(); - rdf.initCoreContainer(cc); - rdf.init(new NamedList()); + DirectoryFactory df = directoryFactoryClass.newInstance(); + df.initCoreContainer(cc); + df.init(new NamedList()); // No solr.data.home property set. Absolute instanceDir - assertDataHome("/tmp/inst1/data", "/tmp/inst1", rdf, cc); + assertDataHome("/tmp/inst1/data", "/tmp/inst1", df, cc); // Simulate solr.data.home set in solrconfig.xml tag NamedList args = new NamedList(); args.add("solr.data.home", "/solrdata/"); - rdf.init(args); - assertDataHome("/solrdata/inst_dir/data", "inst_dir", rdf, cc); + df.init(args); + assertDataHome("/solrdata/inst_dir/data", "inst_dir", df, cc); // solr.data.home set with System property, and relative path System.setProperty("solr.data.home", "solrdata"); config = loadNodeConfig("/solr/solr-solrDataHome.xml"); cc = new CoreContainer(config); - rdf = new RAMDirectoryFactory(); - rdf.initCoreContainer(cc); - rdf.init(new NamedList()); - assertDataHome(solrHome.resolve("solrdata/inst_dir/data").toAbsolutePath().toString(), "inst_dir", rdf, cc); + df = directoryFactoryClass.newInstance(); + df.initCoreContainer(cc); + df.init(new NamedList()); + assertDataHome(solrHome.resolve("solrdata/inst_dir/data").toAbsolutePath().toString(), "inst_dir", df, cc); // Test parsing last component of instanceDir, and using custom dataDir - assertDataHome(solrHome.resolve("solrdata/myinst/mydata").toAbsolutePath().toString(), "/path/to/myinst", rdf, cc, "dataDir", "mydata"); + assertDataHome(solrHome.resolve("solrdata/myinst/mydata").toAbsolutePath().toString(), "/path/to/myinst", df, cc, "dataDir", "mydata"); // solr.data.home set but also solrDataHome set in solr.xml, which should override the former System.setProperty("test.solr.data.home", "/foo"); config = loadNodeConfig("/solr/solr-solrDataHome.xml"); cc = new CoreContainer(config); - rdf = new RAMDirectoryFactory(); - rdf.initCoreContainer(cc); - rdf.init(new NamedList()); - assertDataHome("/foo/inst_dir/data", "inst_dir", rdf, cc); + df = directoryFactoryClass.newInstance(); + df.initCoreContainer(cc); + df.init(new NamedList()); + assertDataHome("/foo/inst_dir/data", "inst_dir", df, cc); } - private void assertDataHome(String expected, String instanceDir, RAMDirectoryFactory rdf, CoreContainer cc, String... properties) throws IOException { - String dataHome = rdf.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties)); + private void assertDataHome(String expected, String instanceDir, DirectoryFactory df, CoreContainer cc, String... properties) throws IOException { + String dataHome = df.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties)); assertEquals(Paths.get(expected).toAbsolutePath(), Paths.get(dataHome).toAbsolutePath()); }