mirror of https://github.com/apache/lucene.git
SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown.
This commit is contained in:
parent
275a259b1f
commit
9aeb745a7d
|
@ -58,6 +58,8 @@ Bug Fixes
|
||||||
* SOLR-8857: HdfsUpdateLog does not use configured or new default number of version buckets and is
|
* SOLR-8857: HdfsUpdateLog does not use configured or new default number of version buckets and is
|
||||||
hard coded to 256. (Mark Miller, yonik, Gregory Chanan)
|
hard coded to 256. (Mark Miller, yonik, Gregory Chanan)
|
||||||
|
|
||||||
|
* SOLR-8855: The HDFS BlockDirectory should not clean up it's cache on shutdown. (Mark Miller)
|
||||||
|
|
||||||
Optimizations
|
Optimizations
|
||||||
----------------------
|
----------------------
|
||||||
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
|
* SOLR-8722: Don't force a full ZkStateReader refresh on every Overseer operation.
|
||||||
|
|
|
@ -25,27 +25,17 @@ import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.IdentityHashMap;
|
import java.util.IdentityHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
import org.apache.lucene.store.AlreadyClosedException;
|
import org.apache.lucene.store.AlreadyClosedException;
|
||||||
import org.apache.lucene.store.Directory;
|
import org.apache.lucene.store.Directory;
|
||||||
import org.apache.lucene.store.IOContext.Context;
|
|
||||||
import org.apache.lucene.store.LockFactory;
|
import org.apache.lucene.store.LockFactory;
|
||||||
import org.apache.lucene.store.NRTCachingDirectory;
|
|
||||||
import org.apache.lucene.store.NativeFSLockFactory;
|
|
||||||
import org.apache.lucene.store.NoLockFactory;
|
|
||||||
import org.apache.lucene.store.SimpleFSLockFactory;
|
|
||||||
import org.apache.lucene.store.SingleInstanceLockFactory;
|
|
||||||
import org.apache.lucene.util.IOUtils;
|
import org.apache.lucene.util.IOUtils;
|
||||||
import org.apache.solr.common.SolrException;
|
import org.apache.solr.common.SolrException;
|
||||||
import org.apache.solr.common.SolrException.ErrorCode;
|
import org.apache.solr.common.SolrException.ErrorCode;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.common.util.ObjectReleaseTracker;
|
import org.apache.solr.common.util.ObjectReleaseTracker;
|
||||||
import org.apache.solr.store.blockcache.BlockDirectory;
|
|
||||||
import org.apache.solr.store.hdfs.HdfsDirectory;
|
|
||||||
import org.apache.solr.store.hdfs.HdfsLockFactory;
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -302,9 +292,15 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void close(CacheValue val) {
|
private void close(CacheValue val) {
|
||||||
|
log.info("Closing directory, CoreContainer#isShutdown={}", coreContainer != null ? coreContainer.isShutDown() : "null");
|
||||||
try {
|
try {
|
||||||
log.info("Closing directory: " + val.path);
|
if (coreContainer != null && coreContainer.isShutDown() && val.directory instanceof ShutdownAwareDirectory) {
|
||||||
val.directory.close();
|
log.info("Closing directory on shutdown: " + val.path);
|
||||||
|
((ShutdownAwareDirectory) val.directory).closeOnShutdown();
|
||||||
|
} else {
|
||||||
|
log.info("Closing directory: " + val.path);
|
||||||
|
val.directory.close();
|
||||||
|
}
|
||||||
assert ObjectReleaseTracker.release(val.directory);
|
assert ObjectReleaseTracker.release(val.directory);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
SolrException.log(log, "Error closing directory", e);
|
SolrException.log(log, "Error closing directory", e);
|
||||||
|
|
|
@ -60,6 +60,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
|
||||||
public final static String LOCK_TYPE_SINGLE = "single";
|
public final static String LOCK_TYPE_SINGLE = "single";
|
||||||
public final static String LOCK_TYPE_NONE = "none";
|
public final static String LOCK_TYPE_NONE = "none";
|
||||||
public final static String LOCK_TYPE_HDFS = "hdfs";
|
public final static String LOCK_TYPE_HDFS = "hdfs";
|
||||||
|
|
||||||
|
protected volatile CoreContainer coreContainer;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Indicates a Directory will no longer be used, and when its ref count
|
* Indicates a Directory will no longer be used, and when its ref count
|
||||||
|
@ -325,4 +327,8 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
|
||||||
FileUtils.deleteDirectory(dirToRm);
|
FileUtils.deleteDirectory(dirToRm);
|
||||||
return !dirToRm.isDirectory();
|
return !dirToRm.isDirectory();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void initCoreContainer(CoreContainer cc) {
|
||||||
|
this.coreContainer = cc;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -136,6 +136,7 @@ public class HdfsDirectoryFactory extends CachingDirectoryFactory implements Sol
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void init(NamedList args) {
|
public void init(NamedList args) {
|
||||||
|
super.init(args);
|
||||||
params = SolrParams.toSolrParams(args);
|
params = SolrParams.toSolrParams(args);
|
||||||
this.hdfsDataDir = getConfig(HDFS_HOME, null);
|
this.hdfsDataDir = getConfig(HDFS_HOME, null);
|
||||||
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
|
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.core;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.lucene.store.Directory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link CachingDirectoryFactory} will call this method
|
||||||
|
* rather than {@link Directory#close()} on shutdown if
|
||||||
|
* a Directory implements this interface.
|
||||||
|
*/
|
||||||
|
public interface ShutdownAwareDirectory {
|
||||||
|
public void closeOnShutdown() throws IOException;
|
||||||
|
}
|
|
@ -125,6 +125,7 @@ import static org.apache.solr.common.params.CommonParams.PATH;
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public final class SolrCore implements SolrInfoMBean, Closeable {
|
public final class SolrCore implements SolrInfoMBean, Closeable {
|
||||||
|
|
||||||
public static final String version="1.0";
|
public static final String version="1.0";
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
@ -480,10 +481,13 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
|
||||||
if (info != null) {
|
if (info != null) {
|
||||||
log.info(info.className);
|
log.info(info.className);
|
||||||
dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class);
|
dirFactory = getResourceLoader().newInstance(info.className, DirectoryFactory.class);
|
||||||
|
// allow DirectoryFactory instances to access the CoreContainer
|
||||||
|
dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer());
|
||||||
dirFactory.init(info.initArgs);
|
dirFactory.init(info.initArgs);
|
||||||
} else {
|
} else {
|
||||||
log.info("solr.NRTCachingDirectoryFactory");
|
log.info("solr.NRTCachingDirectoryFactory");
|
||||||
dirFactory = new NRTCachingDirectoryFactory();
|
dirFactory = new NRTCachingDirectoryFactory();
|
||||||
|
dirFactory.initCoreContainer(getCoreDescriptor().getCoreContainer());
|
||||||
}
|
}
|
||||||
return dirFactory;
|
return dirFactory;
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.lucene.store.FilterDirectory;
|
||||||
import org.apache.lucene.store.IOContext;
|
import org.apache.lucene.store.IOContext;
|
||||||
import org.apache.lucene.store.IndexInput;
|
import org.apache.lucene.store.IndexInput;
|
||||||
import org.apache.lucene.store.IndexOutput;
|
import org.apache.lucene.store.IndexOutput;
|
||||||
|
import org.apache.solr.core.ShutdownAwareDirectory;
|
||||||
import org.apache.solr.store.hdfs.HdfsDirectory;
|
import org.apache.solr.store.hdfs.HdfsDirectory;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
|
||||||
/**
|
/**
|
||||||
* @lucene.experimental
|
* @lucene.experimental
|
||||||
*/
|
*/
|
||||||
public class BlockDirectory extends FilterDirectory {
|
public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory {
|
||||||
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
|
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
|
||||||
|
@ -223,6 +224,13 @@ public class BlockDirectory extends FilterDirectory {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void closeOnShutdown() throws IOException {
|
||||||
|
LOG.info("BlockDirectory closing on shutdown");
|
||||||
|
// we are shutting down, no need to clean up cache
|
||||||
|
super.close();
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
try {
|
try {
|
||||||
|
|
Loading…
Reference in New Issue