Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4916: Add support to write and read Solr index files and transaction log files to and from HDFS. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1497072 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-06-26 20:18:33 +00:00
parent 58dec03041
commit b9e1537a7e
135 changed files with 11302 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
# hdfs
/solr/example/hdfs
*.jar
# .

2
build.xml
View File

@ -205,7 +205,7 @@
<!-- TODO: find a better way to exclude duplicate JAR files & fix the servlet-api mess! -->
<pathconvert property="eclipse.fileset.libs" pathsep="|" dirsep="/">
<fileset dir="${basedir}/lucene" includes="**/lib/*.jar" excludes="**/*servlet-api*.jar, analysis/uima/**, tools/**, build/**"/>
<fileset dir="${basedir}/solr" includes="**/lib/*.jar" excludes="core/lib/*servlet-api*.jar, contrib/analysis-extras/**, test-framework/**, build/**, dist/**, package/**" />
<fileset dir="${basedir}/solr" includes="**/lib/*.jar" excludes="core/lib/*servlet-api*.jar, contrib/analysis-extras/**, test-framework/lib/junit*, test-framework/lib/ant*, test-framework/lib/randomizedtesting*, build/**, dist/**, package/**" />
<map from="${basedir}/" to=""/>
</pathconvert>
<xslt in="${ant.file}" out=".classpath" style="dev-tools/eclipse/dot.classpath.xsl" force="true">

2
lucene/common-build.xml
View File

@ -354,7 +354,7 @@
<target name="resolve" depends="ivy-availability-check,ivy-configure">
<!-- todo, make this a property or something.
only special cases need bundles -->
<ivy:retrieve type="jar,bundle" log="download-only"
<ivy:retrieve type="jar,bundle,tests" log="download-only"
conf="${ivy.default.configuration}" sync="${ivy.sync}"/>
</target>

2
lucene/tools/custom-tasks.xml
View File

@ -43,7 +43,7 @@
<!-- Typical version patterns. -->
<replaceregex pattern="\.rc[0-9]+" replace="" flags="gi" />
<replaceregex pattern="\-(r)?([0-9\-\_\.])+(b(eta)?([0-9\-\.])*)?$" replace="" flags="gi" />
<replaceregex pattern="\-(r)?([0-9\-\_\.])+((b(eta)?)|((a(lpha)?))([0-9\-\.])*)?(\-tests)?$" replace="" flags="gi" />
<!-- git hashcode pattern: its always 40 chars right? -->
<replaceregex pattern="\-[a-z0-9]{40,40}$" replace="" flags="gi" />

4
lucene/tools/junit4/tests.policy
View File

@ -59,6 +59,10 @@ grant {
permission javax.management.MBeanPermission "*", "*";
permission javax.management.MBeanServerPermission "*";
permission javax.management.MBeanTrustPermission "*";
permission javax.security.auth.AuthPermission "*";
permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
// TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug!
permission java.security.SecurityPermission "putProviderProperty.BC";

3
solr/CHANGES.txt
View File

@ -118,6 +118,9 @@ New Features
* SOLR-4921: Admin UI now supports adding documents to Solr (gsingers, steffkes)
* SOLR-4916: Add support to write and read Solr index files and transaction log
files to and from HDFS. (phunt, Mark Miller, Greg Chanan)
Bug Fixes
----------------------

4
solr/NOTICE.txt
View File

@ -52,6 +52,7 @@ including, but not limited to:
- Apache Jakarta Regexp
- Apache Commons
- Apache Xerces
- Apache Blur
ICU4J, (under analysis/icu) is licensed under an MIT styles license
and Copyright (c) 1995-2008 International Business Machines Corporation and others
@ -72,6 +73,9 @@ http://bitbucket.org/jpbarrette/moman/overview/
The class org.apache.lucene.util.WeakIdentityMap was derived from
the Apache CXF project and is Apache License 2.0.
The HdfsDirectory and BlockDirectory were derived from
the Apache Blur incubating project and are Apache License 2.0.
The Google Code Prettify is Apache License 2.0.
See http://code.google.com/p/google-code-prettify/

11
solr/build.xml
View File

@ -40,7 +40,7 @@
<!-- ========================================================================= -->
<target name="example" description="Creates a runnable example configuration."
depends="dist-contrib,dist-war">
depends="dist-contrib,dist-war,setup-alt-examples">
<copy file="${dist}/${fullnamever}.war"
tofile="${example}/webapps/${ant.project.name}.war"/>
<jar destfile="${example}/exampledocs/post.jar"
@ -56,6 +56,15 @@
<echo>See ${example}/README.txt for how to run the Solr example configuration.</echo>
</target>
<target name="setup-alt-examples">
<copy todir="${example}/hdfs" overwrite="true">
<fileset dir="${example}/solr"/>
</copy>
<copy todir="${example}/hdfs/collection1/conf" overwrite="true">
<fileset dir="${example}/alt-configs/hdfs"/>
</copy>
</target>
<target name="run-example" depends="example"
description="Run Solr interactively, via Jetty. -Dexample.debug=true to enable JVM debugger">
<property name="example.solr.home" location="example/solr"/>

6
solr/common-build.xml
View File

@ -96,6 +96,12 @@
<path id="solr.test.base.classpath">
<pathelement path="${common-solr.dir}/build/solr-test-framework/classes/java"/>
<fileset dir="${common-solr.dir}/test-framework/lib">
<include name="*.jar"/>
<exclude name="junit-*.jar" />
<exclude name="randomizedtesting-runner-*.jar" />
<exclude name="ant*.jar" />
</fileset>
<pathelement path="${build.dir}/test-files"/>
<path refid="test.base.classpath"/>
</path>

13
solr/core/ivy.xml
View File

@ -16,6 +16,9 @@
specific language governing permissions and limitations
under the License.
-->
<!DOCTYPE ivy-module [
<!ENTITY hadoop.version "2.0.5-alpha">
]>
<ivy-module version="2.0">
<info organisation="org.apache.solr" module="core"/>
@ -32,6 +35,14 @@
<dependency org="javax.servlet" name="javax.servlet-api" rev="3.0.1" transitive="false"/>
<dependency org="org.restlet.jee" name="org.restlet" rev="2.1.1" transitive="false"/>
<dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="2.1.1" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
<dependency org="org.apache.hadoop" name="hadoop-common" rev="&hadoop.version;" transitive="false"/>
<dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="&hadoop.version;" transitive="false"/>
<dependency org="org.apache.hadoop" name="hadoop-annotations" rev="&hadoop.version;" transitive="false"/>
<dependency org="org.apache.hadoop" name="hadoop-auth" rev="&hadoop.version;" transitive="false"/>
<dependency org="commons-configuration" name="commons-configuration" rev="1.6" transitive="false"/>
<dependency org="com.google.protobuf" name="protobuf-java" rev="2.4.0a" transitive="false"/>
<dependency org="com.googlecode.concurrentlinkedhashmap" name="concurrentlinkedhashmap-lru" rev="1.2" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>

26
solr/core/src/java/org/apache/solr/SolrLogFormatter.java
View File

@ -17,20 +17,24 @@ package org.apache.solr;
* limitations under the License.
*/
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.WeakHashMap;
import java.util.logging.*;
import java.util.logging.ConsoleHandler;
import java.util.logging.Formatter;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.slf4j.LoggerFactory;
public class SolrLogFormatter extends Formatter {
@ -259,7 +263,7 @@ sb.append("(group_name=").append(tg.getName()).append(")");
private Map<String,Object> getReplicaProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
Replica replica = zkController.getClusterState().getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()));
Replica replica = zkController.getClusterState().getReplica(collection, core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
if(replica!=null) {
return replica.getProperties();
}

10
solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
View File

@ -85,6 +85,7 @@ public class JettySolrRunner {
private String shards;
private String dataDir;
private String solrUlogDir;
private volatile boolean startedBefore = false;
@ -359,6 +360,9 @@ public class JettySolrRunner {
if( dataDir != null) {
System.setProperty("solr.data.dir", dataDir);
}
if( solrUlogDir != null) {
System.setProperty("solr.ulog.dir", solrUlogDir);
}
if(shards != null) {
System.setProperty("shard", shards);
}
@ -382,6 +386,8 @@ public class JettySolrRunner {
System.clearProperty("shard");
System.clearProperty("solr.data.dir");
System.clearProperty("coreNodeName");
System.clearProperty("solr.ulog.dir");
}
public void stop() throws Exception {
@ -485,6 +491,10 @@ public class JettySolrRunner {
public void setDataDir(String dataDir) {
this.dataDir = dataDir;
}
public void setUlogDir(String ulogDir) {
this.solrUlogDir = ulogDir;
}
public void setCoreNodeName(String coreNodeName) {
this.coreNodeName = coreNodeName;

26
solr/core/src/java/org/apache/solr/cloud/AssignShard.java → solr/core/src/java/org/apache/solr/cloud/Assign.java
View File

@ -23,12 +23,36 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
public class AssignShard {
public class Assign {
private static Pattern COUNT = Pattern.compile("core_node(\\d+)");
public static String assignNode(String collection, ClusterState state) {
Map<String, Slice> sliceMap = state.getSlicesMap(collection);
if (sliceMap == null) {
return "core_node1";
}
int max = 0;
for (Slice slice : sliceMap.values()) {
for (Replica replica : slice.getReplicas()) {
Matcher m = COUNT.matcher(replica.getName());
if (m.matches()) {
max = Math.max(max, Integer.parseInt(m.group(1)));
}
}
}
return "core_node" + (max + 1);
}
/**
* Assign a new unique id up to slices count - then add replicas evenly.
*

4
solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
View File

@ -44,6 +44,10 @@ public class CloudDescriptor {
public boolean isLeader() {
return isLeader;
}
public void setLeader(boolean isLeader) {
this.isLeader = isLeader;
}
public void setShardId(String shardId) {
this.shardId = shardId;

29
solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
View File

@ -47,9 +47,9 @@ public abstract class ElectionContext {
String leaderSeqPath;
private SolrZkClient zkClient;
public ElectionContext(final String shardZkNodeName,
public ElectionContext(final String coreNodeName,
final String electionPath, final String leaderPath, final ZkNodeProps leaderProps, final SolrZkClient zkClient) {
this.id = shardZkNodeName;
this.id = coreNodeName;
this.electionPath = electionPath;
this.leaderPath = leaderPath;
this.leaderProps = leaderProps;
@ -78,8 +78,8 @@ class ShardLeaderElectionContextBase extends ElectionContext {
protected LeaderElector leaderElector;
public ShardLeaderElectionContextBase(LeaderElector leaderElector, final String shardId,
final String collection, final String shardZkNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
super(shardZkNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
final String collection, final String coreNodeName, ZkNodeProps props, ZkStateReader zkStateReader) {
super(coreNodeName, ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/leader_elect/"
+ shardId, ZkStateReader.getShardLeadersPath(collection, shardId),
props, zkStateReader.getZkClient());
this.leaderElector = leaderElector;
@ -95,7 +95,7 @@ class ShardLeaderElectionContextBase extends ElectionContext {
zkClient.makePath(leaderPath, ZkStateReader.toJSON(leaderProps),
CreateMode.EPHEMERAL, true);
assert shardId != null;
ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, "leader",
ZkNodeProps m = ZkNodeProps.fromKeyVals(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
collection, ZkStateReader.BASE_URL_PROP, leaderProps.getProperties()
.get(ZkStateReader.BASE_URL_PROP), ZkStateReader.CORE_NAME_PROP,
@ -119,8 +119,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
public ShardLeaderElectionContext(LeaderElector leaderElector,
final String shardId, final String collection,
final String shardZkNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
super(leaderElector, shardId, collection, shardZkNodeName, props,
final String coreNodeName, ZkNodeProps props, ZkController zkController, CoreContainer cc) {
super(leaderElector, shardId, collection, coreNodeName, props,
zkController.getZkStateReader());
this.zkController = zkController;
this.cc = cc;
@ -138,12 +138,12 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
@Override
void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
InterruptedException, IOException {
log.info("Running the leader process.");
log.info("Running the leader process for shard " + shardId);
String coreName = leaderProps.getStr(ZkStateReader.CORE_NAME_PROP);
// clear the leader in clusterstate
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "leader",
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, ZkStateReader.LEADER_PROP,
ZkStateReader.SHARD_ID_PROP, shardId, ZkStateReader.COLLECTION_PROP,
collection);
Overseer.getInQueue(zkClient).offer(ZkStateReader.toJSON(m));
@ -243,8 +243,8 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
}
log.info("I am the new leader: "
+ ZkCoreNodeProps.getCoreUrl(leaderProps));
core.getCoreDescriptor().getCloudDescriptor().isLeader = true;
+ ZkCoreNodeProps.getCoreUrl(leaderProps) + " " + shardId);
core.getCoreDescriptor().getCloudDescriptor().setLeader(true);
} finally {
if (core != null) {
core.close();
@ -254,16 +254,17 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
try {
super.runLeaderProcess(weAreReplacement);
} catch (Throwable t) {
SolrException.log(log, "There was a problem trying to register as the leader", t);
cancelElection();
try {
core = cc.getCore(coreName);
if (core == null) {
cancelElection();
throw new SolrException(ErrorCode.SERVER_ERROR,
"Fatal Error, SolrCore not found:" + coreName + " in "
+ cc.getCoreNames());
}
core.getCoreDescriptor().getCloudDescriptor().isLeader = false;
core.getCoreDescriptor().getCloudDescriptor().setLeader(false);
// we could not publish ourselves as leader - rejoin election
rejoinLeaderElection(leaderSeqPath, core);
@ -332,7 +333,7 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
return;
} else {
if (cnt % 40 == 0) {
log.info("Waiting until we see more replicas up: total="
log.info("Waiting until we see more replicas up for shard " + shardId + ": total="
+ slices.getReplicasMap().size() + " found=" + found
+ " timeoutin=" + (timeoutAt - System.currentTimeMillis()));
}

47
solr/core/src/java/org/apache/solr/cloud/Overseer.java
View File

@ -50,15 +50,16 @@ import org.slf4j.LoggerFactory;
*/
public class Overseer {
public static final String QUEUE_OPERATION = "operation";
public static final String DELETECORE = "deletecore";
public static final String REMOVECOLLECTION = "removecollection";
private static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
private static Logger log = LoggerFactory.getLogger(Overseer.class);
private class ClusterStateUpdater implements Runnable, ClosableThread {
private static final String DELETECORE = "deletecore";
private final ZkStateReader reader;
private final SolrZkClient zkClient;
private final String myId;
@ -267,8 +268,14 @@ public class Overseer {
final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
String coreNodeName = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
if (coreNodeName == null) {
// it must be the default then
coreNodeName = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
coreNodeName = getAssignedCoreNodeName(state, message);
if (coreNodeName != null) {
log.info("node=" + coreNodeName + " is already registered");
} else {
// if coreNodeName is null, auto assign one
coreNodeName = Assign.assignNode(collection, state);
}
message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
}
Integer numShards = message.getStr(ZkStateReader.NUM_SHARDS_PROP)!=null?Integer.parseInt(message.getStr(ZkStateReader.NUM_SHARDS_PROP)):null;
log.info("Update state numShards={} message={}", numShards, message);
@ -281,7 +288,6 @@ public class Overseer {
// use the provided non null shardId
String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);
if (sliceName == null) {
//String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);
//get shardId from ClusterState
sliceName = getAssignedId(state, coreNodeName, message);
if (sliceName != null) {
@ -295,8 +301,8 @@ public class Overseer {
numShards = state.getCollectionStates().get(collection).getSlices().size();
log.info("Collection already exists with " + ZkStateReader.NUM_SHARDS_PROP + "=" + numShards);
}
sliceName = AssignShard.assignShard(collection, state, numShards);
log.info("Assigning new node to shard=" + sliceName);
sliceName = Assign.assignShard(collection, state, numShards);
log.info("Assigning new node to shard shard=" + sliceName);
}
Slice slice = state.getSlice(collection, sliceName);
@ -320,8 +326,11 @@ public class Overseer {
}
}
// we don't put num_shards in the clusterstate
// we don't put these in the clusterstate
replicaProps.remove(ZkStateReader.NUM_SHARDS_PROP);
replicaProps.remove(ZkStateReader.CORE_NODE_NAME_PROP);
replicaProps.remove(ZkStateReader.SHARD_ID_PROP);
replicaProps.remove(ZkStateReader.COLLECTION_PROP);
replicaProps.remove(QUEUE_OPERATION);
// remove any props with null values
@ -417,6 +426,26 @@ public class Overseer {
return null;
}
private String getAssignedCoreNodeName(ClusterState state, ZkNodeProps message) {
Collection<Slice> slices = state.getSlices(message.getStr(ZkStateReader.COLLECTION_PROP));
if (slices != null) {
for (Slice slice : slices) {
for (Replica replica : slice.getReplicas()) {
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String msgBaseUrl = message.getStr(ZkStateReader.BASE_URL_PROP);
String msgCore = message.getStr(ZkStateReader.CORE_NAME_PROP);
if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
return replica.getName();
}
}
}
}
return null;
}
private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
// System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
// System.out.println("Updating slice:" + slice);
@ -526,10 +555,6 @@ public class Overseer {
private ClusterState removeCore(final ClusterState clusterState, ZkNodeProps message) {
String cnn = message.getStr(ZkStateReader.CORE_NODE_NAME_PROP);
if (cnn == null) {
// it must be the default then
cnn = message.getStr(ZkStateReader.NODE_NAME_PROP) + "_" + message.getStr(ZkStateReader.CORE_NAME_PROP);
}
final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);

35
solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
View File

@ -400,10 +400,11 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
// wait for parent leader to acknowledge the sub-shard core
log.info("Asking parent leader to wait for: " + subShardName + " to be alive on: " + nodeName);
String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(nodeName), subShardName);
CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState();
cmd.setCoreName(subShardName);
cmd.setNodeName(nodeName);
cmd.setCoreNodeName(nodeName + "_" + subShardName);
cmd.setCoreNodeName(coreNodeName);
cmd.setState(ZkStateReader.ACTIVE);
cmd.setCheckLive(true);
cmd.setOnlyIfLeader(true);
@ -506,12 +507,13 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
sendShardRequest(subShardNodeName, params);
String coreNodeName = waitForCoreNodeName(collection, zkStateReader.getZkClient().getBaseUrlForNodeName(subShardNodeName), shardName);
// wait for the replicas to be seen as active on sub shard leader
log.info("Asking sub shard leader to wait for: " + shardName + " to be alive on: " + subShardNodeName);
CoreAdminRequest.WaitForState cmd = new CoreAdminRequest.WaitForState();
cmd.setCoreName(subShardNames.get(i-1));
cmd.setNodeName(subShardNodeName);
cmd.setCoreNodeName(subShardNodeName + "_" + shardName);
cmd.setCoreNodeName(coreNodeName);
cmd.setState(ZkStateReader.ACTIVE);
cmd.setCheckLive(true);
cmd.setOnlyIfLeader(true);
@ -545,6 +547,35 @@ public class OverseerCollectionProcessor implements Runnable, ClosableThread {
throw new SolrException(ErrorCode.SERVER_ERROR, null, e);
}
}
private String waitForCoreNodeName(DocCollection collection, String msgBaseUrl, String msgCore) {
int retryCount = 320;
while (retryCount-- > 0) {
Map<String,Slice> slicesMap = zkStateReader.getClusterState()
.getSlicesMap(collection.getName());
if (slicesMap != null) {
for (Slice slice : slicesMap.values()) {
for (Replica replica : slice.getReplicas()) {
// TODO: for really large clusters, we could 'index' on this
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
return replica.getName();
}
}
}
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName");
}
private void collectShardResponses(NamedList results, boolean abortOnError, String msgOnError) {
ShardResponse srsp;

6
solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
View File

@ -91,7 +91,7 @@ public class RecoveryStrategy extends Thread implements ClosableThread {
zkController = cc.getZkController();
zkStateReader = zkController.getZkStateReader();
baseUrl = zkController.getBaseUrl();
coreZkNodeName = zkController.getCoreNodeName(cd);
coreZkNodeName = cd.getCloudDescriptor().getCoreNodeName();
}
public void setRecoveringAfterStartup(boolean recoveringAfterStartup) {
@ -325,10 +325,10 @@ public class RecoveryStrategy extends Thread implements ClosableThread {
String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
boolean isLeader = leaderUrl.equals(ourUrl);
if (isLeader && !cloudDesc.isLeader) {
if (isLeader && !cloudDesc.isLeader()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Cloud state still says we are leader.");
}
if (cloudDesc.isLeader) {
if (cloudDesc.isLeader()) {
// we are now the leader - no one else must have been suitable
log.warn("We have not yet recovered - but we are now the leader! core=" + coreName);
log.info("Finished recovery process. core=" + coreName);

5
solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
View File

@ -160,8 +160,7 @@ public class SyncStrategy {
private boolean syncWithReplicas(ZkController zkController, SolrCore core,
ZkNodeProps props, String collection, String shardId) {
List<ZkCoreNodeProps> nodes = zkController.getZkStateReader()
.getReplicaProps(collection, shardId,
zkController.getCoreNodeName(core.getCoreDescriptor()),
.getReplicaProps(collection, shardId,core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName(),
props.getStr(ZkStateReader.CORE_NAME_PROP));
if (nodes == null) {
@ -189,7 +188,7 @@ public class SyncStrategy {
List<ZkCoreNodeProps> nodes = zkController
.getZkStateReader()
.getReplicaProps(collection, shardId,
zkController.getCoreNodeName(cd),
cd.getCloudDescriptor().getCoreNodeName(),
leaderProps.getStr(ZkStateReader.CORE_NAME_PROP));
if (nodes == null) {
log.info(ZkCoreNodeProps.getCoreUrl(leaderProps) + " has no replicas");

135
solr/core/src/java/org/apache/solr/cloud/ZkController.java
View File

@ -132,6 +132,8 @@ public final class ZkController {
protected volatile Overseer overseer;
private String leaderVoteWait;
private boolean genericCoreNodeNames;
private int clientTimeout;
@ -140,11 +142,11 @@ public final class ZkController {
private UpdateShardHandler updateShardHandler;
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
String localHostContext, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
String localHostContext, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
TimeoutException, IOException {
if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
this.cc = cc;
this.genericCoreNodeNames = genericCoreNodeNames;
// be forgiving and strip this off leading/trailing slashes
// this allows us to support users specifying hostContext="/" in
// solr.xml to indicate the root context, instead of hostContext=""
@ -254,9 +256,9 @@ public final class ZkController {
// before registering as live, make sure everyone is in a
// down state
for (CoreDescriptor descriptor : descriptors) {
final String coreZkNodeName = getCoreNodeName(descriptor);
final String coreZkNodeName = descriptor.getCloudDescriptor().getCoreNodeName();
try {
descriptor.getCloudDescriptor().isLeader = false;
descriptor.getCloudDescriptor().setLeader(false);
publish(descriptor, ZkStateReader.DOWN, updateLastPublished);
} catch (Exception e) {
if (isClosed) {
@ -323,7 +325,7 @@ public final class ZkController {
.getCurrentDescriptors();
if (descriptors != null) {
for (CoreDescriptor descriptor : descriptors) {
descriptor.getCloudDescriptor().isLeader = false;
descriptor.getCloudDescriptor().setLeader(false);
}
}
}
@ -544,7 +546,6 @@ public final class ZkController {
if (replica.getNodeName().equals(getNodeName())
&& !(replica.getStr(ZkStateReader.STATE_PROP)
.equals(ZkStateReader.DOWN))) {
assert replica.getStr(ZkStateReader.SHARD_ID_PROP) != null;
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state",
ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
ZkStateReader.BASE_URL_PROP, getBaseUrl(),
@ -555,8 +556,7 @@ public final class ZkController {
ZkStateReader.NODE_NAME_PROP, getNodeName(),
ZkStateReader.SHARD_ID_PROP,
replica.getStr(ZkStateReader.SHARD_ID_PROP),
ZkStateReader.COLLECTION_PROP,
replica.getStr(ZkStateReader.COLLECTION_PROP),
ZkStateReader.COLLECTION_PROP, collectionName,
ZkStateReader.CORE_NODE_NAME_PROP, replica.getName());
updatedNodes.add(replica.getStr(ZkStateReader.CORE_NAME_PROP));
overseerJobQueue.offer(ZkStateReader.toJSON(m));
@ -735,7 +735,8 @@ public final class ZkController {
final CloudDescriptor cloudDesc = desc.getCloudDescriptor();
final String collection = cloudDesc.getCollectionName();
final String coreZkNodeName = getCoreNodeName(desc);
final String coreZkNodeName = desc.getCloudDescriptor().getCoreNodeName();
assert coreZkNodeName != null : "we should have a coreNodeName by now";
String shardId = cloudDesc.getShardId();
@ -923,16 +924,16 @@ public final class ZkController {
props.put(ZkStateReader.CORE_NAME_PROP, cd.getName());
props.put(ZkStateReader.NODE_NAME_PROP, getNodeName());
final String coreZkNodeName = getCoreNodeName(cd);
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
ZkNodeProps ourProps = new ZkNodeProps(props);
String collection = cd.getCloudDescriptor()
.getCollectionName();
ElectionContext context = new ShardLeaderElectionContext(leaderElector, shardId,
collection, coreZkNodeName, ourProps, this, cc);
collection, coreNodeName, ourProps, this, cc);
leaderElector.setup(context);
electionContexts.put(coreZkNodeName, context);
electionContexts.put(coreNodeName, context);
leaderElector.joinElection(context, false);
}
@ -1017,7 +1018,7 @@ public final class ZkController {
final CloudDescriptor cloudDesc = desc.getCloudDescriptor();
final String shardId = state.getShardId(coreNodeName);
final String shardId = state.getShardId(getBaseUrl(), desc.getName());
if (shardId != null) {
cloudDesc.setShardId(shardId);
@ -1028,16 +1029,21 @@ public final class ZkController {
public void unregister(String coreName, CoreDescriptor cd)
throws InterruptedException, KeeperException {
final String zkNodeName = getCoreNodeName(cd);
ElectionContext context = electionContexts.remove(zkNodeName);
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
ElectionContext context = electionContexts.remove(coreNodeName);
assert context != null : coreNodeName;
if (context != null) {
context.cancelElection();
}
CloudDescriptor cloudDescriptor = cd.getCloudDescriptor();
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
"deletecore", ZkStateReader.CORE_NAME_PROP, coreName,
Overseer.DELETECORE, ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.NODE_NAME_PROP, getNodeName(),
ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor().getCollectionName());
ZkStateReader.COLLECTION_PROP, cloudDescriptor.getCollectionName(),
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
overseerJobQueue.offer(ZkStateReader.toJSON(m));
}
@ -1206,13 +1212,60 @@ public final class ZkController {
return zkStateReader;
}
private String doGetShardIdProcess(String coreName, CoreDescriptor descriptor) {
final String coreNodeName = getCoreNodeName(descriptor);
private void doGetShardIdAndNodeNameProcess(CoreDescriptor cd) {
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
if (coreNodeName != null) {
waitForShardId(cd);
} else {
// if no explicit coreNodeName, we want to match by base url and core name
waitForCoreNodeName(cd);
waitForShardId(cd);
}
}
private void waitForCoreNodeName(CoreDescriptor descriptor) {
int retryCount = 320;
log.info("look for our core node name");
while (retryCount-- > 0) {
Map<String,Slice> slicesMap = zkStateReader.getClusterState()
.getSlicesMap(descriptor.getCloudDescriptor().getCollectionName());
if (slicesMap != null) {
for (Slice slice : slicesMap.values()) {
for (Replica replica : slice.getReplicas()) {
// TODO: for really large clusters, we could 'index' on this
String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String msgBaseUrl = getBaseUrl();
String msgCore = descriptor.getName();
if (baseUrl.equals(msgBaseUrl) && core.equals(msgCore)) {
descriptor.getCloudDescriptor()
.setCoreNodeName(replica.getName());
return;
}
}
}
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
private void waitForShardId(CoreDescriptor cd) {
log.info("waiting to find shard id in clusterstate for " + cd.getName());
int retryCount = 320;
while (retryCount-- > 0) {
final String shardId = zkStateReader.getClusterState().getShardId(coreNodeName);
final String shardId = zkStateReader.getClusterState().getShardId(getBaseUrl(), cd.getName());
if (shardId != null) {
return shardId;
cd.getCloudDescriptor().setShardId(shardId);
return;
}
try {
Thread.sleep(1000);
@ -1222,7 +1275,7 @@ public final class ZkController {
}
throw new SolrException(ErrorCode.SERVER_ERROR,
"Could not get shard_id for core: " + coreName + " coreNodeName:" + coreNodeName);
"Could not get shard id for core: " + cd.getName());
}
public static void uploadToZK(SolrZkClient zkClient, File dir, String zkPath) throws IOException, KeeperException, InterruptedException {
@ -1261,7 +1314,7 @@ public final class ZkController {
public String getCoreNodeName(CoreDescriptor descriptor){
String coreNodeName = descriptor.getCloudDescriptor().getCoreNodeName();
if (coreNodeName == null) {
if (coreNodeName == null && !genericCoreNodeNames) {
// it's the default
return getNodeName() + "_" + descriptor.getName();
}
@ -1277,27 +1330,33 @@ public final class ZkController {
downloadFromZK(zkClient, ZkController.CONFIGS_ZKNODE + "/" + configName, dir);
}
public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException {
// before becoming available, make sure we are not live and active
// this also gets us our assigned shard id if it was not specified
publish(cd, ZkStateReader.DOWN, false);
String coreNodeName = getCoreNodeName(cd);
public void preRegister(CoreDescriptor cd ) {
String coreNodeName = getCoreNodeName(cd);
// make sure the node name is set on the descriptor
if (cd.getCloudDescriptor().getCoreNodeName() == null) {
cd.getCloudDescriptor().setCoreNodeName(coreNodeName);
}
// before becoming available, make sure we are not live and active
// this also gets us our assigned shard id if it was not specified
try {
publish(cd, ZkStateReader.DOWN, false);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
}
if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), coreNodeName)) {
String shardId;
shardId = doGetShardIdProcess(cd.getName(), cd);
cd.getCloudDescriptor().setShardId(shardId);
doGetShardIdAndNodeNameProcess(cd);
} else {
// still wait till we see us in local state
doGetShardIdProcess(cd.getName(), cd);
doGetShardIdAndNodeNameProcess(cd);
}
}
@ -1332,7 +1391,7 @@ public final class ZkController {
}
}
}
String leaderBaseUrl = leaderProps.getBaseUrl();
String leaderCoreName = leaderProps.getCoreName();
@ -1494,11 +1553,11 @@ public final class ZkController {
}
/**
* utilitiy method fro trimming and leading and/or trailing slashes from
* Utility method for trimming and leading and/or trailing slashes from
* it's input. May return the empty string. May return null if and only
* if the input is null.
*/
private static String trimLeadingAndTrailingSlashes(final String in) {
public static String trimLeadingAndTrailingSlashes(final String in) {
if (null == in) return in;
String out = in;

24
solr/core/src/java/org/apache/solr/core/CachingDirectoryFactory.java
View File

@ -32,6 +32,7 @@ import java.util.Set;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.NativeFSLockFactory;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RateLimitedDirectoryWrapper;
@ -40,6 +41,9 @@ import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.store.blockcache.BlockDirectory;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.apache.solr.store.hdfs.HdfsLockFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -498,6 +502,24 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
} else if ("single".equals(lockType)) {
if (!(dir.getLockFactory() instanceof SingleInstanceLockFactory)) dir
.setLockFactory(new SingleInstanceLockFactory());
} else if ("hdfs".equals(lockType)) {
Directory del = dir;
if (dir instanceof NRTCachingDirectory) {
del = ((NRTCachingDirectory) del).getDelegate();
}
if (del instanceof BlockDirectory) {
del = ((BlockDirectory) del).getDirectory();
}
if (!(del instanceof HdfsDirectory)) {
throw new SolrException(ErrorCode.FORBIDDEN, "Directory: "
+ del.getClass().getName()
+ ", but hdfs lock factory can only be used with HdfsDirectory");
}
dir.setLockFactory(new HdfsLockFactory(((HdfsDirectory)del).getHdfsDirPath(), ((HdfsDirectory)del).getConfiguration()));
} else if ("none".equals(lockType)) {
// Recipe for disaster
log.error("CONFIGURATION WARNING: locks are disabled on " + dir);
@ -519,7 +541,7 @@ public abstract class CachingDirectoryFactory extends DirectoryFactory {
return path;
}
private String stripTrailingSlash(String path) {
protected String stripTrailingSlash(String path) {
if (path.endsWith("/")) {
path = path.substring(0, path.length() - 1);
}

1
solr/core/src/java/org/apache/solr/core/ConfigSolr.java
View File

@ -63,6 +63,7 @@ public abstract class ConfigSolr {
SOLR_SHARDHANDLERFACTORY_SOCKETTIMEOUT,
SOLR_SHARESCHEMA,
SOLR_TRANSIENTCACHESIZE,
SOLR_GENERICCORENODENAMES,
SOLR_ZKCLIENTTIMEOUT,
SOLR_ZKHOST,

2
solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
View File

@ -77,6 +77,7 @@ public class ConfigSolrXml extends ConfigSolr {
failIfFound("solr/cores/@hostContext");
failIfFound("solr/cores/@hostPort");
failIfFound("solr/cores/@leaderVoteWait");
failIfFound("solr/cores/@genericCoreNodeNames");
failIfFound("solr/cores/@managementPath");
failIfFound("solr/cores/@shareSchema");
failIfFound("solr/cores/@transientCacheSize");
@ -118,6 +119,7 @@ public class ConfigSolrXml extends ConfigSolr {
propMap.put(CfgProp.SOLR_HOSTCONTEXT, doSub("solr/solrcloud/str[@name='hostContext']"));
propMap.put(CfgProp.SOLR_HOSTPORT, doSub("solr/solrcloud/int[@name='hostPort']"));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT, doSub("solr/solrcloud/int[@name='leaderVoteWait']"));
propMap.put(CfgProp.SOLR_GENERICCORENODENAMES, doSub("solr/solrcloud/bool[@name='genericCoreNodeNames']"));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH, doSub("solr/str[@name='managementPath']"));
propMap.put(CfgProp.SOLR_SHAREDLIB, doSub("solr/str[@name='sharedLib']"));
propMap.put(CfgProp.SOLR_SHARESCHEMA, doSub("solr/str[@name='shareSchema']"));

3
solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
View File

@ -73,6 +73,7 @@ public class ConfigSolrXmlOld extends ConfigSolr {
failIfFound("solr/solrcloud/str[@name='hostContext']");
failIfFound("solr/solrcloud/int[@name='hostPort']");
failIfFound("solr/solrcloud/int[@name='leaderVoteWait']");
failIfFound("solr/solrcloud/int[@name='genericCoreNodeNames']");
failIfFound("solr/str[@name='managementPath']");
failIfFound("solr/str[@name='sharedLib']");
failIfFound("solr/str[@name='shareSchema']");
@ -125,6 +126,8 @@ public class ConfigSolrXmlOld extends ConfigSolr {
config.getVal("solr/cores/@hostPort", false));
propMap.put(CfgProp.SOLR_LEADERVOTEWAIT,
config.getVal("solr/cores/@leaderVoteWait", false));
propMap.put(CfgProp.SOLR_GENERICCORENODENAMES,
config.getVal("solr/cores/@genericCoreNodeNames", false));
propMap.put(CfgProp.SOLR_MANAGEMENTPATH,
config.getVal("solr/cores/@managementPath", false));
propMap.put(CfgProp.SOLR_SHARESCHEMA,

21
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -182,7 +182,7 @@ public class CoreContainer
cores = new CoreContainer(solrHome);
// first we find zkhost, then we check for solr.xml in zk
// 1. look for zkhost from sys prop 2. look for zkhost in {solr.home}/solr.properties
// 1. look for zkhost from sys prop 2. TODO: look for zkhost in {solr.home}/solr.properties
// Either we have a config file or not.
@ -317,13 +317,15 @@ public class CoreContainer
transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
boolean genericCoreNodeNames = cfg.getBool(ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, false);
if (shareSchema) {
indexSchemaCache = new ConcurrentHashMap<String,IndexSchema>();
}
zkClientTimeout = Integer.parseInt(System.getProperty("zkClientTimeout",
Integer.toString(zkClientTimeout)));
zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout);
zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout);
if (isZooKeeperAware() && coreLoadThreads <= 1) {
throw new SolrException(ErrorCode.SERVER_ERROR,
@ -1143,6 +1145,8 @@ public class CoreContainer
zkSys.getHostContext());
addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, "leaderVoteWait",
zkSys.getLeaderVoteWait(), LEADER_VOTE_WAIT);
addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, "genericCoreNodeNames",
Boolean.toString(zkSys.getGenericCoreNodeNames()), "false");
if (transientCacheSize != Integer.MAX_VALUE) { // This test
// is a consequence of testing. I really hate it.
addAttrib(coresAttribs, ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, "transientCacheSize",
@ -1220,18 +1224,7 @@ public class CoreContainer
}
public void preRegisterInZk(final CoreDescriptor p) {
try {
zkSys.getZkController().preRegister(p);
} catch (KeeperException e) {
log.error("", e);
throw new ZooKeeperException(
SolrException.ErrorCode.SERVER_ERROR, "", e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("", e);
throw new ZooKeeperException(
SolrException.ErrorCode.SERVER_ERROR, "", e);
}
zkSys.getZkController().preRegister(p);
}
public String getSolrHome() {

19
solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
View File

@ -167,7 +167,6 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
*/
public abstract void release(Directory directory) throws IOException;
/**
* Normalize a given path.
*
@ -229,5 +228,21 @@ public abstract class DirectoryFactory implements NamedListInitializedPlugin,
}
return isSuccess;
}
/**
* If your implementation can count on delete-on-last-close semantics
* or throws an exception when trying to remove a file in use, return
* false (eg NFS). Otherwise, return true. Defaults to returning false.
*
* @return true if factory impl requires that Searcher's explicitly
* reserve commit points.
*/
public boolean searchersReserveCommitPoints() {
return false;
}
public String getDataHome(CoreDescriptor cd) throws IOException {
// by default, we go off the instance directory
return normalize(SolrResourceLoader.normalizeDir(cd.getInstanceDir()) + cd.getDataDir());
}
}

12
solr/core/src/java/org/apache/solr/core/EphemeralDirectoryFactory.java
View File

@ -51,4 +51,16 @@ public abstract class EphemeralDirectoryFactory extends CachingDirectoryFactory
public boolean isAbsolute(String path) {
return true;
}
@Override
public void remove(Directory dir) throws IOException {
// ram dir does not persist its dir anywhere
}
@Override
public void remove(String path) throws IOException {
// ram dir does not persist its dir anywhere
}
}

288
solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java Normal file
View File

@ -0,0 +1,288 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.store.blockcache.BlockCache;
import org.apache.solr.store.blockcache.BlockDirectory;
import org.apache.solr.store.blockcache.BlockDirectoryCache;
import org.apache.solr.store.blockcache.BufferStore;
import org.apache.solr.store.blockcache.Cache;
import org.apache.solr.store.blockcache.Metrics;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.apache.solr.util.HdfsUtil;
import org.apache.solr.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsDirectoryFactory extends CachingDirectoryFactory {
public static Logger LOG = LoggerFactory
.getLogger(HdfsDirectoryFactory.class);
public static final String BLOCKCACHE_SLAB_COUNT = "solr.hdfs.blockcache.slab.count";
public static final String BLOCKCACHE_DIRECT_MEMORY_ALLOCATION = "solr.hdfs.blockcache.direct.memory.allocation";
public static final String BLOCKCACHE_ENABLED = "solr.hdfs.blockcache.enabled";
public static final String BLOCKCACHE_READ_ENABLED = "solr.hdfs.blockcache.read.enabled";
public static final String BLOCKCACHE_WRITE_ENABLED = "solr.hdfs.blockcache.write.enabled";
public static final String NRTCACHINGDIRECTORY_ENABLE = "solr.hdfs.nrtcachingdirectory.enable";
public static final String NRTCACHINGDIRECTORY_MAXMERGESIZEMB = "solr.hdfs.nrtcachingdirectory.maxmergesizemb";
public static final String NRTCACHINGDIRECTORY_MAXCACHEMB = "solr.hdfs.nrtcachingdirectory.maxcachedmb";
public static final String NUMBEROFBLOCKSPERBANK = "solr.hdfs.blockcache.blocksperbank";
public static final String KERBEROS_ENABLED = "solr.hdfs.security.kerberos.enabled";
public static final String KERBEROS_KEYTAB = "solr.hdfs.security.kerberos.keytabfile";
public static final String KERBEROS_PRINCIPAL = "solr.hdfs.security.kerberos.principal";
public static final String HDFS_HOME = "solr.hdfs.home";
public static final String CONFIG_DIRECTORY = "solr.hdfs.confdir";
private SolrParams params;
private String hdfsDataDir;
private String confDir;
public static Metrics metrics;
private static Boolean kerberosInit;
@Override
public void init(NamedList args) {
params = SolrParams.toSolrParams(args);
this.hdfsDataDir = params.get(HDFS_HOME);
if (this.hdfsDataDir != null && this.hdfsDataDir.length() == 0) {
this.hdfsDataDir = null;
}
boolean kerberosEnabled = params.getBool(KERBEROS_ENABLED, false);
LOG.info("Solr Kerberos Authentication "
+ (kerberosEnabled ? "enabled" : "disabled"));
if (kerberosEnabled) {
initKerberos();
}
}
@Override
protected Directory create(String path, DirContext dirContext)
throws IOException {
LOG.info("creating directory factory for path {}", path);
Configuration conf = getConf();
if (metrics == null) {
metrics = new Metrics(conf);
}
boolean blockCacheEnabled = params.getBool(BLOCKCACHE_ENABLED, true);
boolean blockCacheReadEnabled = params.getBool(BLOCKCACHE_READ_ENABLED,
true);
boolean blockCacheWriteEnabled = params.getBool(BLOCKCACHE_WRITE_ENABLED, true);
Directory dir = null;
if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
int numberOfBlocksPerBank = params.getInt(NUMBEROFBLOCKSPERBANK, 16384);
int blockSize = BlockDirectory.BLOCK_SIZE;
int bankCount = params.getInt(BLOCKCACHE_SLAB_COUNT, 1);
boolean directAllocation = params.getBool(
BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
BlockCache blockCache;
int slabSize = numberOfBlocksPerBank * blockSize;
LOG.info(
"Number of slabs of block cache [{}] with direct memory allocation set to [{}]",
bankCount, directAllocation);
LOG.info(
"Block cache target memory usage, slab size of [{}] will allocate [{}] slabs and use ~[{}] bytes",
new Object[] {slabSize, bankCount,
((long) bankCount * (long) slabSize)});
int _1024Size = params.getInt("solr.hdfs.blockcache.bufferstore.1024",
8192);
int _8192Size = params.getInt("solr.hdfs.blockcache.bufferstore.8192",
8192);
BufferStore.init(_1024Size, _8192Size, metrics);
long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank
* (long) blockSize;
try {
blockCache = new BlockCache(metrics, directAllocation, totalMemory,
slabSize, blockSize);
} catch (OutOfMemoryError e) {
throw new RuntimeException(
"The max direct memory is likely too low. Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)"
+ " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap,"
+ " your java heap size might not be large enough."
+ " Failed allocating ~" + totalMemory / 1000000.0 + " MB.", e);
}
Cache cache = new BlockDirectoryCache(blockCache, metrics);
HdfsDirectory hdfsDirectory = new HdfsDirectory(new Path(path), conf);
dir = new BlockDirectory("solrcore", hdfsDirectory, cache, null,
blockCacheReadEnabled, blockCacheWriteEnabled);
} else {
dir = new HdfsDirectory(new Path(path), conf);
}
boolean nrtCachingDirectory = params.getBool(NRTCACHINGDIRECTORY_ENABLE, true);
if (nrtCachingDirectory) {
double nrtCacheMaxMergeSizeMB = params.getInt(
NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
double nrtCacheMaxCacheMB = params.getInt(NRTCACHINGDIRECTORY_MAXCACHEMB,
192);
return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB,
nrtCacheMaxCacheMB);
}
return dir;
}
@Override
public boolean exists(String path) {
Path hdfsDirPath = new Path(path);
Configuration conf = getConf();
FileSystem fileSystem = null;
try {
fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), conf);
return fileSystem.exists(hdfsDirPath);
} catch (IOException e) {
LOG.error("Error checking if hdfs path exists", e);
throw new RuntimeException("Error checking if hdfs path exists", e);
} finally {
IOUtils.closeQuietly(fileSystem);
}
}
private Configuration getConf() {
Configuration conf = new Configuration();
confDir = params.get(CONFIG_DIRECTORY, null);
HdfsUtil.addHdfsResources(conf, confDir);
return conf;
}
protected synchronized void removeDirectory(CacheValue cacheValue)
throws IOException {
Configuration conf = getConf();
FileSystem fileSystem = null;
try {
fileSystem = FileSystem.newInstance(new URI(cacheValue.path), conf);
boolean success = fileSystem.delete(new Path(cacheValue.path), true);
if (!success) {
throw new RuntimeException("Could not remove directory");
}
} catch (Exception e) {
LOG.error("Could not remove directory", e);
throw new SolrException(ErrorCode.SERVER_ERROR,
"Could not remove directory", e);
} finally {
IOUtils.closeQuietly(fileSystem);
}
}
@Override
public boolean isAbsolute(String path) {
return path.startsWith("hdfs:/");
}
@Override
public boolean isPersistent() {
return true;
}
@Override
public boolean searchersReserveCommitPoints() {
return true;
}
@Override
public String getDataHome(CoreDescriptor cd) throws IOException {
if (hdfsDataDir == null) {
throw new SolrException(ErrorCode.SERVER_ERROR, "You must set the "
+ this.getClass().getSimpleName() + " param " + HDFS_HOME
+ " for relative dataDir paths to work");
}
// by default, we go off the instance directory
String path;
if (cd.getCloudDescriptor() != null) {
path = URLEncoder.encode(cd.getCloudDescriptor().getCollectionName(),
"UTF-8")
+ "/"
+ URLEncoder.encode(cd.getCloudDescriptor().getCoreNodeName(),
"UTF-8");
} else {
path = cd.getName();
}
return normalize(SolrResourceLoader.normalizeDir(ZkController
.trimLeadingAndTrailingSlashes(hdfsDataDir)
+ "/"
+ path
+ "/"
+ cd.getDataDir()));
}
public String getConfDir() {
return confDir;
}
private void initKerberos() {
String keytabFile = params.get(KERBEROS_KEYTAB, "").trim();
if (keytabFile.length() == 0) {
throw new IllegalArgumentException(KERBEROS_KEYTAB + " required because "
+ KERBEROS_ENABLED + " set to true");
}
String principal = params.get(KERBEROS_PRINCIPAL, "");
if (principal.length() == 0) {
throw new IllegalArgumentException(KERBEROS_PRINCIPAL
+ " required because " + KERBEROS_ENABLED + " set to true");
}
synchronized (HdfsDirectoryFactory.class) {
if (kerberosInit == null) {
kerberosInit = new Boolean(true);
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
LOG.info(
"Attempting to acquire kerberos ticket with keytab: {}, principal: {} ",
keytabFile, principal);
try {
UserGroupInformation.loginUserFromKeytab(principal, keytabFile);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
LOG.info("Got Kerberos ticket");
}
}
}
}

6
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -673,11 +673,10 @@ public final class SolrCore implements SolrInfoMBean {
if (dataDir == null) {
if (cd.usingDefaultDataDir()) dataDir = config.getDataDir();
if (dataDir == null) {
dataDir = cd.getDataDir();
try {
dataDir = cd.getDataDir();
if (!directoryFactory.isAbsolute(dataDir)) {
dataDir = directoryFactory.normalize(SolrResourceLoader
.normalizeDir(cd.getInstanceDir()) + dataDir);
dataDir = directoryFactory.getDataHome(cd);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
@ -748,7 +747,6 @@ public final class SolrCore implements SolrInfoMBean {
this.codec = initCodec(solrConfig, schema);
if (updateHandler == null) {
initDirectoryFactory();
solrCoreState = new DefaultSolrCoreState(getDirectoryFactory());
} else {
solrCoreState = updateHandler.getSolrCoreState();

10
solr/core/src/java/org/apache/solr/core/ZkContainer.java
View File

@ -50,6 +50,7 @@ public class ZkContainer {
private String hostContext;
private String host;
private String leaderVoteWait;
private Boolean genericCoreNodeNames;
private int distribUpdateConnTimeout;
public SolrZkServer getZkServer() {
@ -75,6 +76,10 @@ public class ZkContainer {
public String getLeaderVoteWait() {
return leaderVoteWait;
}
public boolean getGenericCoreNodeNames() {
return genericCoreNodeNames;
}
public int getDistribUpdateConnTimeout() {
return distribUpdateConnTimeout;
@ -90,7 +95,7 @@ public class ZkContainer {
}
public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
ZkController zkController = null;
// if zkHost sys property is not set, we are not using ZooKeeper
@ -108,6 +113,7 @@ public class ZkContainer {
this.hostContext = hostContext;
this.host = host;
this.leaderVoteWait = leaderVoteWait;
this.genericCoreNodeNames = genericCoreNodeNames;
this.distribUpdateConnTimeout = distribUpdateConnTimeout;
this.distribUpdateSoTimeout = distribUpdateSoTimeout;
@ -163,7 +169,7 @@ public class ZkContainer {
}
zkController = new ZkController(cc, zookeeperHost, zkClientTimeout,
zkClientConnectTimeout, host, hostPort, hostContext,
leaderVoteWait, distribUpdateConnTimeout, distribUpdateSoTimeout,
leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout,
new CurrentCoreDescriptorProvider() {
@Override

10
solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
View File

@ -362,20 +362,18 @@ public class HttpShardHandler extends ShardHandler {
Map<String, Replica> sliceShards = slice.getReplicasMap();
// For now, recreate the | delimited list of equivalent servers
Set<String> liveNodes = clusterState.getLiveNodes();
StringBuilder sliceShardsStr = new StringBuilder();
boolean first = true;
for (ZkNodeProps nodeProps : sliceShards.values()) {
ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
if (!liveNodes.contains(coreNodeProps.getNodeName())
|| !coreNodeProps.getState().equals(
for (Replica replica : sliceShards.values()) {
if (!clusterState.liveNodesContain(replica.getNodeName())
|| !replica.getStr(ZkStateReader.STATE_PROP).equals(
ZkStateReader.ACTIVE)) continue;
if (first) {
first = false;
} else {
sliceShardsStr.append('|');
}
String url = coreNodeProps.getCoreUrl();
String url = ZkCoreNodeProps.getCoreUrl(replica);
if (url.startsWith("http://"))
url = url.substring(7);
sliceShardsStr.append(url);

14
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -193,7 +193,13 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
this.name = "Searcher@" + Integer.toHexString(hashCode()) + (name!=null ? " "+name : "");
log.info("Opening " + this.name);
Directory dir = this.reader.directory();
if (directoryFactory.searchersReserveCommitPoints()) {
// reserve commit point for life of searcher
core.getDeletionPolicy().saveCommitPoint(
reader.getIndexCommit().getGeneration());
}
Directory dir = getIndexReader().directory();
this.reserveDirectory = reserveDirectory;
this.createdDirectory = r == null;
@ -331,12 +337,18 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable,SolrIn
// super.close();
// can't use super.close() since it just calls reader.close() and that may only be called once
// per reader (even if incRef() was previously called).
long cpg = reader.getIndexCommit().getGeneration();
try {
if (closeReader) reader.decRef();
} catch (Throwable t) {
SolrException.log(log, "Problem dec ref'ing reader", t);
}
if (directoryFactory.searchersReserveCommitPoints()) {
core.getDeletionPolicy().releaseCommitPoint(cpg);
}
for (SolrCache cache : cacheList) {
cache.close();
}

199
solr/core/src/java/org/apache/solr/store/blockcache/BlockCache.java Normal file
View File

@ -0,0 +1,199 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
import com.googlecode.concurrentlinkedhashmap.EvictionListener;
public class BlockCache {
public static final int _128M = 134217728;
public static final int _32K = 32768;
private final ConcurrentMap<BlockCacheKey,BlockCacheLocation> cache;
private final ByteBuffer[] banks;
private final BlockLocks[] locks;
private final AtomicInteger[] lockCounters;
private final int blockSize;
private final int numberOfBlocksPerBank;
private final int maxEntries;
private final Metrics metrics;
public BlockCache(Metrics metrics, boolean directAllocation, long totalMemory) {
this(metrics, directAllocation, totalMemory, _128M);
}
public BlockCache(Metrics metrics, boolean directAllocation,
long totalMemory, int slabSize) {
this(metrics, directAllocation, totalMemory, slabSize, _32K);
}
public BlockCache(Metrics metrics, boolean directAllocation,
long totalMemory, int slabSize, int blockSize) {
this.metrics = metrics;
numberOfBlocksPerBank = slabSize / blockSize;
int numberOfBanks = (int) (totalMemory / slabSize);
banks = new ByteBuffer[numberOfBanks];
locks = new BlockLocks[numberOfBanks];
lockCounters = new AtomicInteger[numberOfBanks];
maxEntries = (numberOfBlocksPerBank * numberOfBanks) - 1;
for (int i = 0; i < numberOfBanks; i++) {
if (directAllocation) {
banks[i] = ByteBuffer.allocateDirect(numberOfBlocksPerBank * blockSize);
} else {
banks[i] = ByteBuffer.allocate(numberOfBlocksPerBank * blockSize);
}
locks[i] = new BlockLocks(numberOfBlocksPerBank);
lockCounters[i] = new AtomicInteger();
}
EvictionListener<BlockCacheKey,BlockCacheLocation> listener = new EvictionListener<BlockCacheKey,BlockCacheLocation>() {
@Override
public void onEviction(BlockCacheKey key, BlockCacheLocation location) {
releaseLocation(location);
}
};
cache = new ConcurrentLinkedHashMap.Builder<BlockCacheKey,BlockCacheLocation>()
.maximumWeightedCapacity(maxEntries).listener(listener).build();
this.blockSize = blockSize;
}
private void releaseLocation(BlockCacheLocation location) {
if (location == null) {
return;
}
int bankId = location.getBankId();
int block = location.getBlock();
location.setRemoved(true);
locks[bankId].clear(block);
lockCounters[bankId].decrementAndGet();
metrics.blockCacheEviction.incrementAndGet();
metrics.blockCacheSize.decrementAndGet();
}
public boolean store(BlockCacheKey blockCacheKey, int blockOffset,
byte[] data, int offset, int length) {
if (length + blockOffset > blockSize) {
throw new RuntimeException("Buffer size exceeded, expecting max ["
+ blockSize + "] got length [" + length + "] with blockOffset ["
+ blockOffset + "]");
}
BlockCacheLocation location = cache.get(blockCacheKey);
boolean newLocation = false;
if (location == null) {
newLocation = true;
location = new BlockCacheLocation();
if (!findEmptyLocation(location)) {
return false;
}
}
if (location.isRemoved()) {
return false;
}
int bankId = location.getBankId();
int bankOffset = location.getBlock() * blockSize;
ByteBuffer bank = getBank(bankId);
bank.position(bankOffset + blockOffset);
bank.put(data, offset, length);
if (newLocation) {
releaseLocation(cache.put(blockCacheKey.clone(), location));
metrics.blockCacheSize.incrementAndGet();
}
return true;
}
public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer,
int blockOffset, int off, int length) {
BlockCacheLocation location = cache.get(blockCacheKey);
if (location == null) {
return false;
}
if (location.isRemoved()) {
return false;
}
int bankId = location.getBankId();
int offset = location.getBlock() * blockSize;
location.touch();
ByteBuffer bank = getBank(bankId);
bank.position(offset + blockOffset);
bank.get(buffer, off, length);
return true;
}
public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer) {
checkLength(buffer);
return fetch(blockCacheKey, buffer, 0, 0, blockSize);
}
private boolean findEmptyLocation(BlockCacheLocation location) {
// This is a tight loop that will try and find a location to
// place the block before giving up
for (int j = 0; j < 10; j++) {
OUTER: for (int bankId = 0; bankId < banks.length; bankId++) {
AtomicInteger bitSetCounter = lockCounters[bankId];
BlockLocks bitSet = locks[bankId];
if (bitSetCounter.get() == numberOfBlocksPerBank) {
// if bitset is full
continue OUTER;
}
// this check needs to spin, if a lock was attempted but not obtained
// the rest of the bank should not be skipped
int bit = bitSet.nextClearBit(0);
INNER: while (bit != -1) {
if (bit >= numberOfBlocksPerBank) {
// bit set is full
continue OUTER;
}
if (!bitSet.set(bit)) {
// lock was not obtained
// this restarts at 0 because another block could have been unlocked
// while this was executing
bit = bitSet.nextClearBit(0);
continue INNER;
} else {
// lock obtained
location.setBankId(bankId);
location.setBlock(bit);
bitSetCounter.incrementAndGet();
return true;
}
}
}
}
return false;
}
private void checkLength(byte[] buffer) {
if (buffer.length != blockSize) {
throw new RuntimeException("Buffer wrong size, expecting [" + blockSize
+ "] got [" + buffer.length + "]");
}
}
private ByteBuffer getBank(int bankId) {
return banks[bankId].duplicate();
}
public int getSize() {
return cache.size();
}
}

69
solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheKey.java Normal file
View File

@ -0,0 +1,69 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class BlockCacheKey implements Cloneable {
private long block;
private int file;
public long getBlock() {
return block;
}
public int getFile() {
return file;
}
public void setBlock(long block) {
this.block = block;
}
public void setFile(int file) {
this.file = file;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (block ^ (block >>> 32));
result = prime * result + file;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
BlockCacheKey other = (BlockCacheKey) obj;
if (block != other.block) return false;
if (file != other.file) return false;
return true;
}
@Override
public BlockCacheKey clone() {
try {
return (BlockCacheKey) super.clone();
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
}
}

67
solr/core/src/java/org/apache/solr/store/blockcache/BlockCacheLocation.java Normal file
View File

@ -0,0 +1,67 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.concurrent.atomic.AtomicBoolean;
public class BlockCacheLocation {
private int block;
private int bankId;
private long lastAccess = System.currentTimeMillis();
private long accesses;
private AtomicBoolean removed = new AtomicBoolean(false);
public void setBlock(int block) {
this.block = block;
}
public void setBankId(int bankId) {
this.bankId = bankId;
}
public int getBlock() {
return block;
}
public int getBankId() {
return bankId;
}
public void touch() {
lastAccess = System.currentTimeMillis();
accesses++;
}
public long getLastAccess() {
return lastAccess;
}
public long getNumberOfAccesses() {
return accesses;
}
public boolean isRemoved() {
return removed.get();
}
public void setRemoved(boolean removed) {
this.removed.set(removed);
}
}

385
solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectory.java Normal file
View File

@ -0,0 +1,385 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Collection;
import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BlockDirectory extends Directory {
public static Logger LOG = LoggerFactory.getLogger(BlockDirectory.class);
public static final long BLOCK_SHIFT = 13; // 2^13 = 8,192 bytes per block
public static final long BLOCK_MOD = 0x1FFF;
public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT;
public static long getBlock(long pos) {
return pos >>> BLOCK_SHIFT;
}
public static long getPosition(long pos) {
return pos & BLOCK_MOD;
}
public static long getRealPosition(long block, long positionInBlock) {
return (block << BLOCK_SHIFT) + positionInBlock;
}
public static Cache NO_CACHE = new Cache() {
@Override
public void update(String name, long blockId, int blockOffset,
byte[] buffer, int offset, int length) {}
@Override
public boolean fetch(String name, long blockId, int blockOffset, byte[] b,
int off, int lengthToReadInBlock) {
return false;
}
@Override
public void delete(String name) {
}
@Override
public long size() {
return 0;
}
@Override
public void renameCacheFile(String source, String dest) {}
};
private Directory directory;
private int blockSize;
private String dirName;
private Cache cache;
private Set<String> blockCacheFileTypes;
private final boolean blockCacheReadEnabled;
private final boolean blockCacheWriteEnabled;
public BlockDirectory(String dirName, Directory directory, Cache cache,
Set<String> blockCacheFileTypes, boolean blockCacheReadEnabled,
boolean blockCacheWriteEnabled) throws IOException {
this.dirName = dirName;
this.directory = directory;
blockSize = BLOCK_SIZE;
this.cache = cache;
if (blockCacheFileTypes == null || blockCacheFileTypes.isEmpty()) {
this.blockCacheFileTypes = null;
} else {
this.blockCacheFileTypes = blockCacheFileTypes;
}
this.blockCacheReadEnabled = blockCacheReadEnabled;
if (!blockCacheReadEnabled) {
LOG.info("Block cache on read is disabled");
}
this.blockCacheWriteEnabled = blockCacheWriteEnabled;
if (!blockCacheWriteEnabled) {
LOG.info("Block cache on write is disabled");
}
if (directory.getLockFactory() != null) {
setLockFactory(directory.getLockFactory());
}
}
private IndexInput openInput(String name, int bufferSize, IOContext context)
throws IOException {
final IndexInput source = directory.openInput(name, context);
if (useReadCache(name, context)) {
return new CachedIndexInput(source, blockSize, name,
getFileCacheName(name), cache, bufferSize);
}
return source;
}
private boolean isCachableFile(String name) {
for (String ext : blockCacheFileTypes) {
if (name.endsWith(ext)) {
return true;
}
}
return false;
}
@Override
public IndexInput openInput(final String name, IOContext context)
throws IOException {
return openInput(name, blockSize, context);
}
static class CachedIndexInput extends CustomBufferedIndexInput {
private IndexInput _source;
private int _blockSize;
private long _fileLength;
private String _cacheName;
private Cache _cache;
public CachedIndexInput(IndexInput source, int blockSize, String name,
String cacheName, Cache cache, int bufferSize) {
super(name, bufferSize);
_source = source;
_blockSize = blockSize;
_fileLength = source.length();
_cacheName = cacheName;
_cache = cache;
}
@Override
public IndexInput clone() {
CachedIndexInput clone = (CachedIndexInput) super.clone();
clone._source = (IndexInput) _source.clone();
return clone;
}
@Override
public long length() {
return _source.length();
}
@Override
protected void seekInternal(long pos) throws IOException {}
@Override
protected void readInternal(byte[] b, int off, int len) throws IOException {
long position = getFilePointer();
while (len > 0) {
int length = fetchBlock(position, b, off, len);
position += length;
len -= length;
off += length;
}
}
private int fetchBlock(long position, byte[] b, int off, int len)
throws IOException {
// read whole block into cache and then provide needed data
long blockId = getBlock(position);
int blockOffset = (int) getPosition(position);
int lengthToReadInBlock = Math.min(len, _blockSize - blockOffset);
if (checkCache(blockId, blockOffset, b, off, lengthToReadInBlock)) {
return lengthToReadInBlock;
} else {
readIntoCacheAndResult(blockId, blockOffset, b, off,
lengthToReadInBlock);
}
return lengthToReadInBlock;
}
private void readIntoCacheAndResult(long blockId, int blockOffset,
byte[] b, int off, int lengthToReadInBlock) throws IOException {
long position = getRealPosition(blockId, 0);
int length = (int) Math.min(_blockSize, _fileLength - position);
_source.seek(position);
byte[] buf = BufferStore.takeBuffer(_blockSize);
_source.readBytes(buf, 0, length);
System.arraycopy(buf, blockOffset, b, off, lengthToReadInBlock);
_cache.update(_cacheName, blockId, 0, buf, 0, _blockSize);
BufferStore.putBuffer(buf);
}
private boolean checkCache(long blockId, int blockOffset, byte[] b,
int off, int lengthToReadInBlock) {
return _cache.fetch(_cacheName, blockId, blockOffset, b, off,
lengthToReadInBlock);
}
@Override
protected void closeInternal() throws IOException {
_source.close();
}
}
@Override
public void close() throws IOException {
try {
String[] files = listAll();
for (String file : files) {
cache.delete(getFileCacheName(file));
}
} catch (FileNotFoundException e) {
// the local file system folder may be gone
} finally {
directory.close();
}
}
String getFileCacheName(String name) throws IOException {
return getFileCacheLocation(name) + ":" + getFileModified(name);
}
private long getFileModified(String name) throws IOException {
if (directory instanceof FSDirectory) {
File directory = ((FSDirectory) this.directory).getDirectory();
File file = new File(directory, name);
if (!file.exists()) {
throw new FileNotFoundException("File [" + name + "] not found");
}
return file.lastModified();
} else if (directory instanceof HdfsDirectory) {
return ((HdfsDirectory) directory).fileModified(name);
} else {
throw new RuntimeException("Not supported");
}
}
public void clearLock(String name) throws IOException {
directory.clearLock(name);
}
String getFileCacheLocation(String name) {
return dirName + "/" + name;
}
@Override
public void copy(Directory to, String src, String dest, IOContext context)
throws IOException {
directory.copy(to, src, dest, context);
}
public LockFactory getLockFactory() {
return directory.getLockFactory();
}
public String getLockID() {
return directory.getLockID();
}
public Lock makeLock(String name) {
return directory.makeLock(name);
}
public void setLockFactory(LockFactory lockFactory) throws IOException {
directory.setLockFactory(lockFactory);
}
@Override
public void sync(Collection<String> names) throws IOException {
directory.sync(names);
}
// @SuppressWarnings("deprecation")
// public void sync(String name) throws IOException {
// _directory.sync(name);
// }
public String toString() {
return directory.toString();
}
/**
* Determine whether read caching should be used for a particular
* file/context.
*/
boolean useReadCache(String name, IOContext context) {
if (!blockCacheReadEnabled) {
return false;
}
if (blockCacheFileTypes != null && !isCachableFile(name)) {
return false;
}
switch (context.context) {
default: {
return true;
}
}
}
/**
* Determine whether write caching should be used for a particular
* file/context.
*/
boolean useWriteCache(String name, IOContext context) {
if (!blockCacheWriteEnabled) {
return false;
}
if (blockCacheFileTypes != null && !isCachableFile(name)) {
return false;
}
switch (context.context) {
case MERGE: {
// we currently don't cache any merge context writes
return false;
}
default: {
return true;
}
}
}
@Override
public IndexOutput createOutput(String name, IOContext context)
throws IOException {
IndexOutput dest = directory.createOutput(name, context);
if (useWriteCache(name, context)) {
return new CachedIndexOutput(this, dest, blockSize, name, cache,
blockSize);
}
return dest;
}
public void deleteFile(String name) throws IOException {
cache.delete(getFileCacheName(name));
directory.deleteFile(name);
}
public boolean fileExists(String name) throws IOException {
return directory.fileExists(name);
}
public long fileLength(String name) throws IOException {
return directory.fileLength(name);
}
// @SuppressWarnings("deprecation")
// public long fileModified(String name) throws IOException {
// return _directory.fileModified(name);
// }
public String[] listAll() throws IOException {
return directory.listAll();
}
// @SuppressWarnings("deprecation")
// public void touchFile(String name) throws IOException {
// _directory.touchFile(name);
// }
public Directory getDirectory() {
return directory;
}
}

87
solr/core/src/java/org/apache/solr/store/blockcache/BlockDirectoryCache.java Normal file
View File

@ -0,0 +1,87 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
public class BlockDirectoryCache implements Cache {
private BlockCache blockCache;
private AtomicInteger counter = new AtomicInteger();
private Map<String,Integer> names = new ConcurrentHashMap<String,Integer>();
private Metrics metrics;
public BlockDirectoryCache(BlockCache blockCache, Metrics metrics) {
this.blockCache = blockCache;
this.metrics = metrics;
}
@Override
public void delete(String name) {
names.remove(name);
}
@Override
public void update(String name, long blockId, int blockOffset, byte[] buffer,
int offset, int length) {
Integer file = names.get(name);
if (file == null) {
file = counter.incrementAndGet();
names.put(name, file);
}
BlockCacheKey blockCacheKey = new BlockCacheKey();
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
blockCache.store(blockCacheKey, blockOffset, buffer, offset, length);
}
@Override
public boolean fetch(String name, long blockId, int blockOffset, byte[] b,
int off, int lengthToReadInBlock) {
Integer file = names.get(name);
if (file == null) {
return false;
}
BlockCacheKey blockCacheKey = new BlockCacheKey();
blockCacheKey.setBlock(blockId);
blockCacheKey.setFile(file);
boolean fetch = blockCache.fetch(blockCacheKey, b, blockOffset, off,
lengthToReadInBlock);
if (fetch) {
metrics.blockCacheHit.incrementAndGet();
} else {
metrics.blockCacheMiss.incrementAndGet();
}
return fetch;
}
@Override
public long size() {
return blockCache.getSize();
}
@Override
public void renameCacheFile(String source, String dest) {
Integer file = names.remove(source);
// possible if the file is empty
if (file != null) {
names.put(dest, file);
}
}
}

96
solr/core/src/java/org/apache/solr/store/blockcache/BlockLocks.java Normal file
View File

@ -0,0 +1,96 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.concurrent.atomic.AtomicLongArray;
import org.apache.lucene.util.OpenBitSet;
public class BlockLocks {
private AtomicLongArray bits;
private int wlen;
public BlockLocks(long numBits) {
int length = OpenBitSet.bits2words(numBits);
bits = new AtomicLongArray(length);
wlen = length;
}
/**
* Find the next clear bit in the bit set.
*
* @param index
* index
* @return next next bit
*/
public int nextClearBit(int index) {
int i = index >> 6;
if (i >= wlen) return -1;
int subIndex = index & 0x3f; // index within the word
long word = ~bits.get(i) >> subIndex; // skip all the bits to the right of
// index
if (word != 0) {
return (i << 6) + subIndex + Long.numberOfTrailingZeros(word);
}
while (++i < wlen) {
word = ~bits.get(i);
if (word != 0) {
return (i << 6) + Long.numberOfTrailingZeros(word);
}
}
return -1;
}
/**
* Thread safe set operation that will set the bit if and only if the bit was
* not previously set.
*
* @param index
* the index position to set.
* @return returns true if the bit was set and false if it was already set.
*/
public boolean set(int index) {
int wordNum = index >> 6; // div 64
int bit = index & 0x3f; // mod 64
long bitmask = 1L << bit;
long word, oword;
do {
word = bits.get(wordNum);
// if set another thread stole the lock
if ((word & bitmask) != 0) {
return false;
}
oword = word;
word |= bitmask;
} while (!bits.compareAndSet(wordNum, oword, word));
return true;
}
public void clear(int index) {
int wordNum = index >> 6;
int bit = index & 0x03f;
long bitmask = 1L << bit;
long word, oword;
do {
word = bits.get(wordNum);
oword = word;
word &= ~bitmask;
} while (!bits.compareAndSet(wordNum, oword, word));
}
}

110
solr/core/src/java/org/apache/solr/store/blockcache/BufferStore.java Normal file
View File

@ -0,0 +1,110 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class BufferStore {
public static Logger LOG = LoggerFactory.getLogger(BufferStore.class);
private static BlockingQueue<byte[]> _1024 = setupBuffers(1024, 1);
private static BlockingQueue<byte[]> _8192 = setupBuffers(8192, 1);
public static AtomicLong shardBuffercacheLost = new AtomicLong();
public static AtomicLong shardBuffercacheAllocate1024 = new AtomicLong();
public static AtomicLong shardBuffercacheAllocate8192 = new AtomicLong();
public static AtomicLong shardBuffercacheAllocateOther = new AtomicLong();
public static void init(int _1024Size, int _8192Size, Metrics metrics) {
LOG.info("Initializing the 1024 buffers with [{}] buffers.", _1024Size);
_1024 = setupBuffers(1024, _1024Size);
LOG.info("Initializing the 8192 buffers with [{}] buffers.", _8192Size);
_8192 = setupBuffers(8192, _8192Size);
shardBuffercacheLost = metrics.shardBuffercacheLost;
shardBuffercacheAllocate1024 = metrics.shardBuffercacheAllocate1024;
shardBuffercacheAllocate8192 = metrics.shardBuffercacheAllocate8192;
shardBuffercacheAllocateOther = metrics.shardBuffercacheAllocateOther;
}
private static BlockingQueue<byte[]> setupBuffers(int bufferSize, int count) {
BlockingQueue<byte[]> queue = new ArrayBlockingQueue<byte[]>(count);
for (int i = 0; i < count; i++) {
queue.add(new byte[bufferSize]);
}
return queue;
}
public static byte[] takeBuffer(int bufferSize) {
switch (bufferSize) {
case 1024:
return newBuffer1024(_1024.poll());
case 8192:
return newBuffer8192(_8192.poll());
default:
return newBuffer(bufferSize);
}
}
public static void putBuffer(byte[] buffer) {
if (buffer == null) {
return;
}
int bufferSize = buffer.length;
switch (bufferSize) {
case 1024:
checkReturn(_1024.offer(buffer));
return;
case 8192:
checkReturn(_8192.offer(buffer));
return;
}
}
private static void checkReturn(boolean offer) {
if (!offer) {
shardBuffercacheLost.incrementAndGet();
}
}
private static byte[] newBuffer1024(byte[] buf) {
if (buf != null) {
return buf;
}
shardBuffercacheAllocate1024.incrementAndGet();
return new byte[1024];
}
private static byte[] newBuffer8192(byte[] buf) {
if (buf != null) {
return buf;
}
shardBuffercacheAllocate8192.incrementAndGet();
return new byte[8192];
}
private static byte[] newBuffer(int size) {
shardBuffercacheAllocateOther.incrementAndGet();
return new byte[size];
}
}

62
solr/core/src/java/org/apache/solr/store/blockcache/Cache.java Normal file
View File

@ -0,0 +1,62 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public interface Cache {
/**
* Remove a file from the cache.
*
* @param name
* cache file name
*/
void delete(String name);
/**
* Update the content of the specified cache file. Creates cache entry if
* necessary.
*
*/
void update(String name, long blockId, int blockOffset, byte[] buffer,
int offset, int length);
/**
* Fetch the specified cache file content.
*
* @return true if cached content found, otherwise return false
*/
boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off,
int lengthToReadInBlock);
/**
* Number of entries in the cache.
*/
long size();
/**
* Expert: Rename the specified file in the cache. Allows a file to be moved
* without invalidating the cache.
*
* @param source
* original name
* @param dest
* final name
*/
void renameCacheFile(String source, String dest);
}

91
solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java Normal file
View File

@ -0,0 +1,91 @@
package org.apache.solr.store.blockcache;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
/*
* Cache the blocks as they are written. The cache file name is the name of
* the file until the file is closed, at which point the cache is updated
* to include the last modified date (which is unknown until that point).
*/
public class CachedIndexOutput extends ReusedBufferedIndexOutput {
private final BlockDirectory directory;
private final IndexOutput dest;
private final int blockSize;
private final String name;
private final String location;
private final Cache cache;
public CachedIndexOutput(BlockDirectory directory, IndexOutput dest,
int blockSize, String name, Cache cache, int bufferSize) {
super(bufferSize);
this.directory = directory;
this.dest = dest;
this.blockSize = blockSize;
this.name = name;
this.location = directory.getFileCacheLocation(name);
this.cache = cache;
}
@Override
public void flushInternal() throws IOException {
dest.flush();
}
@Override
public void closeInternal() throws IOException {
dest.close();
cache.renameCacheFile(location, directory.getFileCacheName(name));
}
@Override
public void seekInternal(long pos) throws IOException {
throw new IOException("Seek not supported");
}
private int writeBlock(long position, byte[] b, int offset, int length)
throws IOException {
// read whole block into cache and then provide needed data
long blockId = BlockDirectory.getBlock(position);
int blockOffset = (int) BlockDirectory.getPosition(position);
int lengthToWriteInBlock = Math.min(length, blockSize - blockOffset);
// write the file and copy into the cache
dest.writeBytes(b, offset, lengthToWriteInBlock);
cache.update(location, blockId, blockOffset, b, offset,
lengthToWriteInBlock);
return lengthToWriteInBlock;
}
@Override
public void writeInternal(byte[] b, int offset, int length)
throws IOException {
long position = getBufferStart();
while (length > 0) {
int len = writeBlock(position, b, offset, length);
position += len;
length -= len;
offset += len;
}
}
}

272
solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java Normal file
View File

@ -0,0 +1,272 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.EOFException;
import java.io.IOException;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
public abstract class CustomBufferedIndexInput extends IndexInput {
public static final int BUFFER_SIZE = 1024;
private int bufferSize = BUFFER_SIZE;
protected byte[] buffer;
private long bufferStart = 0; // position in file of buffer
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
@Override
public byte readByte() throws IOException {
if (bufferPosition >= bufferLength) refill();
return buffer[bufferPosition++];
}
public CustomBufferedIndexInput(String resourceDesc) {
this(resourceDesc, BUFFER_SIZE);
}
public CustomBufferedIndexInput(String resourceDesc, int bufferSize) {
super(resourceDesc);
checkBufferSize(bufferSize);
this.bufferSize = bufferSize;
}
private void checkBufferSize(int bufferSize) {
if (bufferSize <= 0) throw new IllegalArgumentException(
"bufferSize must be greater than 0 (got " + bufferSize + ")");
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
readBytes(b, offset, len, true);
}
@Override
public void readBytes(byte[] b, int offset, int len, boolean useBuffer)
throws IOException {
if (len <= (bufferLength - bufferPosition)) {
// the buffer contains enough data to satisfy this request
if (len > 0) // to allow b to be null if len is 0...
System.arraycopy(buffer, bufferPosition, b, offset, len);
bufferPosition += len;
} else {
// the buffer does not have enough data. First serve all we've got.
int available = bufferLength - bufferPosition;
if (available > 0) {
System.arraycopy(buffer, bufferPosition, b, offset, available);
offset += available;
len -= available;
bufferPosition += available;
}
// and now, read the remaining 'len' bytes:
if (useBuffer && len < bufferSize) {
// If the amount left to read is small enough, and
// we are allowed to use our buffer, do it in the usual
// buffered way: fill the buffer and copy from it:
refill();
if (bufferLength < len) {
// Throw an exception when refill() could not read len bytes:
System.arraycopy(buffer, 0, b, offset, bufferLength);
throw new IOException("read past EOF");
} else {
System.arraycopy(buffer, 0, b, offset, len);
bufferPosition = len;
}
} else {
// The amount left to read is larger than the buffer
// or we've been asked to not use our buffer -
// there's no performance reason not to read it all
// at once. Note that unlike the previous code of
// this function, there is no need to do a seek
// here, because there's no need to reread what we
// had in the buffer.
long after = bufferStart + bufferPosition + len;
if (after > length()) throw new IOException("read past EOF");
readInternal(b, offset, len);
bufferStart = after;
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read
}
}
}
@Override
public int readInt() throws IOException {
if (4 <= (bufferLength - bufferPosition)) {
return ((buffer[bufferPosition++] & 0xFF) << 24)
| ((buffer[bufferPosition++] & 0xFF) << 16)
| ((buffer[bufferPosition++] & 0xFF) << 8)
| (buffer[bufferPosition++] & 0xFF);
} else {
return super.readInt();
}
}
@Override
public long readLong() throws IOException {
if (8 <= (bufferLength - bufferPosition)) {
final int i1 = ((buffer[bufferPosition++] & 0xff) << 24)
| ((buffer[bufferPosition++] & 0xff) << 16)
| ((buffer[bufferPosition++] & 0xff) << 8)
| (buffer[bufferPosition++] & 0xff);
final int i2 = ((buffer[bufferPosition++] & 0xff) << 24)
| ((buffer[bufferPosition++] & 0xff) << 16)
| ((buffer[bufferPosition++] & 0xff) << 8)
| (buffer[bufferPosition++] & 0xff);
return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
} else {
return super.readLong();
}
}
@Override
public int readVInt() throws IOException {
if (5 <= (bufferLength - bufferPosition)) {
byte b = buffer[bufferPosition++];
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = buffer[bufferPosition++];
i |= (b & 0x7F) << shift;
}
return i;
} else {
return super.readVInt();
}
}
@Override
public long readVLong() throws IOException {
if (9 <= bufferLength - bufferPosition) {
byte b = buffer[bufferPosition++];
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = buffer[bufferPosition++];
i |= (b & 0x7FL) << shift;
}
return i;
} else {
return super.readVLong();
}
}
private void refill() throws IOException {
long start = bufferStart + bufferPosition;
long end = start + bufferSize;
if (end > length()) // don't read past EOF
end = length();
int newLength = (int) (end - start);
if (newLength <= 0) throw new EOFException("read past EOF");
if (buffer == null) {
buffer = BufferStore.takeBuffer(bufferSize);
seekInternal(bufferStart);
}
readInternal(buffer, 0, newLength);
bufferLength = newLength;
bufferStart = start;
bufferPosition = 0;
}
@Override
public final void close() throws IOException {
closeInternal();
BufferStore.putBuffer(buffer);
buffer = null;
}
protected abstract void closeInternal() throws IOException;
/**
* Expert: implements buffer refill. Reads bytes from the current position in
* the input.
*
* @param b
* the array to read bytes into
* @param offset
* the offset in the array to start storing bytes
* @param length
* the number of bytes to read
*/
protected abstract void readInternal(byte[] b, int offset, int length)
throws IOException;
@Override
public long getFilePointer() {
return bufferStart + bufferPosition;
}
@Override
public void seek(long pos) throws IOException {
if (pos >= bufferStart && pos < (bufferStart + bufferLength)) bufferPosition = (int) (pos - bufferStart); // seek
// within
// buffer
else {
bufferStart = pos;
bufferPosition = 0;
bufferLength = 0; // trigger refill() on read()
seekInternal(pos);
}
}
/**
* Expert: implements seek. Sets current position in this file, where the next
* {@link #readInternal(byte[],int,int)} will occur.
*
* @see #readInternal(byte[],int,int)
*/
protected abstract void seekInternal(long pos) throws IOException;
@Override
public IndexInput clone() {
CustomBufferedIndexInput clone = (CustomBufferedIndexInput) super.clone();
clone.buffer = null;
clone.bufferLength = 0;
clone.bufferPosition = 0;
clone.bufferStart = getFilePointer();
return clone;
}
/**
* Flushes the in-memory bufer to the given output, copying at most
* <code>numBytes</code>.
* <p>
* <b>NOTE:</b> this method does not refill the buffer, however it does
* advance the buffer position.
*
* @return the number of bytes actually flushed from the in-memory buffer.
*/
protected int flushBuffer(IndexOutput out, long numBytes) throws IOException {
int toCopy = bufferLength - bufferPosition;
if (toCopy > numBytes) {
toCopy = (int) numBytes;
}
if (toCopy > 0) {
out.writeBytes(buffer, bufferPosition, toCopy);
bufferPosition += toCopy;
}
return toCopy;
}
}

126
solr/core/src/java/org/apache/solr/store/blockcache/Metrics.java Normal file
View File

@ -0,0 +1,126 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics.MetricsContext;
import org.apache.hadoop.metrics.MetricsRecord;
import org.apache.hadoop.metrics.MetricsUtil;
import org.apache.hadoop.metrics.Updater;
import org.apache.hadoop.metrics.jvm.JvmMetrics;
public class Metrics implements Updater {
public static class MethodCall {
public AtomicLong invokes = new AtomicLong();
public AtomicLong times = new AtomicLong();
}
public AtomicLong blockCacheHit = new AtomicLong(0);
public AtomicLong blockCacheMiss = new AtomicLong(0);
public AtomicLong blockCacheEviction = new AtomicLong(0);
public AtomicLong blockCacheSize = new AtomicLong(0);
public AtomicLong rowReads = new AtomicLong(0);
public AtomicLong rowWrites = new AtomicLong(0);
public AtomicLong recordReads = new AtomicLong(0);
public AtomicLong recordWrites = new AtomicLong(0);
public AtomicLong queriesExternal = new AtomicLong(0);
public AtomicLong queriesInternal = new AtomicLong(0);
public AtomicLong shardBuffercacheAllocate1024 = new AtomicLong(0);
public AtomicLong shardBuffercacheAllocate8192 = new AtomicLong(0);
public AtomicLong shardBuffercacheAllocateOther = new AtomicLong(0);
public AtomicLong shardBuffercacheLost = new AtomicLong(0);
public Map<String,MethodCall> methodCalls = new ConcurrentHashMap<String, MethodCall>();
public AtomicLong tableCount = new AtomicLong(0);
public AtomicLong rowCount = new AtomicLong(0);
public AtomicLong recordCount = new AtomicLong(0);
public AtomicLong indexCount = new AtomicLong(0);
public AtomicLong indexMemoryUsage = new AtomicLong(0);
public AtomicLong segmentCount = new AtomicLong(0);
private MetricsRecord metricsRecord;
private long previous = System.nanoTime();
public static void main(String[] args) throws InterruptedException {
Configuration conf = new Configuration();
Metrics metrics = new Metrics(conf);
MethodCall methodCall = new MethodCall();
metrics.methodCalls.put("test",methodCall);
for (int i = 0; i < 100; i++) {
metrics.blockCacheHit.incrementAndGet();
metrics.blockCacheMiss.incrementAndGet();
methodCall.invokes.incrementAndGet();
methodCall.times.addAndGet(56000000);
Thread.sleep(500);
}
}
public Metrics(Configuration conf) {
JvmMetrics.init("blockcache", Long.toString(System.currentTimeMillis()));
MetricsContext metricsContext = MetricsUtil.getContext("blockcache");
metricsRecord = MetricsUtil.createRecord(metricsContext, "metrics");
metricsContext.registerUpdater(this);
}
@Override
public void doUpdates(MetricsContext context) {
synchronized (this) {
long now = System.nanoTime();
float seconds = (now - previous) / 1000000000.0f;
metricsRecord.setMetric("blockcache.hit", getPerSecond(blockCacheHit.getAndSet(0), seconds));
metricsRecord.setMetric("blockcache.miss", getPerSecond(blockCacheMiss.getAndSet(0), seconds));
metricsRecord.setMetric("blockcache.eviction", getPerSecond(blockCacheEviction.getAndSet(0), seconds));
metricsRecord.setMetric("blockcache.size", blockCacheSize.get());
metricsRecord.setMetric("row.reads", getPerSecond(rowReads.getAndSet(0), seconds));
metricsRecord.setMetric("row.writes", getPerSecond(rowWrites.getAndSet(0), seconds));
metricsRecord.setMetric("record.reads", getPerSecond(recordReads.getAndSet(0), seconds));
metricsRecord.setMetric("record.writes", getPerSecond(recordWrites.getAndSet(0), seconds));
metricsRecord.setMetric("query.external", getPerSecond(queriesExternal.getAndSet(0), seconds));
metricsRecord.setMetric("query.internal", getPerSecond(queriesInternal.getAndSet(0), seconds));
for (Entry<String,MethodCall> entry : methodCalls.entrySet()) {
String key = entry.getKey();
MethodCall value = entry.getValue();
long invokes = value.invokes.getAndSet(0);
long times = value.times.getAndSet(0);
float avgTimes = (times / (float) invokes) / 1000000000.0f;
metricsRecord.setMetric("methodcalls." + key + ".count", getPerSecond(invokes, seconds));
metricsRecord.setMetric("methodcalls." + key + ".time", avgTimes);
}
metricsRecord.setMetric("tables", tableCount.get());
metricsRecord.setMetric("rows", rowCount.get());
metricsRecord.setMetric("records", recordCount.get());
metricsRecord.setMetric("index.count", indexCount.get());
metricsRecord.setMetric("index.memoryusage", indexMemoryUsage.get());
metricsRecord.setMetric("index.segments", segmentCount.get());
previous = now;
}
metricsRecord.update();
}
private float getPerSecond(long value, float seconds) {
return (float) (value / seconds);
}
}

178
solr/core/src/java/org/apache/solr/store/blockcache/ReusedBufferedIndexOutput.java Normal file
View File

@ -0,0 +1,178 @@
package org.apache.solr.store.blockcache;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
public abstract class ReusedBufferedIndexOutput extends IndexOutput {
public static final int BUFFER_SIZE = 1024;
private int bufferSize = BUFFER_SIZE;
protected byte[] buffer;
/** position in the file of buffer */
private long bufferStart = 0;
/** end of valid bytes */
private int bufferLength = 0;
/** next byte to write */
private int bufferPosition = 0;
/** total length of the file */
private long fileLength = 0;
public ReusedBufferedIndexOutput() {
this(BUFFER_SIZE);
}
public ReusedBufferedIndexOutput(int bufferSize) {
checkBufferSize(bufferSize);
this.bufferSize = bufferSize;
buffer = BufferStore.takeBuffer(this.bufferSize);
}
protected long getBufferStart() {
return bufferStart;
}
private void checkBufferSize(int bufferSize) {
if (bufferSize <= 0) throw new IllegalArgumentException(
"bufferSize must be greater than 0 (got " + bufferSize + ")");
}
/** Write the buffered bytes to cache */
private void flushBufferToCache() throws IOException {
writeInternal(buffer, 0, bufferLength);
bufferStart += bufferLength;
bufferLength = 0;
bufferPosition = 0;
}
protected abstract void flushInternal() throws IOException;
@Override
public void flush() throws IOException {
flushBufferToCache();
flushInternal();
}
protected abstract void closeInternal() throws IOException;
@Override
public void close() throws IOException {
flushBufferToCache();
closeInternal();
BufferStore.putBuffer(buffer);
buffer = null;
}
@Override
public long getFilePointer() {
return bufferStart + bufferPosition;
}
protected abstract void seekInternal(long pos) throws IOException;
@Override
public long length() throws IOException {
return fileLength;
}
@Override
public void writeByte(byte b) throws IOException {
if (bufferPosition >= bufferSize) {
flushBufferToCache();
}
if (getFilePointer() >= fileLength) {
fileLength++;
}
buffer[bufferPosition++] = b;
if (bufferPosition > bufferLength) {
bufferLength = bufferPosition;
}
}
/**
* Expert: implements buffer flushing to cache. Writes bytes to the current
* position in the output.
*
* @param b
* the array of bytes to write
* @param offset
* the offset in the array of bytes to write
* @param length
* the number of bytes to write
*/
protected abstract void writeInternal(byte[] b, int offset, int length)
throws IOException;
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
if (getFilePointer() + length > fileLength) {
fileLength = getFilePointer() + length;
}
if (length <= bufferSize - bufferPosition) {
// the buffer contains enough space to satisfy this request
if (length > 0) { // to allow b to be null if len is 0...
System.arraycopy(b, offset, buffer, bufferPosition, length);
}
bufferPosition += length;
if (bufferPosition > bufferLength) {
bufferLength = bufferPosition;
}
} else {
// the buffer does not have enough space. First buffer all we've got.
int available = bufferSize - bufferPosition;
if (available > 0) {
System.arraycopy(b, offset, buffer, bufferPosition, available);
offset += available;
length -= available;
bufferPosition = bufferSize;
bufferLength = bufferSize;
}
flushBufferToCache();
// and now, write the remaining 'length' bytes:
if (length < bufferSize) {
// If the amount left to write is small enough do it in the usual
// buffered way:
System.arraycopy(b, offset, buffer, 0, length);
bufferPosition = length;
bufferLength = length;
} else {
// The amount left to write is larger than the buffer
// there's no performance reason not to write it all
// at once.
writeInternal(b, offset, length);
bufferStart += length;
bufferPosition = 0;
bufferLength = 0;
}
}
}
@Override
protected Object clone() throws CloneNotSupportedException {
throw new CloneNotSupportedException();
}
}

29
solr/core/src/java/org/apache/solr/store/blockcache/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
An HDFS blockcache implementation.
</p>
</body>
</html>

262
solr/core/src/java/org/apache/solr/store/hdfs/HdfsDirectory.java Normal file
View File

@ -0,0 +1,262 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NoLockFactory;
import org.apache.solr.store.blockcache.CustomBufferedIndexInput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsDirectory extends Directory {
public static Logger LOG = LoggerFactory.getLogger(HdfsDirectory.class);
public static final int BUFFER_SIZE = 8192;
private static final String LF_EXT = ".lf";
protected static final String SEGMENTS_GEN = "segments.gen";
protected static final IndexOutput NULL_WRITER = new NullIndexOutput();
protected Path hdfsDirPath;
protected Configuration configuration;
private final FileSystem fileSystem;
public HdfsDirectory(Path hdfsDirPath, Configuration configuration)
throws IOException {
assert hdfsDirPath.toString().startsWith("hdfs:/") : hdfsDirPath.toString();
setLockFactory(NoLockFactory.getNoLockFactory());
this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration;
fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration);
try {
if (!fileSystem.exists(hdfsDirPath)) {
fileSystem.mkdirs(hdfsDirPath);
}
} catch (Exception e) {
IOUtils.closeQuietly(fileSystem);
throw new RuntimeException("Problem creating directory: " + hdfsDirPath,
e);
}
}
@Override
public void close() throws IOException {
LOG.info("Closing hdfs directory {}", hdfsDirPath);
fileSystem.close();
}
@Override
public IndexOutput createOutput(String name, IOContext context)
throws IOException {
if (SEGMENTS_GEN.equals(name)) {
return NULL_WRITER;
}
HdfsFileWriter writer = new HdfsFileWriter(getFileSystem(), new Path(
hdfsDirPath, name));
return new HdfsIndexOutput(writer);
}
private String[] getNormalNames(List<String> files) {
int size = files.size();
for (int i = 0; i < size; i++) {
String str = files.get(i);
files.set(i, toNormalName(str));
}
return files.toArray(new String[] {});
}
private String toNormalName(String name) {
if (name.endsWith(LF_EXT)) {
return name.substring(0, name.length() - 3);
}
return name;
}
@Override
public IndexInput openInput(String name, IOContext context)
throws IOException {
return openInput(name, BUFFER_SIZE);
}
private IndexInput openInput(String name, int bufferSize) throws IOException {
return new HdfsNormalIndexInput(name, getFileSystem(), new Path(
hdfsDirPath, name), BUFFER_SIZE);
}
@Override
public void deleteFile(String name) throws IOException {
Path path = new Path(hdfsDirPath, name);
LOG.debug("Deleting {}", path);
getFileSystem().delete(path, false);
}
@Override
public boolean fileExists(String name) throws IOException {
return getFileSystem().exists(new Path(hdfsDirPath, name));
}
@Override
public long fileLength(String name) throws IOException {
return HdfsFileReader.getLength(getFileSystem(),
new Path(hdfsDirPath, name));
}
public long fileModified(String name) throws IOException {
FileStatus fileStatus = getFileSystem().getFileStatus(
new Path(hdfsDirPath, name));
return fileStatus.getModificationTime();
}
@Override
public String[] listAll() throws IOException {
FileStatus[] listStatus = getFileSystem().listStatus(hdfsDirPath);
List<String> files = new ArrayList<String>();
if (listStatus == null) {
return new String[] {};
}
for (FileStatus status : listStatus) {
if (!status.isDirectory()) {
files.add(status.getPath().getName());
}
}
return getNormalNames(files);
}
public Path getHdfsDirPath() {
return hdfsDirPath;
}
public FileSystem getFileSystem() {
return fileSystem;
}
public Configuration getConfiguration() {
return configuration;
}
static class HdfsNormalIndexInput extends CustomBufferedIndexInput {
public static Logger LOG = LoggerFactory
.getLogger(HdfsNormalIndexInput.class);
private final Path path;
private final FSDataInputStream inputStream;
private final long length;
private boolean clone = false;
public HdfsNormalIndexInput(String name, FileSystem fileSystem, Path path,
int bufferSize) throws IOException {
super(name);
this.path = path;
LOG.debug("Opening normal index input on {}", path);
FileStatus fileStatus = fileSystem.getFileStatus(path);
length = fileStatus.getLen();
inputStream = fileSystem.open(path, bufferSize);
}
@Override
protected void readInternal(byte[] b, int offset, int length)
throws IOException {
inputStream.read(getFilePointer(), b, offset, length);
}
@Override
protected void seekInternal(long pos) throws IOException {
inputStream.seek(pos);
}
@Override
protected void closeInternal() throws IOException {
LOG.debug("Closing normal index input on {}", path);
if (!clone) {
inputStream.close();
}
}
@Override
public long length() {
return length;
}
@Override
public IndexInput clone() {
HdfsNormalIndexInput clone = (HdfsNormalIndexInput) super.clone();
clone.clone = true;
return clone;
}
}
static class HdfsIndexOutput extends IndexOutput {
private HdfsFileWriter writer;
public HdfsIndexOutput(HdfsFileWriter writer) {
this.writer = writer;
}
@Override
public void close() throws IOException {
writer.close();
}
@Override
public void flush() throws IOException {
writer.flush();
}
@Override
public long getFilePointer() {
return writer.getPosition();
}
@Override
public long length() {
return writer.length();
}
@Override
public void writeByte(byte b) throws IOException {
writer.writeByte(b);
}
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
writer.writeBytes(b, offset, length);
}
}
@Override
public void sync(Collection<String> names) throws IOException {
LOG.debug("Sync called on {}", Arrays.toString(names.toArray()));
}
}

102
solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileReader.java Normal file
View File

@ -0,0 +1,102 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.store.DataInput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsFileReader extends DataInput {
public static Logger LOG = LoggerFactory.getLogger(HdfsFileReader.class);
private final Path path;
private FSDataInputStream inputStream;
private long length;
private boolean isClone;
public HdfsFileReader(FileSystem fileSystem, Path path, int bufferSize)
throws IOException {
this.path = path;
LOG.debug("Opening reader on {}", path);
if (!fileSystem.exists(path)) {
throw new FileNotFoundException(path.toString());
}
inputStream = fileSystem.open(path, bufferSize);
FileStatus fileStatus = fileSystem.getFileStatus(path);
length = fileStatus.getLen();
}
public HdfsFileReader(FileSystem fileSystem, Path path) throws IOException {
this(fileSystem, path, HdfsDirectory.BUFFER_SIZE);
}
public long length() {
return length;
}
public void seek(long pos) throws IOException {
inputStream.seek(pos);
}
public void close() throws IOException {
if (!isClone) {
inputStream.close();
}
LOG.debug("Closing reader on {}", path);
}
/**
* This method should never be used!
*/
@Override
public byte readByte() throws IOException {
LOG.warn("Should not be used!");
return inputStream.readByte();
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
while (len > 0) {
int lenRead = inputStream.read(b, offset, len);
offset += lenRead;
len -= lenRead;
}
}
public static long getLength(FileSystem fileSystem, Path path)
throws IOException {
FileStatus fileStatus = fileSystem.getFileStatus(path);
return fileStatus.getLen();
}
@Override
public DataInput clone() {
HdfsFileReader reader = (HdfsFileReader) super.clone();
reader.isClone = true;
return reader;
}
}

95
solr/core/src/java/org/apache/solr/store/hdfs/HdfsFileWriter.java Normal file
View File

@ -0,0 +1,95 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.lucene.store.DataOutput;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HdfsFileWriter extends DataOutput {
public static Logger LOG = LoggerFactory.getLogger(HdfsFileWriter.class);
public static final String HDFS_SYNC_BLOCK = "solr.hdfs.sync.block";
private final Path path;
private FSDataOutputStream outputStream;
private long currentPosition;
public HdfsFileWriter(FileSystem fileSystem, Path path) throws IOException {
LOG.debug("Creating writer on {}", path);
this.path = path;
Configuration conf = fileSystem.getConf();
FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE,
CreateFlag.OVERWRITE);
if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
flags.add(CreateFlag.SYNC_BLOCK);
}
outputStream = fileSystem.create(path, FsPermission.getDefault()
.applyUMask(FsPermission.getUMask(conf)), flags, fsDefaults
.getFileBufferSize(), fsDefaults.getReplication(), fsDefaults
.getBlockSize(), null);
}
public long length() {
return currentPosition;
}
public void seek(long pos) throws IOException {
LOG.error("Invalid seek called on {}", path);
throw new IOException("Seek not supported");
}
public void flush() throws IOException {
// flush to the network, not guarantees it makes it to the DN (vs hflush)
outputStream.flush();
LOG.debug("Flushed file {}", path);
}
public void close() throws IOException {
outputStream.close();
LOG.debug("Closed writer on {}", path);
}
@Override
public void writeByte(byte b) throws IOException {
outputStream.write(b & 0xFF);
currentPosition++;
}
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
outputStream.write(b, offset, length);
currentPosition += length;
}
public long getPosition() {
return currentPosition;
}
}

138
solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java Normal file
View File

@ -0,0 +1,138 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.LockReleaseFailedException;
import org.apache.solr.util.IOUtils;
public class HdfsLockFactory extends LockFactory {
private Path lockPath;
private Configuration configuration;
public HdfsLockFactory(Path lockPath, Configuration configuration) {
this.lockPath = lockPath;
this.configuration = configuration;
}
@Override
public Lock makeLock(String lockName) {
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
HdfsLock lock = new HdfsLock(lockPath, lockName, configuration);
return lock;
}
@Override
public void clearLock(String lockName) throws IOException {
FileSystem fs = null;
try {
fs = FileSystem.newInstance(lockPath.toUri(), configuration);
if (fs.exists(lockPath)) {
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
Path lockFile = new Path(lockPath, lockName);
if (fs.exists(lockFile) && !fs.delete(lockFile, false)) {
throw new IOException("Cannot delete " + lockFile);
}
}
} finally {
IOUtils.closeQuietly(fs);
}
}
public Path getLockPath() {
return lockPath;
}
public void setLockPath(Path lockPath) {
this.lockPath = lockPath;
}
static class HdfsLock extends Lock {
private Path lockPath;
private String lockName;
private Configuration conf;
public HdfsLock(Path lockPath, String lockName, Configuration conf) {
this.lockPath = lockPath;
this.lockName = lockName;
this.conf = conf;
}
@Override
public boolean obtain() throws IOException {
FSDataOutputStream file = null;
FileSystem fs = null;
try {
fs = FileSystem.newInstance(lockPath.toUri(), conf);
file = fs.create(new Path(lockPath, lockName), false);
} catch (IOException e) {
return false;
} finally {
IOUtils.closeQuietly(file);
IOUtils.closeQuietly(fs);
}
return true;
}
@Override
public void release() throws IOException {
FileSystem fs = FileSystem.newInstance(lockPath.toUri(), conf);
try {
if (fs.exists(new Path(lockPath, lockName))
&& !fs.delete(new Path(lockPath, lockName), false)) throw new LockReleaseFailedException(
"failed to delete " + new Path(lockPath, lockName));
} finally {
IOUtils.closeQuietly(fs);
}
}
@Override
public boolean isLocked() throws IOException {
boolean isLocked = false;
FileSystem fs = FileSystem.newInstance(lockPath.toUri(), conf);
try {
isLocked = fs.exists(new Path(lockPath, lockName));
} finally {
IOUtils.closeQuietly(fs);
}
return isLocked;
}
}
}

66
solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java Normal file
View File

@ -0,0 +1,66 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.store.IndexOutput;
public class NullIndexOutput extends IndexOutput {
private long pos;
private long length;
@Override
public void close() throws IOException {
}
@Override
public void flush() throws IOException {
}
@Override
public long getFilePointer() {
return pos;
}
@Override
public long length() throws IOException {
return length;
}
@Override
public void writeByte(byte b) throws IOException {
pos++;
}
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
pos += length;
updateLength();
}
private void updateLength() {
if (pos > length) {
length = pos;
}
}
}

29
solr/core/src/java/org/apache/solr/store/hdfs/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
An HDFS Directory implementation.
</p>
</body>
</html>

581
solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java Normal file
View File

@ -0,0 +1,581 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.DataInputInputStream;
import org.apache.solr.common.util.FastInputStream;
import org.apache.solr.common.util.FastOutputStream;
import org.apache.solr.common.util.JavaBinCodec;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Log Format: List{Operation, Version, ...}
* ADD, VERSION, DOC
* DELETE, VERSION, ID_BYTES
* DELETE_BY_QUERY, VERSION, String
*
* TODO: keep two files, one for [operation, version, id] and the other for the actual
* document data. That way we could throw away document log files more readily
* while retaining the smaller operation log files longer (and we can retrieve
* the stored fields from the latest documents from the index).
*
* This would require keeping all source fields stored of course.
*
* This would also allow to not log document data for requests with commit=true
* in them (since we know that if the request succeeds, all docs will be committed)
*
*/
public class HdfsTransactionLog extends TransactionLog {
public static Logger log = LoggerFactory.getLogger(HdfsTransactionLog.class);
Path tlogFile;
private FSDataOutputStream tlogOutStream;
private FileSystem fs;
HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings) {
this(fs, tlogFile, globalStrings, false);
}
HdfsTransactionLog(FileSystem fs, Path tlogFile, Collection<String> globalStrings, boolean openExisting) {
super();
boolean success = false;
this.fs = fs;
try {
if (debug) {
//log.debug("New TransactionLog file=" + tlogFile + ", exists=" + tlogFile.exists() + ", size=" + tlogFile.length() + ", openExisting=" + openExisting);
}
this.tlogFile = tlogFile;
// TODO: look into forcefully taking over any lease
if (fs.exists(tlogFile) && openExisting) {
tlogOutStream = fs.append(tlogFile);
} else {
fs.delete(tlogFile, false);
tlogOutStream = fs.create(tlogFile, (short)1);
tlogOutStream.hsync();
}
fos = new FastOutputStream(tlogOutStream, new byte[65536], 0);
long start = tlogOutStream.getPos();
if (openExisting) {
if (start > 0) {
readHeader(null);
// we should already be at the end
// raf.seek(start);
// assert channel.position() == start;
fos.setWritten(start); // reflect that we aren't starting at the beginning
//assert fos.size() == channel.size();
} else {
addGlobalStrings(globalStrings);
}
} else {
if (start > 0) {
log.error("New transaction log already exists:" + tlogFile + " size=" + tlogOutStream.size());
}
addGlobalStrings(globalStrings);
}
success = true;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
} finally {
if (!success && tlogOutStream != null) {
try {
tlogOutStream.close();
} catch (Exception e) {
log.error("Error closing tlog file (after error opening)", e);
}
}
}
}
@Override
public boolean endsWithCommit() throws IOException {
long size;
synchronized (this) {
fos.flush();
tlogOutStream.hflush();
size = fos.size();
}
// the end of the file should have the end message (added during a commit) plus a 4 byte size
byte[] buf = new byte[ END_MESSAGE.length() ];
long pos = size - END_MESSAGE.length() - 4;
if (pos < 0) return false;
FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile), pos);
try {
//ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
dis.read(buf);
for (int i=0; i<buf.length; i++) {
if (buf[i] != END_MESSAGE.charAt(i)) return false;
}
} finally {
dis.close();
}
return true;
}
// This could mess with any readers or reverse readers that are open, or anything that might try to do a log lookup.
// This should only be used to roll back buffered updates, not actually applied updates.
@Override
public void rollback(long pos) throws IOException {
synchronized (this) {
assert snapshot_size == pos;
fos.flush();
tlogOutStream.hflush();
// TODO: how do we rollback with hdfs?? We need HDFS-3107
//raf.setLength(pos);
fos.setWritten(pos);
assert fos.size() == pos;
numRecords = snapshot_numRecords;
}
}
private void readHeader(FastInputStream fis) throws IOException {
// read existing header
boolean closeFis = false;
if (fis == null) closeFis = true;
fis = fis != null ? fis : new FSDataFastInputStream(fs.open(tlogFile), 0);
Map header = null;
try {
LogCodec codec = new LogCodec(resolver);
header = (Map) codec.unmarshal(fis);
fis.readInt(); // skip size
} finally {
if (fis != null && closeFis) {
fis.close();
}
}
// needed to read other records
synchronized (this) {
globalStringList = (List<String>)header.get("strings");
globalStringMap = new HashMap<String, Integer>(globalStringList.size());
for (int i=0; i<globalStringList.size(); i++) {
globalStringMap.put( globalStringList.get(i), i+1);
}
}
}
@Override
public long writeCommit(CommitUpdateCommand cmd, int flags) {
LogCodec codec = new LogCodec(resolver);
synchronized (this) {
try {
long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
if (pos == 0) {
writeLogHeader(codec);
pos = fos.size();
}
codec.init(fos);
codec.writeTag(JavaBinCodec.ARR, 3);
codec.writeInt(UpdateLog.COMMIT | flags); // should just take one byte
codec.writeLong(cmd.getVersion());
codec.writeStr(END_MESSAGE); // ensure these bytes are (almost) last in the file
endRecord(pos);
fos.flush(); // flush since this will be the last record in a log fill
tlogOutStream.hflush();
//assert fos.size() == channel.size();
return pos;
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
}
/* This method is thread safe */
@Override
public Object lookup(long pos) {
// A negative position can result from a log replay (which does not re-log, but does
// update the version map. This is OK since the node won't be ACTIVE when this happens.
if (pos < 0) return null;
try {
// make sure any unflushed buffer has been flushed
synchronized (this) {
// TODO: optimize this by keeping track of what we have flushed up to
fos.flushBuffer();
// flush to hdfs
tlogOutStream.hflush();
/***
System.out.println("###flushBuffer to " + fos.size() + " raf.length()=" + raf.length() + " pos="+pos);
if (fos.size() != raf.length() || pos >= fos.size() ) {
throw new RuntimeException("ERROR" + "###flushBuffer to " + fos.size() + " raf.length()=" + raf.length() + " pos="+pos);
}
***/
}
FSDataFastInputStream dis = new FSDataFastInputStream(fs.open(tlogFile),
pos);
try {
dis.seek(pos);
LogCodec codec = new LogCodec(resolver);
return codec.readVal(new FastInputStream(dis));
} finally {
dis.close();
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "pos=" + pos, e);
}
}
@Override
public void finish(UpdateLog.SyncLevel syncLevel) {
if (syncLevel == UpdateLog.SyncLevel.NONE) return;
try {
synchronized (this) {
fos.flushBuffer();
// we must flush to hdfs
// TODO: we probably don't need to
// hsync below if we do this - I
// think they are equivalent.
tlogOutStream.hflush();
}
if (syncLevel == UpdateLog.SyncLevel.FSYNC) {
// Since fsync is outside of synchronized block, we can end up with a partial
// last record on power failure (which is OK, and does not represent an error...
// we just need to be aware of it when reading).
//raf.getFD().sync();
tlogOutStream.hsync();
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
@Override
protected void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
}
synchronized (this) {
fos.flush();
tlogOutStream.hflush();
fos.close();
tlogOutStream.close();
}
if (deleteOnClose) {
fs.delete(tlogFile, true);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
public String toString() {
return "hdfs tlog{file=" + tlogFile.toString() + " refcount=" + refcount.get() + "}";
}
/** Returns a reader that can be used while a log is still in use.
* Currently only *one* LogReader may be outstanding, and that log may only
* be used from a single thread. */
@Override
public LogReader getReader(long startingPos) {
return new HDFSLogReader(startingPos);
}
/** Returns a single threaded reverse reader */
@Override
public ReverseReader getReverseReader() throws IOException {
return new HDFSReverseReader();
}
public class HDFSLogReader extends LogReader{
FSDataFastInputStream fis;
private LogCodec codec = new LogCodec(resolver);
public HDFSLogReader(long startingPos) {
super();
incref();
try {
FSDataInputStream fdis = fs.open(tlogFile);
fis = new FSDataFastInputStream(fdis, startingPos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
* @throws IOException If there is a low-level I/O error.
*/
public Object next() throws IOException, InterruptedException {
long pos = fis.position();
synchronized (HdfsTransactionLog.this) {
if (trace) {
log.trace("Reading log record. pos="+pos+" currentSize="+fos.size());
}
if (pos >= fos.size()) {
return null;
}
fos.flushBuffer();
tlogOutStream.hflush();
// we actually need a new reader
fis.close();
try {
FSDataInputStream fdis = fs.open(tlogFile);
fis = new FSDataFastInputStream(fdis, pos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (pos == 0) {
readHeader(fis);
// shouldn't currently happen - header and first record are currently written at the same time
synchronized (HdfsTransactionLog.this) {
if (fis.position() >= fos.size()) {
return null;
}
pos = fis.position();
}
}
tlogOutStream.hflush();
Object o = codec.readVal(fis);
// skip over record size
int size = fis.readInt();
assert size == fis.position() - pos - 4;
return o;
}
public void close() {
try {
fis.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
decref();
}
@Override
public String toString() {
synchronized (HdfsTransactionLog.this) {
return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + fos.size() + "}";
}
}
}
public class HDFSReverseReader extends ReverseReader {
FSDataFastInputStream fis;
private LogCodec codec = new LogCodec(resolver) {
@Override
public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
// Given that the SolrInputDocument is last in an add record, it's OK to just skip
// reading it completely.
return null;
}
};
int nextLength; // length of the next record (the next one closer to the start of the log file)
long prevPos; // where we started reading from last time (so prevPos - nextLength == start of next record)
public HDFSReverseReader() throws IOException {
incref();
long sz;
synchronized (HdfsTransactionLog.this) {
fos.flushBuffer();
// this must be an hflush
tlogOutStream.hflush();
sz = fos.size();
//assert sz == channel.size();
}
fis = new FSDataFastInputStream(fs.open(tlogFile), 0);
if (sz >=4) {
// readHeader(fis); // should not be needed
prevPos = sz - 4;
fis.seek(prevPos);
nextLength = fis.readInt();
}
}
/** Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
* @throws IOException If there is a low-level I/O error.
*/
public Object next() throws IOException {
if (prevPos <= 0) return null;
long endOfThisRecord = prevPos;
int thisLength = nextLength;
long recordStart = prevPos - thisLength; // back up to the beginning of the next record
prevPos = recordStart - 4; // back up 4 more to read the length of the next record
if (prevPos <= 0) return null; // this record is the header
long bufferPos = fis.getBufferPos();
if (prevPos >= bufferPos) {
// nothing to do... we're within the current buffer
} else {
// Position buffer so that this record is at the end.
// For small records, this will cause subsequent calls to next() to be within the buffer.
long seekPos = endOfThisRecord - fis.getBufferSize();
seekPos = Math.min(seekPos, prevPos); // seek to the start of the record if it's larger then the block size.
seekPos = Math.max(seekPos, 0);
fis.seek(seekPos);
fis.peek(); // cause buffer to be filled
}
fis.seek(prevPos);
nextLength = fis.readInt(); // this is the length of the *next* record (i.e. closer to the beginning)
// TODO: optionally skip document data
Object o = codec.readVal(fis);
// assert fis.position() == prevPos + 4 + thisLength; // this is only true if we read all the data (and we currently skip reading SolrInputDocument
return o;
}
/* returns the position in the log file of the last record returned by next() */
public long position() {
return prevPos + 4; // skip the length
}
public void close() {
try {
fis.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
decref();
}
@Override
public String toString() {
synchronized (HdfsTransactionLog.this) {
return "LogReader{" + "file=" + tlogFile + ", position=" + fis.position() + ", end=" + fos.size() + "}";
}
}
}
}
class FSDataFastInputStream extends FastInputStream {
private FSDataInputStream fis;
public FSDataFastInputStream(FSDataInputStream fis, long chPosition) {
// super(null, new byte[10],0,0); // a small buffer size for testing purposes
super(null);
this.fis = fis;
super.readFromStream = chPosition;
}
@Override
public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
return fis.read(readFromStream, target, offset, len);
}
public void seek(long position) throws IOException {
if (position <= readFromStream && position >= getBufferPos()) {
// seek within buffer
pos = (int)(position - getBufferPos());
} else {
// long currSize = ch.size(); // not needed - underlying read should handle (unless read never done)
// if (position > currSize) throw new EOFException("Read past EOF: seeking to " + position + " on file of size " + currSize + " file=" + ch);
readFromStream = position;
end = pos = 0;
}
assert position() == position;
}
/** where is the start of the buffer relative to the whole file */
public long getBufferPos() {
return readFromStream - end;
}
public int getBufferSize() {
return buf.length;
}
@Override
public void close() throws IOException {
fis.close();
}
@Override
public String toString() {
return "readFromStream="+readFromStream +" pos="+pos +" end="+end + " bufferPos="+getBufferPos() + " position="+position() ;
}
}

354
solr/core/src/java/org/apache/solr/update/HdfsUpdateLog.java Normal file
View File

@ -0,0 +1,354 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.util.HdfsUtil;
import org.apache.solr.util.IOUtils;
/** @lucene.experimental */
public class HdfsUpdateLog extends UpdateLog {
private FileSystem fs;
private Path tlogDir;
private String confDir;
public HdfsUpdateLog() {
}
public HdfsUpdateLog(String confDir) {
this.confDir = confDir;
}
public FileSystem getFs() {
return fs;
}
// HACK
// while waiting for HDFS-3107, instead of quickly
// dropping, we slowly apply
// This is somewhat brittle, but current usage
// allows for it
@Override
public boolean dropBufferedUpdates() {
Future<RecoveryInfo> future = applyBufferedUpdates();
if (future != null) {
try {
future.get();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
return true;
}
@Override
public void init(PluginInfo info) {
dataDir = (String) info.initArgs.get("dir");
defaultSyncLevel = SyncLevel.getSyncLevel((String) info.initArgs
.get("syncLevel"));
}
private Configuration getConf() {
Configuration conf = new Configuration();
if (confDir != null) {
HdfsUtil.addHdfsResources(conf, confDir);
}
return conf;
}
@Override
public void init(UpdateHandler uhandler, SolrCore core) {
// ulogDir from CoreDescriptor overrides
String ulogDir = core.getCoreDescriptor().getUlogDir();
if (ulogDir != null) {
dataDir = ulogDir;
}
if (dataDir == null || dataDir.length()==0) {
dataDir = core.getDataDir();
}
if (!core.getDirectoryFactory().isAbsolute(dataDir)) {
try {
dataDir = core.getDirectoryFactory().getDataHome(core.getCoreDescriptor());
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
}
try {
fs = FileSystem.newInstance(new Path(dataDir).toUri(), getConf());
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
this.uhandler = uhandler;
if (dataDir.equals(lastDataDir)) {
if (debug) {
log.debug("UpdateHandler init: tlogDir=" + tlogDir + ", next id=" + id,
" this is a reopen... nothing else to do.");
}
versionInfo.reload();
// on a normal reopen, we currently shouldn't have to do anything
return;
}
lastDataDir = dataDir;
tlogDir = new Path(dataDir, TLOG_NAME);
try {
if (!fs.exists(tlogDir)) {
boolean success = fs.mkdirs(tlogDir);
if (!success) {
throw new RuntimeException("Could not create directory:" + tlogDir);
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
tlogFiles = getLogList(fs, tlogDir);
id = getLastLogId() + 1; // add 1 since we will create a new log for the
// next update
if (debug) {
log.debug("UpdateHandler init: tlogDir=" + tlogDir + ", existing tlogs="
+ Arrays.asList(tlogFiles) + ", next id=" + id);
}
TransactionLog oldLog = null;
for (String oldLogName : tlogFiles) {
Path f = new Path(tlogDir, oldLogName);
try {
oldLog = new HdfsTransactionLog(fs, f, null, true);
addOldLog(oldLog, false); // don't remove old logs on startup since more
// than one may be uncapped.
} catch (Exception e) {
SolrException.log(log, "Failure to open existing log file (non fatal) "
+ f, e);
try {
fs.delete(f, false);
} catch (IOException e1) {
throw new RuntimeException(e1);
}
}
}
// Record first two logs (oldest first) at startup for potential tlog
// recovery.
// It's possible that at abnormal shutdown both "tlog" and "prevTlog" were
// uncapped.
for (TransactionLog ll : logs) {
newestLogsOnStartup.addFirst(ll);
if (newestLogsOnStartup.size() >= 2) break;
}
try {
versionInfo = new VersionInfo(this, 256);
} catch (SolrException e) {
log.error("Unable to use updateLog: " + e.getMessage(), e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unable to use updateLog: " + e.getMessage(), e);
}
// TODO: these startingVersions assume that we successfully recover from all
// non-complete tlogs.
HdfsUpdateLog.RecentUpdates startingUpdates = getRecentUpdates();
try {
startingVersions = startingUpdates.getVersions(numRecordsToKeep);
startingOperation = startingUpdates.getLatestOperation();
// populate recent deletes list (since we can't get that info from the
// index)
for (int i = startingUpdates.deleteList.size() - 1; i >= 0; i--) {
DeleteUpdate du = startingUpdates.deleteList.get(i);
oldDeletes.put(new BytesRef(du.id), new LogPtr(-1, du.version));
}
// populate recent deleteByQuery commands
for (int i = startingUpdates.deleteByQueryList.size() - 1; i >= 0; i--) {
Update update = startingUpdates.deleteByQueryList.get(i);
List<Object> dbq = (List<Object>) update.log.lookup(update.pointer);
long version = (Long) dbq.get(1);
String q = (String) dbq.get(2);
trackDeleteByQuery(q, version);
}
} finally {
startingUpdates.close();
}
}
@Override
public String getLogDir() {
return tlogDir.toUri().toString();
}
public static String[] getLogList(FileSystem fs, Path tlogDir) {
final String prefix = TLOG_NAME + '.';
assert fs != null;
FileStatus[] fileStatuses;
try {
fileStatuses = fs.listStatus(tlogDir, new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().startsWith(prefix);
}
});
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
}
String[] names = new String[fileStatuses.length];
for (int i = 0; i < fileStatuses.length; i++) {
names[i] = fileStatuses[i].getPath().getName();
}
Arrays.sort(names);
return names;
}
@Override
public void close(boolean committed) {
synchronized (this) {
super.close(committed);
IOUtils.closeQuietly(fs);
}
}
@Override
protected void ensureLog() {
if (tlog == null) {
String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN,
TLOG_NAME, id);
tlog = new HdfsTransactionLog(fs, new Path(tlogDir, newLogName),
globalStrings);
}
}
/**
* Clears the logs on the file system. Only call before init.
*
* @param core the SolrCore
* @param ulogPluginInfo the init info for the UpdateHandler
*/
@Override
public void clearLog(SolrCore core, PluginInfo ulogPluginInfo) {
if (ulogPluginInfo == null) return;
Path tlogDir = new Path(getTlogDir(core, ulogPluginInfo));
try {
if (fs.exists(tlogDir)) {
String[] files = getLogList(tlogDir);
for (String file : files) {
Path f = new Path(tlogDir, file);
boolean s = fs.delete(f, false);
if (!s) {
log.error("Could not remove tlog file:" + f);
}
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private String[] getLogList(Path tlogDir) throws FileNotFoundException, IOException {
final String prefix = TLOG_NAME+'.';
FileStatus[] files = fs.listStatus(tlogDir, new PathFilter() {
@Override
public boolean accept(Path name) {
return name.getName().startsWith(prefix);
}
});
List<String> fileList = new ArrayList<String>(files.length);
for (FileStatus file : files) {
fileList.add(file.getPath().getName());
}
return fileList.toArray(new String[0]);
}
/**
* Returns true if we were able to drop buffered updates and return to the
* ACTIVE state
*/
// public boolean dropBufferedUpdates() {
// versionInfo.blockUpdates();
// try {
// if (state != State.BUFFERING) return false;
//
// if (log.isInfoEnabled()) {
// log.info("Dropping buffered updates " + this);
// }
//
// // since we blocked updates, this synchronization shouldn't strictly be
// necessary.
// synchronized (this) {
// if (tlog != null) {
// tlog.rollback(recoveryInfo.positionOfStart);
// }
// }
//
// state = State.ACTIVE;
// operationFlags &= ~FLAG_GAP;
// } catch (IOException e) {
// SolrException.log(log,"Error attempting to roll back log", e);
// return false;
// }
// finally {
// versionInfo.unblockUpdates();
// }
// return true;
// }
public String toString() {
return "HDFSUpdateLog{state=" + getState() + ", tlog=" + tlog + "}";
}
}

65
solr/core/src/java/org/apache/solr/update/TransactionLog.java
View File

@ -74,7 +74,7 @@ public class TransactionLog {
FastOutputStream fos; // all accesses to this stream should be synchronized on "this" (The TransactionLog)
int numRecords;
volatile boolean deleteOnClose = true; // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
protected volatile boolean deleteOnClose = true; // we can delete old tlogs since they are currently only used for real-time-get (and in the future, recovery)
AtomicInteger refcount = new AtomicInteger(1);
Map<String,Integer> globalStringMap = new HashMap<String, Integer>();
@ -97,7 +97,7 @@ public class TransactionLog {
};
public class LogCodec extends JavaBinCodec {
public LogCodec() {
public LogCodec(JavaBinCodec.ObjectResolver resolver) {
super(resolver);
}
@ -190,6 +190,9 @@ public class TransactionLog {
}
}
// for subclasses
protected TransactionLog() {}
/** Returns the number of records in the log (currently includes the header and an optional commit).
* Note: currently returns 0 for reopened existing log files.
*/
@ -244,7 +247,7 @@ public class TransactionLog {
public long writeData(Object o) {
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
try {
long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
codec.init(fos);
@ -259,7 +262,7 @@ public class TransactionLog {
private void readHeader(FastInputStream fis) throws IOException {
// read existing header
fis = fis != null ? fis : new ChannelFastInputStream(channel, 0);
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
Map header = (Map)codec.unmarshal(fis);
fis.readInt(); // skip size
@ -275,7 +278,7 @@ public class TransactionLog {
}
}
private void addGlobalStrings(Collection<String> strings) {
protected void addGlobalStrings(Collection<String> strings) {
if (strings == null) return;
int origSize = globalStringMap.size();
for (String s : strings) {
@ -296,7 +299,7 @@ public class TransactionLog {
}
}
private void writeLogHeader(LogCodec codec) throws IOException {
protected void writeLogHeader(LogCodec codec) throws IOException {
long pos = fos.size();
assert pos == 0;
@ -308,7 +311,7 @@ public class TransactionLog {
endRecord(pos);
}
private void endRecord(long startRecordPosition) throws IOException {
protected void endRecord(long startRecordPosition) throws IOException {
fos.writeInt((int)(fos.size() - startRecordPosition));
numRecords++;
}
@ -332,7 +335,7 @@ public class TransactionLog {
int lastAddSize;
public long write(AddUpdateCommand cmd, int flags) {
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
SolrInputDocument sdoc = cmd.getSolrInputDocument();
try {
@ -374,7 +377,7 @@ public class TransactionLog {
}
public long writeDelete(DeleteUpdateCommand cmd, int flags) {
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
try {
checkWriteHeader(codec, null);
@ -404,7 +407,7 @@ public class TransactionLog {
}
public long writeDeleteByQuery(DeleteUpdateCommand cmd, int flags) {
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
try {
checkWriteHeader(codec, null);
@ -430,7 +433,7 @@ public class TransactionLog {
public long writeCommit(CommitUpdateCommand cmd, int flags) {
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
synchronized (this) {
try {
long pos = fos.size(); // if we had flushed, this should be equal to channel.position()
@ -478,7 +481,7 @@ public class TransactionLog {
}
ChannelFastInputStream fis = new ChannelFastInputStream(channel, pos);
LogCodec codec = new LogCodec();
LogCodec codec = new LogCodec(resolver);
return codec.readVal(fis);
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
@ -528,7 +531,7 @@ public class TransactionLog {
}
}
private void close() {
protected void close() {
try {
if (debug) {
log.debug("Closing tlog" + this);
@ -569,19 +572,22 @@ public class TransactionLog {
/** Returns a single threaded reverse reader */
public ReverseReader getReverseReader() throws IOException {
return new ReverseReader();
return new FSReverseReader();
}
public class LogReader {
ChannelFastInputStream fis;
private LogCodec codec = new LogCodec();
private ChannelFastInputStream fis;
private LogCodec codec = new LogCodec(resolver);
public LogReader(long startingPos) {
incref();
fis = new ChannelFastInputStream(channel, startingPos);
}
// for classes that extend
protected LogReader() {}
/** Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
@ -637,9 +643,30 @@ public class TransactionLog {
}
public class ReverseReader {
public abstract class ReverseReader {
/** Returns the next object from the log, or null if none available.
*
* @return The log record, or null if EOF
* @throws IOException If there is a low-level I/O error.
*/
public abstract Object next() throws IOException;
/* returns the position in the log file of the last record returned by next() */
public abstract long position();
public abstract void close();
@Override
public abstract String toString() ;
}
public class FSReverseReader extends ReverseReader {
ChannelFastInputStream fis;
private LogCodec codec = new LogCodec() {
private LogCodec codec = new LogCodec(resolver) {
@Override
public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) {
// Given that the SolrInputDocument is last in an add record, it's OK to just skip
@ -651,7 +678,7 @@ public class TransactionLog {
int nextLength; // length of the next record (the next one closer to the start of the log file)
long prevPos; // where we started reading from last time (so prevPos - nextLength == start of next record)
public ReverseReader() throws IOException {
public FSReverseReader() throws IOException {
incref();
long sz;

55
solr/core/src/java/org/apache/solr/update/UpdateHandler.java
View File

@ -18,10 +18,11 @@
package org.apache.solr.update;
import java.io.File;
import java.io.IOException;
import java.util.Vector;
import org.apache.solr.core.DirectoryFactory;
import org.apache.solr.core.HdfsDirectoryFactory;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrEventListener;
@ -71,24 +72,6 @@ public abstract class UpdateHandler implements SolrInfoMBean {
}
}
// not thread safe - for startup
private void clearLog(PluginInfo ulogPluginInfo) {
if (ulogPluginInfo == null) return;
File tlogDir = UpdateLog.getTlogDir(core, ulogPluginInfo);
log.info("Clearing tlog files, tlogDir=" + tlogDir);
if (tlogDir.exists()) {
String[] files = UpdateLog.getLogList(tlogDir);
for (String file : files) {
File f = new File(tlogDir, file);
boolean s = f.delete();
if (!s) {
log.error("Could not remove tlog file:" + f.getAbsolutePath());
//throw new SolrException(ErrorCode.SERVER_ERROR, "Could not remove tlog file:" + f.getAbsolutePath());
}
}
}
}
protected void callPostCommitCallbacks() {
for (SolrEventListener listener : commitCallbacks) {
listener.postCommit();
@ -117,19 +100,43 @@ public abstract class UpdateHandler implements SolrInfoMBean {
idFieldType = idField!=null ? idField.getType() : null;
parseEventListeners();
PluginInfo ulogPluginInfo = core.getSolrConfig().getPluginInfo(UpdateLog.class.getName());
if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
clearLog(ulogPluginInfo);
}
if (updateLog == null && ulogPluginInfo != null && ulogPluginInfo.isEnabled()) {
ulog = new UpdateLog();
String dataDir = (String)ulogPluginInfo.initArgs.get("dir");
String ulogDir = core.getCoreDescriptor().getUlogDir();
if (ulogDir != null) {
dataDir = ulogDir;
}
if (dataDir == null || dataDir.length()==0) {
dataDir = core.getDataDir();
}
if (dataDir != null && dataDir.startsWith("hdfs:/")) {
DirectoryFactory dirFactory = core.getDirectoryFactory();
if (dirFactory instanceof HdfsDirectoryFactory) {
ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
} else {
ulog = new HdfsUpdateLog();
}
} else {
ulog = new UpdateLog();
}
if (!core.isReloaded() && !core.getDirectoryFactory().isPersistent()) {
ulog.clearLog(core, ulogPluginInfo);
}
ulog.init(ulogPluginInfo);
// ulog = core.createInitInstance(ulogPluginInfo, UpdateLog.class, "update log", "solr.NullUpdateLog");
ulog.init(this, core);
} else {
ulog = updateLog;
}
// ulog.init() when reusing an existing log is deferred (currently at the end of the DUH2 constructor
}
/**

139
solr/core/src/java/org/apache/solr/update/UpdateLog.java
View File

@ -17,15 +17,38 @@
package org.apache.solr.update;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Future;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.fs.FileSystem;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
@ -34,9 +57,6 @@ import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributedUpdateProcessorFactory;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.update.processor.RunUpdateProcessorFactory;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.DefaultSolrThreadFactory;
@ -45,15 +65,6 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase.FROMLEADER;
/** @lucene.experimental */
public class UpdateLog implements PluginInfoInitialized {
@ -64,6 +75,10 @@ public class UpdateLog implements PluginInfoInitialized {
public boolean debug = log.isDebugEnabled();
public boolean trace = log.isTraceEnabled();
// TODO: hack
public FileSystem getFs() {
return null;
}
public enum SyncLevel { NONE, FLUSH, FSYNC;
public static SyncLevel getSyncLevel(String level){
@ -108,27 +123,27 @@ public class UpdateLog implements PluginInfoInitialized {
}
long id = -1;
private State state = State.ACTIVE;
private int operationFlags; // flags to write in the transaction log with operations (i.e. FLAG_GAP)
protected State state = State.ACTIVE;
protected int operationFlags; // flags to write in the transaction log with operations (i.e. FLAG_GAP)
private TransactionLog tlog;
private TransactionLog prevTlog;
private Deque<TransactionLog> logs = new LinkedList<TransactionLog>(); // list of recent logs, newest first
private LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
private int numOldRecords; // number of records in the recent logs
protected TransactionLog tlog;
protected TransactionLog prevTlog;
protected Deque<TransactionLog> logs = new LinkedList<TransactionLog>(); // list of recent logs, newest first
protected LinkedList<TransactionLog> newestLogsOnStartup = new LinkedList<TransactionLog>();
protected int numOldRecords; // number of records in the recent logs
private Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
private Map<BytesRef,LogPtr> prevMap; // used while committing/reopening is happening
private Map<BytesRef,LogPtr> prevMap2; // used while committing/reopening is happening
private TransactionLog prevMapLog; // the transaction log used to look up entries found in prevMap
private TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap
protected Map<BytesRef,LogPtr> map = new HashMap<BytesRef, LogPtr>();
protected Map<BytesRef,LogPtr> prevMap; // used while committing/reopening is happening
protected Map<BytesRef,LogPtr> prevMap2; // used while committing/reopening is happening
protected TransactionLog prevMapLog; // the transaction log used to look up entries found in prevMap
protected TransactionLog prevMapLog2; // the transaction log used to look up entries found in prevMap
private final int numDeletesToKeep = 1000;
private final int numDeletesByQueryToKeep = 100;
protected final int numDeletesToKeep = 1000;
protected final int numDeletesByQueryToKeep = 100;
public final int numRecordsToKeep = 100;
// keep track of deletes only... this is not updated on an add
private LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
protected LinkedHashMap<BytesRef, LogPtr> oldDeletes = new LinkedHashMap<BytesRef, LogPtr>(numDeletesToKeep) {
@Override
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > numDeletesToKeep;
@ -145,21 +160,21 @@ public class UpdateLog implements PluginInfoInitialized {
}
}
private LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
protected LinkedList<DBQ> deleteByQueries = new LinkedList<DBQ>();
private String[] tlogFiles;
private File tlogDir;
private Collection<String> globalStrings;
protected String[] tlogFiles;
protected File tlogDir;
protected Collection<String> globalStrings;
private String dataDir;
private String lastDataDir;
protected String dataDir;
protected String lastDataDir;
private VersionInfo versionInfo;
protected VersionInfo versionInfo;
private SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
protected SyncLevel defaultSyncLevel = SyncLevel.FLUSH;
volatile UpdateHandler uhandler; // a core reload can change this reference!
private volatile boolean cancelApplyBufferUpdate;
protected volatile boolean cancelApplyBufferUpdate;
List<Long> startingVersions;
int startingOperation; // last operation in the logs on startup
@ -199,7 +214,7 @@ public class UpdateLog implements PluginInfoInitialized {
if (ulogDir != null) {
dataDir = ulogDir;
}
if (dataDir == null || dataDir.length()==0) {
dataDir = core.getDataDir();
}
@ -280,8 +295,8 @@ public class UpdateLog implements PluginInfoInitialized {
}
public File getLogDir() {
return tlogDir;
public String getLogDir() {
return tlogDir.getAbsolutePath();
}
public List<Long> getStartingVersions() {
@ -295,7 +310,7 @@ public class UpdateLog implements PluginInfoInitialized {
/* Takes over ownership of the log, keeping it until no longer needed
and then decrementing it's reference and dropping it.
*/
private void addOldLog(TransactionLog oldLog, boolean removeOld) {
protected void addOldLog(TransactionLog oldLog, boolean removeOld) {
if (oldLog == null) return;
numOldRecords += oldLog.numRecords();
@ -326,7 +341,7 @@ public class UpdateLog implements PluginInfoInitialized {
}
public static String[] getLogList(File directory) {
public String[] getLogList(File directory) {
final String prefix = TLOG_NAME+'.';
String[] names = directory.list(new FilenameFilter() {
@Override
@ -334,6 +349,9 @@ public class UpdateLog implements PluginInfoInitialized {
return name.startsWith(prefix);
}
});
if (names == null) {
throw new RuntimeException(new FileNotFoundException(directory.getAbsolutePath()));
}
Arrays.sort(names);
return names;
}
@ -544,7 +562,7 @@ public class UpdateLog implements PluginInfoInitialized {
}
}
private void newMap() {
protected void newMap() {
prevMap2 = prevMap;
prevMapLog2 = prevMapLog;
@ -797,7 +815,7 @@ public class UpdateLog implements PluginInfoInitialized {
}
private void ensureLog() {
protected void ensureLog() {
if (tlog == null) {
String newLogName = String.format(Locale.ROOT, LOG_FILENAME_PATTERN, TLOG_NAME, id);
tlog = new TransactionLog(new File(tlogDir, newLogName), globalStrings);
@ -1145,7 +1163,7 @@ public class UpdateLog implements PluginInfoInitialized {
private RecoveryInfo recoveryInfo;
protected RecoveryInfo recoveryInfo;
class LogReplayer implements Runnable {
private Logger loglog = log; // set to something different?
@ -1422,7 +1440,7 @@ public class UpdateLog implements PluginInfoInitialized {
}
}
public static File getTlogDir(SolrCore core, PluginInfo info) {
protected String getTlogDir(SolrCore core, PluginInfo info) {
String dataDir = (String) info.initArgs.get("dir");
String ulogDir = core.getCoreDescriptor().getUlogDir();
@ -1433,11 +1451,30 @@ public class UpdateLog implements PluginInfoInitialized {
if (dataDir == null || dataDir.length() == 0) {
dataDir = core.getDataDir();
}
return new File(dataDir, TLOG_NAME);
return dataDir + "/" + TLOG_NAME;
}
/**
* Clears the logs on the file system. Only call before init.
*
* @param core the SolrCore
* @param ulogPluginInfo the init info for the UpdateHandler
*/
public void clearLog(SolrCore core, PluginInfo ulogPluginInfo) {
if (ulogPluginInfo == null) return;
File tlogDir = new File(getTlogDir(core, ulogPluginInfo));
if (tlogDir.exists()) {
String[] files = getLogList(tlogDir);
for (String file : files) {
File f = new File(tlogDir, file);
boolean s = f.delete();
if (!s) {
log.error("Could not remove tlog file:" + f);
}
}
}
}
}

25
solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
View File

@ -222,9 +222,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// Replica leader = slice.getLeader();
Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(
collection, shardId);
ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(leaderReplica);
String coreNodeName = zkController.getCoreNodeName(req.getCore().getCoreDescriptor());
isLeader = coreNodeName.equals(leaderReplica.getName());
isLeader = leaderReplica.getName().equals(
req.getCore().getCoreDescriptor().getCloudDescriptor()
.getCoreNodeName());
DistribPhase phase =
DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
@ -240,7 +240,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// so get the replicas...
forwardToLeader = false;
List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader()
.getReplicaProps(collection, shardId, coreNodeName,
.getReplicaProps(collection, shardId, leaderReplica.getName(),
coreName, null, ZkStateReader.DOWN);
if (replicaProps != null) {
@ -272,7 +272,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
} else {
// I need to forward onto the leader...
nodes = new ArrayList<Node>(1);
nodes.add(new RetryNode(leaderProps, zkController.getZkStateReader(), collection, shardId));
nodes.add(new RetryNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId));
forwardToLeader = true;
}
@ -343,7 +343,9 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
if (isLeader && !localIsLeader) {
log.error("ClusterState says we are the leader, but locally we don't think so");
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "ClusterState says we are the leader, but locally we don't think so");
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE,
"ClusterState says we are the leader (" + zkController.getBaseUrl()
+ "/" + req.getCore().getName() + "), but locally we don't think so. Request came from " + from);
}
}
@ -356,16 +358,15 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
try {
Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(
collection, shardId);
String leaderCoreNodeName = leaderReplica.getName();
String coreNodeName = zkController.getCoreNodeName(req.getCore().getCoreDescriptor());
isLeader = coreNodeName.equals(leaderCoreNodeName);
isLeader = leaderReplica.getName().equals(
req.getCore().getCoreDescriptor().getCloudDescriptor()
.getCoreNodeName());
// TODO: what if we are no longer the leader?
forwardToLeader = false;
List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader()
.getReplicaProps(collection, shardId, coreNodeName,
.getReplicaProps(collection, shardId, leaderReplica.getName(),
req.getCore().getName());
if (replicaProps != null) {
nodes = new ArrayList<Node>(replicaProps.size());
@ -897,7 +898,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// Am I the leader for this slice?
ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leader);
String leaderCoreNodeName = leader.getName();
String coreNodeName = zkController.getCoreNodeName(req.getCore().getCoreDescriptor());
String coreNodeName = req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName();
isLeader = coreNodeName.equals(leaderCoreNodeName);
if (isLeader) {

51
solr/core/src/java/org/apache/solr/util/HdfsUtil.java Normal file
View File

@ -0,0 +1,51 @@
package org.apache.solr.util;
import java.io.File;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class HdfsUtil {
private static final String[] HADOOP_CONF_FILES = {"core-site.xml",
"hdfs-site.xml", "mapred-site.xml", "yarn-site.xml", "hadoop-site.xml"};
public static void addHdfsResources(Configuration conf, String confDir) {
if (confDir != null && confDir.length() > 0) {
File confDirFile = new File(confDir);
if (!confDirFile.exists()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Resource directory does not exist: " + confDirFile.getAbsolutePath());
}
if (!confDirFile.isDirectory()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Specified resource directory is not a directory" + confDirFile.getAbsolutePath());
}
if (!confDirFile.canRead()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Resource directory must be readable by the Solr process: " + confDirFile.getAbsolutePath());
}
for (String file : HADOOP_CONF_FILES) {
if (new File(confDirFile, file).exists()) {
conf.addResource(new Path(confDir, file));
}
}
}
}
}

38
solr/core/src/java/org/apache/solr/util/IOUtils.java Normal file
View File

@ -0,0 +1,38 @@
package org.apache.solr.util;
import java.io.Closeable;
import org.apache.solr.core.HdfsDirectoryFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class IOUtils {
public static Logger LOG = LoggerFactory.getLogger(IOUtils.class);
public static void closeQuietly(Closeable closeable) {
try {
if (closeable != null) {
closeable.close();
}
} catch (Exception e) {
LOG.error("Error while closing", e);
}
}
}

1
solr/core/src/test-files/log4j.properties
View File

@ -7,3 +7,4 @@ log4j.appender.CONSOLE.layout=org.apache.solr.util.SolrLogLayout
log4j.appender.CONSOLE.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.hadoop=WARN

3
solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
View File

@ -24,7 +24,10 @@
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
<!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
<bool name="solr.hdfs.blockcache.enabled">${solr.hdfs.blockcache.enabled:true}</bool>
<int name="solr.hdfs.blockcache.blocksperbank">${solr.hdfs.blockcache.blocksperbank:1024}</int>
<str name="solr.hdfs.home">${solr.hdfs.home:}</str>
<str name="solr.hdfs.confdir">${solr.hdfs.confdir:}</str>
</directoryFactory>
<dataDir>${solr.data.dir:}</dataDir>

3
solr/core/src/test-files/solr/solr-no-core.xml
View File

@ -25,6 +25,7 @@
<str name="hostContext">${hostContext:solr}</str>
<int name="hostPort">${hostPort:8983}</int>
<int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>
<bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
<int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:15000}</int>
<int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:120000}</int>
</solrcloud>
@ -35,4 +36,4 @@
<int name="connTimeout">${connTimeout:15000}</int>
</shardHandlerFactory>
</solr>
</solr>

1
solr/core/src/test-files/solr/solr.xml
View File

@ -30,6 +30,7 @@
-->
<cores adminPath="/admin/cores" defaultCoreName="collection1" host="127.0.0.1" hostPort="${hostPort:8983}"
hostContext="${hostContext:solr}" zkClientTimeout="${solr.zkclienttimeout:30000}" numShards="${numShards:3}" shareSchema="${shareSchema:false}"
genericCoreNodeNames="${genericCoreNodeNames:true}"
distribUpdateConnTimeout="${distribUpdateConnTimeout:15000}" distribUpdateSoTimeout="${distribUpdateSoTimeout:120000}">
<core name="collection1" instanceDir="collection1" shard="${shard:}" collection="${collection:collection1}" config="${solrconfig:solrconfig.xml}" schema="${schema:schema.xml}"
coreNodeName="${coreNodeName:}"/>

95
solr/core/src/test/org/apache/solr/cloud/AssignTest.java Normal file
View File

@ -0,0 +1,95 @@
package org.apache.solr.cloud;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class AssignTest extends SolrTestCaseJ4 {
protected static Logger log = LoggerFactory.getLogger(AssignTest.class);
@Override
@Before
public void setUp() throws Exception {
super.setUp();
}
@Override
@After
public void tearDown() throws Exception {
super.tearDown();
}
@Test
public void testAssignNode() throws Exception {
String cname = "collection1";
Map<String,DocCollection> collectionStates = new HashMap<String,DocCollection>();
Map<String,Slice> slices = new HashMap<String,Slice>();
Map<String,Replica> replicas = new HashMap<String,Replica>();
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state",
ZkStateReader.STATE_PROP, "ACTIVE",
ZkStateReader.BASE_URL_PROP, "0.0.0.0",
ZkStateReader.CORE_NAME_PROP, "core1",
ZkStateReader.ROLES_PROP, null,
ZkStateReader.NODE_NAME_PROP, "0_0_0_0",
ZkStateReader.SHARD_ID_PROP, "shard1",
ZkStateReader.COLLECTION_PROP, cname,
ZkStateReader.NUM_SHARDS_PROP, "1",
ZkStateReader.CORE_NODE_NAME_PROP, "core_node1");
Replica replica = new Replica("core_node1" , m.getProperties());
replicas.put("core_node1", replica);
Slice slice = new Slice("slice1", replicas , new HashMap<String,Object>(0));
slices.put("slice1", slice);
DocRouter router = new ImplicitDocRouter();
DocCollection docCollection = new DocCollection(cname, slices, new HashMap<String,Object>(0), router);
collectionStates.put(cname, docCollection);
Set<String> liveNodes = new HashSet<String>();
ClusterState state = new ClusterState(liveNodes, collectionStates);
String nodeName = Assign.assignNode("collection1", state);
assertEquals("core_node2", nodeName);
}
}

30
solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
View File

@ -46,9 +46,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ReplicationHandler;
import org.apache.solr.servlet.SolrDispatchFilter;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.BeforeClass;
@ -176,8 +174,8 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
createCmd.setCoreName(ONE_NODE_COLLECTION + "core");
createCmd.setCollection(ONE_NODE_COLLECTION);
createCmd.setNumShards(1);
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ ONE_NODE_COLLECTION);
createCmd.setDataDir(getDataDir(dataDir.getAbsolutePath() + File.separator
+ ONE_NODE_COLLECTION));
server.request(createCmd);
} catch (Exception e) {
e.printStackTrace();
@ -331,7 +329,7 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
ureq.process(cloudClient);
} catch(SolrServerException e){
// try again
Thread.sleep(500);
Thread.sleep(3500);
ureq.process(cloudClient);
}
@ -415,14 +413,16 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", "/replication");
params.set("command", "backup");
File location = new File(TEMP_DIR, BasicDistributedZk2Test.class.getName() + "-backupdir-" + System.currentTimeMillis());
params.set("location", location.getAbsolutePath());
QueryRequest request = new QueryRequest(params);
NamedList<Object> results = client.request(request );
checkForBackupSuccess(client);
checkForBackupSuccess(client, location);
}
private void checkForBackupSuccess(final HttpSolrServer client)
private void checkForBackupSuccess(final HttpSolrServer client, File location)
throws InterruptedException, IOException {
class CheckStatus extends Thread {
volatile String fail = null;
@ -461,16 +461,6 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
};
}
;
SolrCore core = ((SolrDispatchFilter) shardToJetty.get(SHARD2).get(0).jetty
.getDispatchFilter().getFilter()).getCores().getCore("collection1");
String ddir;
try {
ddir = core.getDataDir();
} finally {
core.close();
}
File dataDir = new File(ddir);
int waitCnt = 0;
CheckStatus checkStatus = new CheckStatus();
@ -482,14 +472,14 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
if (checkStatus.success) {
break;
}
Thread.sleep(200);
if (waitCnt == 20) {
Thread.sleep(500);
if (waitCnt == 90) {
fail("Backup success not detected:" + checkStatus.response);
}
waitCnt++;
}
File[] files = dataDir.listFiles(new FilenameFilter() {
File[] files = location.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {

23
solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
View File

@ -538,11 +538,13 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
Create createCmd = new Create();
createCmd.setCoreName(collection + freezeI);
createCmd.setCollection(collection);
String core3dataDir = dataDir.getAbsolutePath() + File.separator
+ System.currentTimeMillis() + collection + "_3n" + freezeI;
createCmd.setDataDir(core3dataDir);
createCmd.setNumShards(numShards);
try {
String core3dataDir = dataDir.getAbsolutePath() + File.separator
+ System.currentTimeMillis() + collection + "_3n" + freezeI;
createCmd.setDataDir(getDataDir(core3dataDir));
server.request(createCmd);
} catch (SolrServerException e) {
throw new RuntimeException(e);
@ -574,11 +576,13 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
params.set(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
if (createNodeSetStr != null) params.set(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
int clientIndex = random().nextInt(2);
int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0;
List<Integer> list = new ArrayList<Integer>();
list.add(numShards);
list.add(numReplicas);
collectionInfos.put(collectionName, list);
if (collectionInfos != null) {
collectionInfos.put(collectionName, list);
}
params.set("name", collectionName);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
@ -932,8 +936,8 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
if (shardId == null) {
createCmd.setNumShards(2);
}
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ collection + num);
createCmd.setDataDir(getDataDir(dataDir.getAbsolutePath() + File.separator
+ collection + num));
if (shardId != null) {
createCmd.setShardId(shardId);
}
@ -1056,8 +1060,9 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
server.setSoTimeout(30000);
Create createCmd = new Create();
createCmd.setCoreName(collection);
createCmd.setDataDir(dataDir.getAbsolutePath() + File.separator
+ collection + frozeUnique);
createCmd.setDataDir(getDataDir(dataDir.getAbsolutePath() + File.separator
+ collection + frozeUnique));
server.request(createCmd);
} catch (Exception e) {

2
solr/core/src/test/org/apache/solr/cloud/ClusterStateUpdateTest.java
View File

@ -71,11 +71,13 @@ public class ClusterStateUpdateTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() {
System.setProperty("solrcloud.skip.autorecovery", "true");
System.setProperty("genericCoreNodeNames", "false");
}
@AfterClass
public static void afterClass() throws InterruptedException {
System.clearProperty("solrcloud.skip.autorecovery");
System.clearProperty("genericCoreNodeNames");
}
@Override

54
solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
View File

@ -36,6 +36,7 @@ import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@ -98,7 +99,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
zkClient.close();
}
public String publishState(String coreName, String stateName, int numShards)
public String publishState(String coreName, String coreNodeName, String stateName, int numShards)
throws KeeperException, InterruptedException, IOException {
if (stateName == null) {
ElectionContext ec = electionContext.remove(coreName);
@ -108,6 +109,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "deletecore",
ZkStateReader.NODE_NAME_PROP, nodeName,
ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName,
ZkStateReader.COLLECTION_PROP, collection);
DistributedQueue q = Overseer.getInQueue(zkClient);
q.offer(ZkStateReader.toJSON(m));
@ -117,6 +119,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
ZkStateReader.STATE_PROP, stateName,
ZkStateReader.NODE_NAME_PROP, nodeName,
ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName,
ZkStateReader.COLLECTION_PROP, collection,
ZkStateReader.NUM_SHARDS_PROP, Integer.toString(numShards),
ZkStateReader.BASE_URL_PROP, "http://" + nodeName
@ -126,7 +129,8 @@ public class OverseerTest extends SolrTestCaseJ4 {
}
for (int i = 0; i < 120; i++) {
String shardId = getShardId(coreName);
String shardId = getShardId("http://" + nodeName
+ "/solr/", coreName);
if (shardId != null) {
try {
zkClient.makePath("/collections/" + collection + "/leader_elect/"
@ -136,7 +140,8 @@ public class OverseerTest extends SolrTestCaseJ4 {
"http://" + nodeName + "/solr/", ZkStateReader.NODE_NAME_PROP,
nodeName, ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.SHARD_ID_PROP, shardId,
ZkStateReader.COLLECTION_PROP, collection);
ZkStateReader.COLLECTION_PROP, collection,
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
ShardLeaderElectionContextBase ctx = new ShardLeaderElectionContextBase(
elector, shardId, collection, nodeName + "_" + coreName, props,
zkStateReader);
@ -148,13 +153,18 @@ public class OverseerTest extends SolrTestCaseJ4 {
return null;
}
private String getShardId(final String coreName) {
private String getShardId(final String baseUrl, final String coreName) {
Map<String,Slice> slices = zkStateReader.getClusterState().getSlicesMap(
collection);
if (slices != null) {
for (Slice slice : slices.values()) {
if (slice.getReplicasMap().containsKey(nodeName + "_" + coreName)) {
return slice.getName();
for (Replica replica : slice.getReplicas()) {
// TODO: for really large clusters, we could 'index' on this
String rbaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
String rcore = replica.getStr(ZkStateReader.CORE_NAME_PROP);
if (baseUrl.equals(rbaseUrl) && coreName.equals(rcore)) {
return slice.getName();
}
}
}
}
@ -202,7 +212,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
final int numShards=6;
for (int i = 0; i < numShards; i++) {
assertNotNull("shard got no id?", zkController.publishState("core" + (i+1), ZkStateReader.ACTIVE, 3));
assertNotNull("shard got no id?", zkController.publishState("core" + (i+1), "node" + (i+1), ZkStateReader.ACTIVE, 3));
}
assertEquals(2, reader.getClusterState().getSlice("collection1", "shard1").getReplicasMap().size());
@ -277,7 +287,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
final String coreName = "core" + slot;
try {
ids[slot]=controllers[slot % nodeCount].publishState(coreName, ZkStateReader.ACTIVE, sliceCount);
ids[slot]=controllers[slot % nodeCount].publishState(coreName, "node" + slot, ZkStateReader.ACTIVE, sliceCount);
} catch (Throwable e) {
e.printStackTrace();
fail("register threw exception:" + e.getClass());
@ -440,7 +450,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
assertEquals(reader.getClusterState().toString(), ZkStateReader.RECOVERING,
reader.getClusterState().getSlice("collection1", "shard1").getReplicasMap()
.get("node1_core1").getStr(ZkStateReader.STATE_PROP));
.get("core_node1").getStr(ZkStateReader.STATE_PROP));
//publish node state (active)
m = new ZkNodeProps(Overseer.QUEUE_OPERATION, "state",
@ -471,7 +481,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
while(maxIterations-->0) {
Slice slice = reader.getClusterState().getSlice("collection1", "shard1");
if(slice!=null) {
coreState = slice.getReplicasMap().get("node1_core1").getStr(ZkStateReader.STATE_PROP);
coreState = slice.getReplicasMap().get("core_node1").getStr(ZkStateReader.STATE_PROP);
if(coreState.equals(expectedState)) {
return;
}
@ -523,14 +533,14 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient = electNewOverseer(server.getZkAddress());
Thread.sleep(1000);
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
waitForCollections(reader, "collection1");
verifyStatus(reader, ZkStateReader.RECOVERING);
int version = getClusterStateVersion(controllerClient);
mockController.publishState("core1", ZkStateReader.ACTIVE, 1);
mockController.publishState("core1", "core_node1", ZkStateReader.ACTIVE, 1);
while(version == getClusterStateVersion(controllerClient));
@ -539,7 +549,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient.close();
Thread.sleep(1000); //wait for overseer to get killed
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
version = getClusterStateVersion(controllerClient);
overseerClient = electNewOverseer(server.getZkAddress());
@ -553,7 +563,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
assertEquals("Shard count does not match", 1, reader.getClusterState()
.getSlice("collection1", "shard1").getReplicasMap().size());
version = getClusterStateVersion(controllerClient);
mockController.publishState("core1", null,1);
mockController.publishState("core1", "core_node1", null,1);
while(version == getClusterStateVersion(controllerClient));
Thread.sleep(500);
assertFalse("collection1 should be gone after publishing the null state", reader.getClusterState().getCollections().contains("collection1"));
@ -641,16 +651,16 @@ public class OverseerTest extends SolrTestCaseJ4 {
for (int i = 0; i < atLeast(4); i++) {
killCounter.incrementAndGet(); //for each round allow 1 kill
mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
mockController.publishState("core1", "state1",1);
mockController.publishState("core1", "node1", "state1",1);
if(mockController2!=null) {
mockController2.close();
mockController2 = null;
}
mockController.publishState("core1", "state2",1);
mockController.publishState("core1", "node1","state2",1);
mockController2 = new MockZKController(server.getZkAddress(), "node2", "collection1");
mockController.publishState("core1", "state1",1);
mockController.publishState("core1", "node1", "state1",1);
verifyShardLeader(reader, "collection1", "shard1", "core1");
mockController2.publishState("core4", "state2" ,1);
mockController2.publishState("core4", "node2", "state2" ,1);
mockController.close();
mockController = null;
verifyShardLeader(reader, "collection1", "shard1", "core4");
@ -697,7 +707,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient = electNewOverseer(server.getZkAddress());
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
waitForCollections(reader, "collection1");
@ -708,7 +718,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
int version = getClusterStateVersion(controllerClient);
mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
while (version == getClusterStateVersion(controllerClient));
@ -718,7 +728,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
int numFound = 0;
for (DocCollection collection : state.getCollectionStates().values()) {
for (Slice slice : collection.getSlices()) {
if (slice.getReplicasMap().get("node1_core1") != null) {
if (slice.getReplicasMap().get("core_node1") != null) {
numFound++;
}
}
@ -761,7 +771,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerClient = electNewOverseer(server.getZkAddress());
mockController.publishState("core1", ZkStateReader.RECOVERING, 12);
mockController.publishState("core1", "node1", ZkStateReader.RECOVERING, 12);
waitForCollections(reader, "collection1");

12
solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
View File

@ -86,19 +86,25 @@ public class RecoveryZkTest extends AbstractFullDistribZkTestBase {
indexThread.join();
indexThread2.join();
Thread.sleep(500);
Thread.sleep(1000);
waitForThingsToLevelOut(30);
waitForThingsToLevelOut(45);
Thread.sleep(2000);
waitForThingsToLevelOut(30);
Thread.sleep(5000);
waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, false, true);
// test that leader and replica have same doc count
checkShardConsistency("shard1", false, false);
String fail = checkShardConsistency("shard1", false, false);
if (fail != null) {
fail(fail);
}
SolrQuery query = new SolrQuery("*:*");
query.setParam("distrib", "false");
long client1Docs = shardToJetty.get("shard1").get(0).client.solrClient.query(query).getResults().getNumFound();

3
solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
View File

@ -67,7 +67,6 @@ public class SyncSliceTest extends AbstractFullDistribZkTestBase {
super.setUp();
// we expect this time of exception as shards go up and down...
//ignoreException(".*");
useFactory(null);
System.setProperty("numShards", Integer.toString(sliceCount));
}
@ -94,7 +93,7 @@ public class SyncSliceTest extends AbstractFullDistribZkTestBase {
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
waitForThingsToLevelOut(15);
waitForThingsToLevelOut(30);
del("*:*");
List<CloudJettyRunner> skipServers = new ArrayList<CloudJettyRunner>();

14
solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
View File

@ -91,7 +91,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCollection(collection);
String coreDataDir = dataDir.getAbsolutePath() + File.separator
+ System.currentTimeMillis() + collection + "1";
createCmd.setDataDir(coreDataDir);
createCmd.setDataDir(getDataDir(coreDataDir));
createCmd.setNumShards(2);
SolrServer client = clients.get(0);
@ -107,7 +107,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCollection(collection);
coreDataDir = dataDir.getAbsolutePath() + File.separator
+ System.currentTimeMillis() + collection + "2";
createCmd.setDataDir(coreDataDir);
createCmd.setDataDir(getDataDir(coreDataDir));
server.request(createCmd);
@ -171,7 +171,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCollection("unloadcollection");
createCmd.setNumShards(1);
String core1DataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_1n";
createCmd.setDataDir(core1DataDir);
createCmd.setDataDir(getDataDir(core1DataDir));
server.request(createCmd);
ZkStateReader zkStateReader = getCommonCloudSolrServer().getZkStateReader();
@ -189,7 +189,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCoreName("unloadcollection2");
createCmd.setCollection("unloadcollection");
String core2dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection1" + "_2n";
createCmd.setDataDir(core2dataDir);
createCmd.setDataDir(getDataDir(core2dataDir));
server.request(createCmd);
zkStateReader.updateClusterState(true);
@ -227,7 +227,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCoreName("unloadcollection3");
createCmd.setCollection("unloadcollection");
String core3dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_3n";
createCmd.setDataDir(core3dataDir);
createCmd.setDataDir(getDataDir(core3dataDir));
server.request(createCmd);
waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
@ -296,7 +296,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd.setCoreName("unloadcollection4");
createCmd.setCollection("unloadcollection");
String core4dataDir = dataDir.getAbsolutePath() + File.separator + System.currentTimeMillis() + "unloadcollection" + "_4n";
createCmd.setDataDir(core4dataDir);
createCmd.setDataDir(getDataDir(core4dataDir));
server.request(createCmd);
waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);
@ -334,7 +334,7 @@ public class UnloadDistributedZkTest extends BasicDistributedZkTest {
createCmd = new Create();
createCmd.setCoreName(leaderProps.getCoreName());
createCmd.setCollection("unloadcollection");
createCmd.setDataDir(core1DataDir);
createCmd.setDataDir(getDataDir(core1DataDir));
server.request(createCmd);
waitForRecoveriesToFinish("unloadcollection", zkStateReader, false);

4
solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
View File

@ -163,7 +163,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
cc = getCoreContainer();
ZkController zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, 10000,
"127.0.0.1", "8983", "solr", "0", 10000, 10000, new CurrentCoreDescriptorProvider() {
"127.0.0.1", "8983", "solr", "0", true, 10000, 10000, new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {
@ -203,7 +203,7 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
cc = getCoreContainer();
zkController = new ZkController(cc, server.getZkAddress(),
TIMEOUT, 10000, "127.0.0.1", "8983", "solr", "0", 10000, 10000, new CurrentCoreDescriptorProvider() {
TIMEOUT, 10000, "127.0.0.1", "8983", "solr", "0", true, 10000, 10000, new CurrentCoreDescriptorProvider() {
@Override
public List<CoreDescriptor> getCurrentDescriptors() {

106
solr/core/src/test/org/apache/solr/cloud/hdfs/BasicHdfsTest.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud.hdfs;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.cloud.BasicDistributedZkTest;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class BasicHdfsTest extends BasicDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr");
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
System.clearProperty("solr.hdfs.home");
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
public BasicHdfsTest() {
super();
sliceCount = 1;
shardCount = 1;
}
protected String getSolrXml() {
return "solr-no-core.xml";
}
@Override
public void doTest() throws Exception {
createCollection("delete_data_dir", 1, 1, 1);
waitForRecoveriesToFinish("delete_data_dir", false);
cloudClient.setDefaultCollection("delete_data_dir");
cloudClient.getZkStateReader().updateClusterState(true);
NamedList<Object> response = cloudClient.query(
new SolrQuery().setRequestHandler("/admin/system")).getResponse();
NamedList<Object> coreInfo = (NamedList<Object>) response.get("core");
String dataDir = (String) ((NamedList<Object>) coreInfo.get("directory"))
.get("data");
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.DELETE.toString());
params.set("name", "delete_data_dir");
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
cloudClient.request(request);
Configuration conf = new Configuration();
conf.setBoolean("fs.hdfs.impl.disable.cache", true);
FileSystem fs = FileSystem.newInstance(new URI(dataDir), conf);
assertFalse(
"Data directory exists after collection removal : "
+ dataDir, fs.exists(new Path(dataDir)));
fs.close();
}
}

58
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZk2Test.java Normal file
View File

@ -0,0 +1,58 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.BasicDistributedZk2Test;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsBasicDistributedZk2Test extends BasicDistributedZk2Test {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

58
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsBasicDistributedZkTest.java Normal file
View File

@ -0,0 +1,58 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.BasicDistributedZkTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsBasicDistributedZkTest extends BasicDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

69
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsChaosMonkeySafeLeaderTest.java Normal file
View File

@ -0,0 +1,69 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.ChaosMonkeySafeLeaderTest;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsChaosMonkeySafeLeaderTest extends ChaosMonkeySafeLeaderTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
// super class may hard code directory
useFactory("org.apache.solr.core.HdfsDirectoryFactory");
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

64
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsCollectionsAPIDistributedZkTest.java Normal file
View File

@ -0,0 +1,64 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.CollectionsAPIDistributedZkTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsCollectionsAPIDistributedZkTest extends CollectionsAPIDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsCollectionsAPIDistributedZkTest.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
System.setProperty("solr.hdfs.home", dfsCluster.getURI().toString() + "/solr");
System.setProperty("solr.hdfs.blockcache.enabled", "false");
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
System.clearProperty("solr.hdfs.home");
System.clearProperty("solr.hdfs.blockcache.enabled");
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

59
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsRecoveryZkTest.java Normal file
View File

@ -0,0 +1,59 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.RecoveryZkTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsRecoveryZkTest extends RecoveryZkTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
System.setProperty("solr.hdfs.blockcache.blocksperbank", "2048");
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

66
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsSyncSliceTest.java Normal file
View File

@ -0,0 +1,66 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.util.Locale;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.SyncSliceTest;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsSyncSliceTest extends SyncSliceTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsBasicDistributedZk2Test.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

82
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java Normal file
View File

@ -0,0 +1,82 @@
package org.apache.solr.cloud.hdfs;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.Locale;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.solr.SolrTestCaseJ4;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class HdfsTestUtil {
private static Locale savedLocale;
public static MiniDFSCluster setupClass(String dataDir) throws Exception {
savedLocale = Locale.getDefault();
// TODO: we HACK around HADOOP-9643
Locale.setDefault(Locale.ENGLISH);
int dataNodes = 2;
Configuration conf = new Configuration();
conf.set("dfs.block.access.token.enable", "false");
conf.set("dfs.permissions.enabled", "false");
conf.set("hadoop.security.authentication", "simple");
conf.set("hdfs.minidfs.basedir", dataDir + File.separator + "hdfsBaseDir");
conf.set("dfs.namenode.name.dir", dataDir + File.separator + "nameNodeNameDir");
System.setProperty("test.build.data", dataDir + File.separator + "hdfs" + File.separator + "build");
System.setProperty("test.cache.data", dataDir + File.separator + "hdfs" + File.separator + "cache");
System.setProperty("solr.lock.type", "hdfs");
MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, dataNodes, true, null);
SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory");
return dfsCluster;
}
public static void teardownClass(MiniDFSCluster dfsCluster) throws Exception {
SolrTestCaseJ4.resetFactory();
System.clearProperty("solr.lock.type");
System.clearProperty("test.build.data");
System.clearProperty("test.cache.data");
if (dfsCluster != null) {
dfsCluster.shutdown();
}
// TODO: we HACK around HADOOP-9643
Locale.setDefault(savedLocale);
}
public static String getDataDir(MiniDFSCluster dfsCluster, String dataDir)
throws IOException {
URI uri = dfsCluster.getURI();
String dir = uri.toString()
+ "/"
+ new File(dataDir).toString().replaceAll(":", "_")
.replaceAll("/", "_");
return dir;
}
}

58
solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsUnloadDistributedZkTest.java Normal file
View File

@ -0,0 +1,58 @@
package org.apache.solr.cloud.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.UnloadDistributedZkTest;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@Slow
@Nightly
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread(s)
public class HdfsUnloadDistributedZkTest extends UnloadDistributedZkTest {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void setupClass() throws Exception {
dfsCluster = HdfsTestUtil.setupClass(new File(TEMP_DIR,
HdfsUnloadDistributedZkTest.class.getName() + "_"
+ System.currentTimeMillis()).getAbsolutePath());
}
@AfterClass
public static void teardownClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Override
protected String getDataDir(String dataDir) throws IOException {
return HdfsTestUtil.getDataDir(dfsCluster, dataDir);
}
}

2
solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
View File

@ -1507,7 +1507,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
}
public String getDataDir() {
return dataDir.toString();
return dataDir.getAbsolutePath();
}
public String getSolrConfigFile() {

51
solr/core/src/test/org/apache/solr/search/TestRecovery.java
View File

@ -17,6 +17,8 @@
package org.apache.solr.search;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import org.apache.solr.common.SolrException;
import org.noggit.ObjectBuilder;
import org.apache.solr.SolrTestCaseJ4;
@ -42,8 +44,16 @@ import java.util.concurrent.Future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.update.UpdateHandler;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.noggit.ObjectBuilder;
public class TestRecovery extends SolrTestCaseJ4 {
@ -744,16 +754,17 @@ public class TestRecovery extends SolrTestCaseJ4 {
clearIndex();
assertU(commit());
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
h.close();
String[] files = UpdateLog.getLogList(logDir);
String[] files = ulog.getLogList(logDir);
for (String file : files) {
new File(logDir, file).delete();
}
assertEquals(0, UpdateLog.getLogList(logDir).length);
assertEquals(0, ulog.getLogList(logDir).length);
createCore();
@ -771,7 +782,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertU(commit());
assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
assertEquals(2, UpdateLog.getLogList(logDir).length);
assertEquals(2, ulog.getLogList(logDir).length);
addDocs(105, start, versions); start+=105;
assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
@ -779,7 +790,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
// previous two logs should be gone now
assertEquals(1, UpdateLog.getLogList(logDir).length);
assertEquals(1, ulog.getLogList(logDir).length);
addDocs(1, start, versions); start+=1;
h.close();
@ -799,14 +810,14 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertJQ(req("qt","/get", "getVersions",""+maxReq), "/versions==" + versions.subList(0,Math.min(maxReq,start)));
// previous logs should be gone now
assertEquals(1, UpdateLog.getLogList(logDir).length);
assertEquals(1, ulog.getLogList(logDir).length);
//
// test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before that tlog file.
//
addDocs(1, start, new LinkedList<Long>()); // don't add this to the versions list because we are going to lose it...
h.close();
files = UpdateLog.getLogList(logDir);
files = ulog.getLogList(logDir);
Arrays.sort(files);
RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length-1]), "rw");
raf.writeChars("This is a trashed log file that really shouldn't work at all, but we'll see...");
@ -854,7 +865,8 @@ public class TestRecovery extends SolrTestCaseJ4 {
}
};
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
clearIndex();
assertU(commit());
@ -864,7 +876,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertU(adoc("id","F3"));
h.close();
String[] files = UpdateLog.getLogList(logDir);
String[] files = ulog.getLogList(logDir);
Arrays.sort(files);
RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length-1]), "rw");
raf.seek(raf.length()); // seek to end
@ -908,7 +920,8 @@ public class TestRecovery extends SolrTestCaseJ4 {
try {
DirectUpdateHandler2.commitOnClose = false;
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
clearIndex();
assertU(commit());
@ -920,7 +933,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
h.close();
String[] files = UpdateLog.getLogList(logDir);
String[] files = ulog.getLogList(logDir);
Arrays.sort(files);
RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length-1]), "rw");
long len = raf.length();
@ -991,7 +1004,8 @@ public class TestRecovery extends SolrTestCaseJ4 {
}
};
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
clearIndex();
assertU(commit());
@ -1001,7 +1015,7 @@ public class TestRecovery extends SolrTestCaseJ4 {
assertU(adoc("id","CCCCCC"));
h.close();
String[] files = UpdateLog.getLogList(logDir);
String[] files = ulog.getLogList(logDir);
Arrays.sort(files);
String fname = files[files.length-1];
RandomAccessFile raf = new RandomAccessFile(new File(logDir, fname), "rw");
@ -1071,17 +1085,18 @@ public class TestRecovery extends SolrTestCaseJ4 {
// stops the core, removes the transaction logs, restarts the core.
void deleteLogs() throws Exception {
File logDir = h.getCore().getUpdateHandler().getUpdateLog().getLogDir();
UpdateLog ulog = h.getCore().getUpdateHandler().getUpdateLog();
File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
h.close();
try {
String[] files = UpdateLog.getLogList(logDir);
String[] files = ulog.getLogList(logDir);
for (String file : files) {
new File(logDir, file).delete();
}
assertEquals(0, UpdateLog.getLogList(logDir).length);
assertEquals(0, ulog.getLogList(logDir).length);
} finally {
// make sure we create the core again, even if the assert fails so it won't mess
// up the next test.

1176
solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java Normal file
View File

File diff suppressed because it is too large Load Diff

110
solr/core/src/test/org/apache/solr/store/blockcache/BlockCacheTest.java Normal file
View File

@ -0,0 +1,110 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Arrays;
import java.util.Random;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
public class BlockCacheTest extends LuceneTestCase {
@Test
public void testBlockCache() {
int blocksInTest = 2000000;
int blockSize = 1024;
int slabSize = blockSize * 4096;
long totalMemory = 2 * slabSize;
BlockCache blockCache = new BlockCache(new Metrics(new Configuration()), true,totalMemory,slabSize,blockSize);
byte[] buffer = new byte[1024];
Random random = random();
byte[] newData = new byte[blockSize];
AtomicLong hitsInCache = new AtomicLong();
AtomicLong missesInCache = new AtomicLong();
long storeTime = 0;
long fetchTime = 0;
int passes = 10000;
BlockCacheKey blockCacheKey = new BlockCacheKey();
for (int j = 0; j < passes; j++) {
long block = random.nextInt(blocksInTest);
int file = 0;
blockCacheKey.setBlock(block);
blockCacheKey.setFile(file);
if (blockCache.fetch(blockCacheKey, buffer)) {
hitsInCache.incrementAndGet();
} else {
missesInCache.incrementAndGet();
}
byte[] testData = testData(random, blockSize, newData);
long t1 = System.nanoTime();
blockCache.store(blockCacheKey, 0, testData, 0, blockSize);
storeTime += (System.nanoTime() - t1);
long t3 = System.nanoTime();
if (blockCache.fetch(blockCacheKey, buffer)) {
fetchTime += (System.nanoTime() - t3);
assertTrue(Arrays.equals(testData, buffer));
}
}
System.out.println("Cache Hits = " + hitsInCache.get());
System.out.println("Cache Misses = " + missesInCache.get());
System.out.println("Store = " + (storeTime / (double) passes) / 1000000.0);
System.out.println("Fetch = " + (fetchTime / (double) passes) / 1000000.0);
System.out.println("# of Elements = " + blockCache.getSize());
}
/**
* Verify checking of buffer size limits against the cached block size.
*/
@Test
public void testLongBuffer() {
Random random = random();
int blockSize = BlockCache._32K;
int slabSize = blockSize * 1024;
long totalMemory = 2 * slabSize;
BlockCache blockCache = new BlockCache(new Metrics(new Configuration()),
true, totalMemory, slabSize);
BlockCacheKey blockCacheKey = new BlockCacheKey();
blockCacheKey.setBlock(0);
blockCacheKey.setFile(0);
byte[] newData = new byte[blockSize*3];
byte[] testData = testData(random, blockSize, newData);
assertTrue(blockCache.store(blockCacheKey, 0, testData, 0, blockSize));
assertTrue(blockCache.store(blockCacheKey, 0, testData, blockSize, blockSize));
assertTrue(blockCache.store(blockCacheKey, 0, testData, blockSize*2, blockSize));
assertTrue(blockCache.store(blockCacheKey, 1, testData, 0, blockSize - 1));
assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize, blockSize - 1));
assertTrue(blockCache.store(blockCacheKey, 1, testData, blockSize*2, blockSize - 1));
}
private static byte[] testData(Random random, int size, byte[] buf) {
random.nextBytes(buf);
return buf;
}
}

251
solr/core/src/test/org/apache/solr/store/blockcache/BlockDirectoryTest.java Normal file
View File

@ -0,0 +1,251 @@
package org.apache.solr.store.blockcache;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.Random;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MergeInfo;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.store.hdfs.HdfsDirectory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
public class BlockDirectoryTest extends LuceneTestCase {
private class MapperCache implements Cache {
public Map<String, byte[]> map = new ConcurrentLinkedHashMap.Builder<String, byte[]>().maximumWeightedCapacity(8).build();
@Override
public void update(String name, long blockId, int blockOffset, byte[] buffer, int offset, int length) {
byte[] cached = map.get(name + blockId);
if (cached != null) {
int newlen = Math.max(cached.length, blockOffset + length);
byte[] b = new byte[newlen];
System.arraycopy(cached, 0, b, 0, cached.length);
System.arraycopy(buffer, offset, b, blockOffset, length);
cached = b;
} else {
cached = copy(blockOffset, buffer, offset, length);
}
map.put(name + blockId, cached);
}
private byte[] copy(int blockOffset, byte[] buffer, int offset, int length) {
byte[] b = new byte[length + blockOffset];
System.arraycopy(buffer, offset, b, blockOffset, length);
return b;
}
@Override
public boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) {
// return false;
byte[] data = map.get(name + blockId);
if (data == null) {
return false;
}
System.arraycopy(data, blockOffset, b, off, lengthToReadInBlock);
return true;
}
@Override
public void delete(String name) {
}
@Override
public long size() {
return map.size();
}
@Override
public void renameCacheFile(String source, String dest) {
}
}
private static final int MAX_NUMBER_OF_WRITES = 10000;
private static final int MIN_FILE_SIZE = 100;
private static final int MAX_FILE_SIZE = 100000;
private static final int MIN_BUFFER_SIZE = 1;
private static final int MAX_BUFFER_SIZE = 12000;
private static final int MAX_NUMBER_OF_READS = 20000;
private BlockDirectory directory;
private File file;
private Random random;
private MapperCache mapperCache;
@Before
public void setUp() throws Exception {
super.setUp();
file = new File(TEMP_DIR, HdfsDirectory.class.getName() + "-" + System.currentTimeMillis());
rm(file);
file.mkdirs();
FSDirectory dir = FSDirectory.open(new File(file, "base"));
mapperCache = new MapperCache();
directory = new BlockDirectory("test", dir, mapperCache, null, true, true);
random = random();
}
@After
public void tearDown() throws Exception {
super.tearDown();
FileUtils.deleteDirectory(file);
}
@Test
public void testEOF() throws IOException {
Directory fsDir = FSDirectory.open(new File(file, "normal"));
String name = "test.eof";
createFile(name, fsDir, directory);
long fsLength = fsDir.fileLength(name);
long hdfsLength = directory.fileLength(name);
assertEquals(fsLength, hdfsLength);
testEof(name, fsDir, fsLength);
testEof(name, directory, hdfsLength);
}
private void testEof(String name, Directory directory, long length) throws IOException {
IndexInput input = directory.openInput(name, new IOContext());
input.seek(length);
try {
input.readByte();
fail("should throw eof");
} catch (IOException e) {
}
}
@Test
public void testRandomAccessWrites() throws IOException {
long t1 = System.nanoTime();
int i = 0;
try {
for (; i < 10; i++) {
Directory fsDir = FSDirectory.open(new File(file, "normal"));
String name = getName();
createFile(name, fsDir, directory);
assertInputsEquals(name, fsDir, directory);
}
} catch (Exception e) {
e.printStackTrace();
fail("Test failed on pass [" + i + "]");
}
long t2 = System.nanoTime();
System.out.println("Total time is " + ((t2 - t1)/1000000) + "ms");
}
@Test
public void testRandomAccessWritesLargeCache() throws IOException {
mapperCache.map = new ConcurrentLinkedHashMap.Builder<String, byte[]>().maximumWeightedCapacity(10000).build();
testRandomAccessWrites();
}
private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException {
int reads = random.nextInt(MAX_NUMBER_OF_READS);
IndexInput fsInput = fsDir.openInput(name, new IOContext());
IndexInput hdfsInput = hdfs.openInput(name, new IOContext());
assertEquals(fsInput.length(), hdfsInput.length());
int fileLength = (int) fsInput.length();
for (int i = 0; i < reads; i++) {
byte[] fsBuf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
byte[] hdfsBuf = new byte[fsBuf.length];
int offset = random.nextInt(fsBuf.length);
int length = random.nextInt(fsBuf.length - offset);
int pos = random.nextInt(fileLength - length);
fsInput.seek(pos);
fsInput.readBytes(fsBuf, offset, length);
hdfsInput.seek(pos);
hdfsInput.readBytes(hdfsBuf, offset, length);
for (int f = offset; f < length; f++) {
if (fsBuf[f] != hdfsBuf[f]) {
fail("read [" + i + "]");
}
}
}
fsInput.close();
hdfsInput.close();
}
private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException {
int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT);
IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT);
for (int i = 0; i < writes; i++) {
byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE, fileLength)) + MIN_BUFFER_SIZE];
random.nextBytes(buf);
int offset = random.nextInt(buf.length);
int length = random.nextInt(buf.length - offset);
fsOutput.writeBytes(buf, offset, length);
hdfsOutput.writeBytes(buf, offset, length);
}
fsOutput.close();
hdfsOutput.close();
}
private String getName() {
return Long.toString(Math.abs(random.nextLong()));
}
public static void rm(File file) {
if (!file.exists()) {
return;
}
if (file.isDirectory()) {
for (File f : file.listFiles()) {
rm(f);
}
}
file.delete();
}
/**
* Verify the configuration options for the block cache are handled
* appropriately.
*/
@Test
public void ensureCacheConfigurable() throws Exception {
IOContext mergeContext = new IOContext(new MergeInfo(1,1,false,1));
BlockDirectory d = directory;
assertTrue(d.useReadCache("", IOContext.DEFAULT));
assertTrue(d.useWriteCache("", IOContext.DEFAULT));
assertFalse(d.useWriteCache("", mergeContext));
d = new BlockDirectory("test", directory, mapperCache, null, true, false);
assertTrue(d.useReadCache("", IOContext.DEFAULT));
assertFalse(d.useWriteCache("", IOContext.DEFAULT));
assertFalse(d.useWriteCache("", mergeContext));
d = new BlockDirectory("test", directory, mapperCache, null, false, true);
assertFalse(d.useReadCache("", IOContext.DEFAULT));
assertTrue(d.useWriteCache("", IOContext.DEFAULT));
assertFalse(d.useWriteCache("", mergeContext));
}
}

222
solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java Normal file
View File

@ -0,0 +1,222 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread (HADOOP-9049)
//@Ignore("this test violates the test security policy because of org.apache.hadoop.fs.RawLocalFileSystem.mkdirs")
public class HdfsDirectoryTest extends SolrTestCaseJ4 {
private static final int MAX_NUMBER_OF_WRITES = 10000;
private static final int MIN_FILE_SIZE = 100;
private static final int MAX_FILE_SIZE = 100000;
private static final int MIN_BUFFER_SIZE = 1;
private static final int MAX_BUFFER_SIZE = 5000;
private static final int MAX_NUMBER_OF_READS = 10000;
private static MiniDFSCluster dfsCluster;
private HdfsDirectory directory;
private Random random;
@BeforeClass
public static void beforeClass() throws Exception {
createTempDir();
dfsCluster = HdfsTestUtil.setupClass(TEMP_DIR.getAbsolutePath()
+ File.separator + HdfsDirectoryTest.class.getName() + "_hdfsdir-"
+ System.currentTimeMillis());
}
@AfterClass
public static void afterClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Before
public void setUp() throws Exception {
super.setUp();
Configuration conf = new Configuration();
conf.set("dfs.permissions.enabled", "false");
directory = new HdfsDirectory(new Path(dfsCluster.getURI().toString() + dataDir.getAbsolutePath() + "/hdfs"), conf);
random = random();
}
@After
public void tearDown() throws Exception {
super.tearDown();
}
@Test
public void testWritingAndReadingAFile() throws IOException {
String[] listAll = directory.listAll();
for (String file : listAll) {
directory.deleteFile(file);
}
IndexOutput output = directory.createOutput("testing.test", new IOContext());
output.writeInt(12345);
output.flush();
output.close();
IndexInput input = directory.openInput("testing.test", new IOContext());
assertEquals(12345, input.readInt());
input.close();
listAll = directory.listAll();
assertEquals(1, listAll.length);
assertEquals("testing.test", listAll[0]);
assertEquals(4, directory.fileLength("testing.test"));
IndexInput input1 = directory.openInput("testing.test", new IOContext());
IndexInput input2 = (IndexInput) input1.clone();
assertEquals(12345, input2.readInt());
input2.close();
assertEquals(12345, input1.readInt());
input1.close();
assertFalse(directory.fileExists("testing.test.other"));
assertTrue(directory.fileExists("testing.test"));
directory.deleteFile("testing.test");
assertFalse(directory.fileExists("testing.test"));
}
@Test
public void testEOF() throws IOException {
Directory fsDir = new RAMDirectory();
String name = "test.eof";
createFile(name, fsDir, directory);
long fsLength = fsDir.fileLength(name);
long hdfsLength = directory.fileLength(name);
assertEquals(fsLength, hdfsLength);
testEof(name,fsDir,fsLength);
testEof(name,directory,hdfsLength);
}
private void testEof(String name, Directory directory, long length) throws IOException {
IndexInput input = directory.openInput(name, new IOContext());
input.seek(length);
try {
input.readByte();
fail("should throw eof");
} catch (IOException e) {
}
}
@Test
public void testRandomAccessWrites() throws IOException {
int i = 0;
try {
Set<String> names = new HashSet<String>();
for (; i< 10; i++) {
Directory fsDir = new RAMDirectory();
String name = getName();
System.out.println("Working on pass [" + i +"] contains [" + names.contains(name) + "]");
names.add(name);
createFile(name,fsDir,directory);
assertInputsEquals(name,fsDir,directory);
fsDir.close();
}
} catch (Exception e) {
e.printStackTrace();
fail("Test failed on pass [" + i + "]");
}
}
private void assertInputsEquals(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
int reads = random.nextInt(MAX_NUMBER_OF_READS);
IndexInput fsInput = fsDir.openInput(name,new IOContext());
IndexInput hdfsInput = hdfs.openInput(name,new IOContext());
assertEquals(fsInput.length(), hdfsInput.length());
int fileLength = (int) fsInput.length();
for (int i = 0; i < reads; i++) {
int nextInt = Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE,fileLength);
byte[] fsBuf = new byte[random.nextInt(nextInt > 0 ? nextInt : 1) + MIN_BUFFER_SIZE];
byte[] hdfsBuf = new byte[fsBuf.length];
int offset = random.nextInt(fsBuf.length);
nextInt = fsBuf.length - offset;
int length = random.nextInt(nextInt > 0 ? nextInt : 1);
nextInt = fileLength - length;
int pos = random.nextInt(nextInt > 0 ? nextInt : 1);
fsInput.seek(pos);
fsInput.readBytes(fsBuf, offset, length);
hdfsInput.seek(pos);
hdfsInput.readBytes(hdfsBuf, offset, length);
for (int f = offset; f < length; f++) {
if (fsBuf[f] != hdfsBuf[f]) {
fail();
}
}
}
fsInput.close();
hdfsInput.close();
}
private void createFile(String name, Directory fsDir, HdfsDirectory hdfs) throws IOException {
int writes = random.nextInt(MAX_NUMBER_OF_WRITES);
int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE;
IndexOutput fsOutput = fsDir.createOutput(name, new IOContext());
fsOutput.setLength(fileLength);
IndexOutput hdfsOutput = hdfs.createOutput(name, new IOContext());
hdfsOutput.setLength(fileLength);
for (int i = 0; i < writes; i++) {
byte[] buf = new byte[random.nextInt(Math.min(MAX_BUFFER_SIZE - MIN_BUFFER_SIZE,fileLength)) + MIN_BUFFER_SIZE];
random.nextBytes(buf);
int offset = random.nextInt(buf.length);
int length = random.nextInt(buf.length - offset);
fsOutput.writeBytes(buf, offset, length);
hdfsOutput.writeBytes(buf, offset, length);
}
fsOutput.close();
hdfsOutput.close();
}
private String getName() {
return Long.toString(Math.abs(random.nextLong()));
}
}

86
solr/core/src/test/org/apache/solr/store/hdfs/HdfsLockFactoryTest.java Normal file
View File

@ -0,0 +1,86 @@
package org.apache.solr.store.hdfs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.store.Lock;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
@ThreadLeakScope(Scope.NONE) // hdfs client currently leaks thread (HADOOP-9049)
public class HdfsLockFactoryTest extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
@BeforeClass
public static void beforeClass() throws Exception {
createTempDir();
dfsCluster = HdfsTestUtil.setupClass(TEMP_DIR.getAbsolutePath()
+ File.separator + HdfsLockFactoryTest.class.getName() + "_hdfsdir-"
+ System.currentTimeMillis());
}
@AfterClass
public static void afterClass() throws Exception {
HdfsTestUtil.teardownClass(dfsCluster);
dfsCluster = null;
}
@Before
public void setUp() throws Exception {
super.setUp();
}
@After
public void tearDown() throws Exception {
super.tearDown();
}
@Test
public void testBasic() throws IOException {
URI uri = dfsCluster.getURI();
Path lockPath = new Path(uri.toString(), "/lock");
HdfsLockFactory lockFactory = new HdfsLockFactory(lockPath, new Configuration());
Lock lock = lockFactory.makeLock("testlock");
boolean success = lock.obtain();
assertTrue("We could not get the lock when it should be available", success);
success = lock.obtain();
assertFalse("We got the lock but it should be unavailble", success);
lock.release();
success = lock.obtain();
assertTrue("We could not get the lock when it should be available", success);
success = lock.obtain();
assertFalse("We got the lock but it should be unavailble", success);
}
}

1807
solr/example/alt-configs/hdfs/solrconfig.xml Normal file
View File

File diff suppressed because it is too large Load Diff

6
solr/example/resources/log4j.properties
View File

@ -1,4 +1,5 @@
# Logging level
solr.log=logs/
log4j.rootLogger=INFO, file, CONSOLE
log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
@ -12,8 +13,9 @@ log4j.appender.file.MaxFileSize=4MB
log4j.appender.file.MaxBackupIndex=9
#- File to log to and log format
log4j.appender.file.File=logs/solr.log
log4j.appender.file.File=${solr.log}/solr.log
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.hadoop=WARN

1
solr/example/solr/solr.xml
View File

@ -33,6 +33,7 @@
<int name="hostPort">${jetty.port:8983}</int>
<str name="hostContext">${hostContext:solr}</str>
<int name="zkClientTimeout">${zkClientTimeout:15000}</int>
<bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
</solrcloud>
<shardHandlerFactory name="shardHandlerFactory"

1
solr/licenses/commons-configuration-1.6.jar.sha1 Normal file
View File

@ -0,0 +1 @@
32cadde23955d7681b0d94a2715846d20b425235

403
solr/licenses/commons-configuration-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,403 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

9
solr/licenses/commons-configuration-NOTICE.txt Normal file
View File

@ -0,0 +1,9 @@
Apache Commons Configuration
Copyright 2001-2008 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).

1
solr/licenses/concurrentlinkedhashmap-lru-1.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
4316d710b6619ffe210c98deb2b0893587dad454

403
solr/licenses/concurrentlinkedhashmap-lru-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,403 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

0
solr/licenses/concurrentlinkedhashmap-lru-NOTICE.txt Normal file
View File

1
solr/licenses/hadoop-annotations-2.0.5-alpha.jar.sha1 Normal file
View File

@ -0,0 +1 @@
64e2b38638f3b3ecf14806a12c919334ebd77ff7

244
solr/licenses/hadoop-annotations-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,244 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
APACHE HADOOP SUBCOMPONENTS:
The Apache Hadoop project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
For the org.apache.hadoop.util.bloom.* classes:
/**
*
* Copyright (c) 2005, European Commission project OneLab under contract
* 034819 (http://www.one-lab.org)
* All rights reserved.
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the distribution.
* - Neither the name of the University Catholique de Louvain - UCL
* nor the names of its contributors may be used to endorse or
* promote products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

2
solr/licenses/hadoop-annotations-NOTICE.txt Normal file
View File

@ -0,0 +1,2 @@
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).

Some files were not shown because too many files have changed in this diff Show More