Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-1301: Clean up. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1547962 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mark Robert Miller 2013-12-04 23:28:50 +00:00
parent e4f83aa1a0
commit 6350b2d709
6 changed files with 9 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
View File

@ -136,7 +136,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
showNonSolrCloud = Arrays.asList(args).contains(SHOW_NON_SOLR_CLOUD); // intercept it first showNonSolrCloud = Arrays.asList(args).contains(SHOW_NON_SOLR_CLOUD); // intercept it first
ArgumentParser parser = ArgumentParsers ArgumentParser parser = ArgumentParsers
.newArgumentParser("hadoop [GenericOptions]... jar search-mr-*-job.jar " + MapReduceIndexerTool.class.getName(), false) .newArgumentParser("hadoop [GenericOptions]... jar solr-map-reduce-*.jar ", false)
.defaultHelp(true) .defaultHelp(true)
.description( .description(
"MapReduce batch job driver that takes a morphline and creates a set of Solr index shards from a set of input files " + "MapReduce batch job driver that takes a morphline and creates a set of Solr index shards from a set of input files " +
@ -197,7 +197,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
"# (Re)index an Avro based Twitter tweet file:\n" + "# (Re)index an Avro based Twitter tweet file:\n" +
"sudo -u hdfs hadoop \\\n" + "sudo -u hdfs hadoop \\\n" +
" --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + " --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
" jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName() + " \\\n" + " jar target/solr-map-reduce-*.jar \\\n" +
" -D 'mapred.child.java.opts=-Xmx500m' \\\n" + " -D 'mapred.child.java.opts=-Xmx500m' \\\n" +
// " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + // " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" +
" --log4j src/test/resources/log4j.properties \\\n" + " --log4j src/test/resources/log4j.properties \\\n" +
@ -213,7 +213,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
"# 3) file was last modified less than 100000 minutes ago\n" + "# 3) file was last modified less than 100000 minutes ago\n" +
"# 4) file size is between 1 MB and 1 GB\n" + "# 4) file size is between 1 MB and 1 GB\n" +
"# Also include extra library jar file containing JSON tweet Java parser:\n" + "# Also include extra library jar file containing JSON tweet Java parser:\n" +
"hadoop jar target/search-mr-*-job.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool" + " \\\n" + "hadoop jar target/solr-map-reduce-*.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool" + " \\\n" +
" -find hdfs:///user/$USER/solrloadtest/twitter/tweets \\\n" + " -find hdfs:///user/$USER/solrloadtest/twitter/tweets \\\n" +
" -type f \\\n" + " -type f \\\n" +
" -name 'sample-statuses*.gz' \\\n" + " -name 'sample-statuses*.gz' \\\n" +
@ -222,7 +222,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
" -size +1000000c \\\n" + " -size +1000000c \\\n" +
"| sudo -u hdfs hadoop \\\n" + "| sudo -u hdfs hadoop \\\n" +
" --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + " --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
" jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName() + " \\\n" + " jar target/solr-map-reduce-*.jar \\\n" +
" -D 'mapred.child.java.opts=-Xmx500m' \\\n" + " -D 'mapred.child.java.opts=-Xmx500m' \\\n" +
// " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + // " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" +
" --log4j src/test/resources/log4j.properties \\\n" + " --log4j src/test/resources/log4j.properties \\\n" +
@ -236,7 +236,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
"# (explicitly specify Solr URLs - for a SolrCloud cluster see next example):\n" + "# (explicitly specify Solr URLs - for a SolrCloud cluster see next example):\n" +
"sudo -u hdfs hadoop \\\n" + "sudo -u hdfs hadoop \\\n" +
" --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + " --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
" jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName() + " \\\n" + " jar target/solr-map-reduce-*.jar \\\n" +
" -D 'mapred.child.java.opts=-Xmx500m' \\\n" + " -D 'mapred.child.java.opts=-Xmx500m' \\\n" +
// " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + // " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" +
" --log4j src/test/resources/log4j.properties \\\n" + " --log4j src/test/resources/log4j.properties \\\n" +
@ -252,7 +252,7 @@ public class MapReduceIndexerTool extends Configured implements Tool {
"# (discover shards and Solr URLs through ZooKeeper):\n" + "# (discover shards and Solr URLs through ZooKeeper):\n" +
"sudo -u hdfs hadoop \\\n" + "sudo -u hdfs hadoop \\\n" +
" --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + " --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
" jar target/search-mr-*-job.jar " + MapReduceIndexerTool.class.getName() + " \\\n" + " jar target/solr-map-reduce-*.jar \\\n" +
" -D 'mapred.child.java.opts=-Xmx500m' \\\n" + " -D 'mapred.child.java.opts=-Xmx500m' \\\n" +
// " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + // " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" +
" --log4j src/test/resources/log4j.properties \\\n" + " --log4j src/test/resources/log4j.properties \\\n" +

4
solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineBasicMiniMRTest.java
View File

@ -38,17 +38,14 @@ import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.lucene.util.Constants; import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.AbstractZkTestCase; import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.hadoop.hack.MiniMRCluster; import org.apache.solr.hadoop.hack.MiniMRCluster;
import org.apache.solr.handler.extraction.ExtractingParams;
import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.ExternalPaths;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
@ -324,7 +321,6 @@ public class MorphlineBasicMiniMRTest extends SolrTestCaseJ4 {
jobConf.setMaxMapAttempts(1); jobConf.setMaxMapAttempts(1);
jobConf.setMaxReduceAttempts(1); jobConf.setMaxReduceAttempts(1);
jobConf.setJar(SEARCH_ARCHIVES_JAR); jobConf.setJar(SEARCH_ARCHIVES_JAR);
jobConf.setBoolean(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
int shards = 2; int shards = 2;
int maxReducers = Integer.MAX_VALUE; int maxReducers = Integer.MAX_VALUE;

2
solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
View File

@ -67,7 +67,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.NamedList;
import org.apache.solr.hadoop.hack.MiniMRClientCluster; import org.apache.solr.hadoop.hack.MiniMRClientCluster;
import org.apache.solr.hadoop.hack.MiniMRClientClusterFactory; import org.apache.solr.hadoop.hack.MiniMRClientClusterFactory;
import org.apache.solr.handler.extraction.ExtractingParams;
import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.ExternalPaths;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
@ -367,7 +366,6 @@ public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
jobConf.setMaxMapAttempts(1); jobConf.setMaxMapAttempts(1);
jobConf.setMaxReduceAttempts(1); jobConf.setMaxReduceAttempts(1);
jobConf.setJar(SEARCH_ARCHIVES_JAR); jobConf.setJar(SEARCH_ARCHIVES_JAR);
jobConf.setBoolean(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);
MapReduceIndexerTool tool; MapReduceIndexerTool tool;
int res; int res;

9
solr/example/scripts/map-reduce/map-reduce-indexer.bat
View File

@ -1,9 +0,0 @@
set JVM=java
REM Find location of this script
set SDIR=%~dp0
if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1%
"%JVM%" -classpath "%SDIR%\..\..\..\dist\*:%SDIR%\..\..\..\contrib\map-reduce\lib\*:%SDIR%\..\..\..\contrib\morphlines-core\lib\*:%SDIR%\..\..\..\contrib\morphlines-cell\lib\*:%SDIR%\..\..\..\contrib\extraction\lib\*:%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*:%SDIR%\..\..\lib\ext\*" org.apache.solr.hadoop.MapReduceIndexerTool %*

10
solr/example/scripts/map-reduce/map-reduce-indexer.sh
View File

@ -1,10 +0,0 @@
#!/usr/bin/env bash
JVM="java"
# Find location of this script
sdir="`dirname \"$0\"`"
PATH=$JAVA_HOME/bin:$PATH $JVM -cp "$sdir/../../../dist/*:$sdir/../../../contrib/map-reduce/lib/*:$sdir/../../../contrib/morphlines-core/lib/*:$sdir/../../../contrib/morphlines-cell/lib/*:$sdir/../../../contrib/extraction/lib/*:$sdir/../../solr-webapp/webapp/WEB-INF/lib/*:$sdir/../../lib/ext/*" org.apache.solr.hadoop.MapReduceIndexerTool ${1+"$@"}

3
solr/example/scripts/map-reduce/set-map-reduce-classpath.sh Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
export HADOOP_CLASSPATH="$sdir/../../../dist/*:$sdir/../../../contrib/map-reduce/lib/*:$sdir/../../../contrib/morphlines-core/lib/*:$sdir/../../../contrib/morphlines-cell/lib/*:$sdir/../../../contrib/extraction/lib/*:$sdir/../../solr-webapp/webapp/WEB-INF/lib/*:$sdir/../../lib/ext/*"