Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-1215 migration; use TextInputFormat instead of FileInputFormat 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@796943 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2009-07-23 05:00:19 +00:00
parent 817b42e983
commit 0063bafbc9
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/java/org/apache/hadoop/hbase/migration/nineteen/HStoreFileToStoreFile.java
View File

@ -37,7 +37,7 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.GenericOptionsParser;
@ -104,24 +104,27 @@ public class HStoreFileToStoreFile {
FileStatus [] infoAndMapfile = fs.listStatus(family); FileStatus [] infoAndMapfile = fs.listStatus(family);
// Assert that only info and mapfile in family dir. // Assert that only info and mapfile in family dir.
if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) { if (infoAndMapfile.length != 0 && infoAndMapfile.length != 2) {
throw new IOException(family.toString() + LOG.warn(family.toString() +
" has more than just info and mapfile: " + infoAndMapfile.length); " has more than just info and mapfile: " + infoAndMapfile.length + ". Continuing...");
continue;
} }
// Make sure directory named info or mapfile. // Make sure directory named info or mapfile.
for (int ll = 0; ll < 2; ll++) { for (int ll = 0; ll < 2; ll++) {
if (infoAndMapfile[ll].getPath().getName().equals("info") || if (infoAndMapfile[ll].getPath().getName().equals("info") ||
infoAndMapfile[ll].getPath().getName().equals("mapfiles")) infoAndMapfile[ll].getPath().getName().equals("mapfiles"))
continue; continue;
throw new IOException("Unexpected directory name: " + LOG.warn("Unexpected directory name: " +
infoAndMapfile[ll].getPath()); infoAndMapfile[ll].getPath() + ". Continuing...");
continue;
} }
// Now in family, there are 'mapfile' and 'info' subdirs. Just // Now in family, there are 'mapfile' and 'info' subdirs. Just
// look in the 'mapfile' subdir. // look in the 'mapfile' subdir.
FileStatus [] familyStatus = FileStatus [] familyStatus =
fs.listStatus(new Path(family, "mapfiles")); fs.listStatus(new Path(family, "mapfiles"));
if (familyStatus.length > 1) { if (familyStatus.length > 1) {
throw new IOException(family.toString() + " has " + familyStatus.length + LOG.warn(family.toString() + " has " + familyStatus.length +
" files."); " files. Continuing...");
continue;
} }
if (familyStatus.length == 1) { if (familyStatus.length == 1) {
// If we got here, then this is good. Add the mapfile to out // If we got here, then this is good. Add the mapfile to out
@ -153,7 +156,7 @@ public class HStoreFileToStoreFile {
job.setJobName(JOBNAME); job.setJobName(JOBNAME);
job.setMapperClass(Map.class); job.setMapperClass(Map.class);
job.setNumReduceTasks(0); job.setNumReduceTasks(0);
FileInputFormat.setInputPaths(job, input); TextInputFormat.setInputPaths(job, input);
Path output = new Path(args[1]); Path output = new Path(args[1]);
FileOutputFormat.setOutputPath(job, output); FileOutputFormat.setOutputPath(job, output);
System.exit(job.waitForCompletion(true) ? 0 : 1); System.exit(job.waitForCompletion(true) ? 0 : 1);