HBASE-451 Remove HTableDescriptor from HRegionInfo -- part 2, some cleanup
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1138120 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
cf25a779aa
commit
b0439269fe
|
@ -0,0 +1,75 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get, remove and modify table descriptors.
|
||||||
|
* Used by servers to host descriptors.
|
||||||
|
*/
|
||||||
|
public interface TableDescriptors {
|
||||||
|
/**
|
||||||
|
* @param tablename
|
||||||
|
* @return HTableDescriptor for tablename
|
||||||
|
* @throws TableExistsException
|
||||||
|
* @throws FileNotFoundException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public HTableDescriptor get(final String tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param tablename
|
||||||
|
* @return HTableDescriptor for tablename
|
||||||
|
* @throws TableExistsException
|
||||||
|
* @throws FileNotFoundException
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public HTableDescriptor get(final byte[] tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get Map of all HTableDescriptors. Populates the descriptor cache as a
|
||||||
|
* side effect.
|
||||||
|
* @param fs
|
||||||
|
* @param rootdir
|
||||||
|
* @return Map of all descriptors.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public Map<String, HTableDescriptor> getAll()
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add or update descriptor
|
||||||
|
* @param htd Descriptor to set into TableDescriptors
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public void add(final HTableDescriptor htd)
|
||||||
|
throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param tablename
|
||||||
|
* @return Instance of table descriptor or null if none found.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public HTableDescriptor remove(final String tablename)
|
||||||
|
throws IOException;
|
||||||
|
}
|
|
@ -25,7 +25,6 @@ import java.io.IOException;
|
||||||
import java.lang.Thread.UncaughtExceptionHandler;
|
import java.lang.Thread.UncaughtExceptionHandler;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
|
@ -46,7 +45,6 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.Chore;
|
import org.apache.hadoop.hbase.Chore;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.NotServingRegionException;
|
import org.apache.hadoop.hbase.NotServingRegionException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
|
@ -64,7 +62,6 @@ import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler;
|
||||||
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
|
import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler;
|
||||||
import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
|
import org.apache.hadoop.hbase.master.handler.SplitRegionHandler;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
import org.apache.hadoop.hbase.util.Threads;
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
import org.apache.hadoop.hbase.util.Writables;
|
import org.apache.hadoop.hbase.util.Writables;
|
||||||
|
@ -141,10 +138,6 @@ public class AssignmentManager extends ZooKeeperListener {
|
||||||
|
|
||||||
private final ExecutorService executorService;
|
private final ExecutorService executorService;
|
||||||
|
|
||||||
private Map<String, HTableDescriptor> tableDescMap =
|
|
||||||
new HashMap<String, HTableDescriptor>();
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructs a new assignment manager.
|
* Constructs a new assignment manager.
|
||||||
*
|
*
|
||||||
|
@ -153,10 +146,11 @@ public class AssignmentManager extends ZooKeeperListener {
|
||||||
* @param catalogTracker
|
* @param catalogTracker
|
||||||
* @param service
|
* @param service
|
||||||
* @throws KeeperException
|
* @throws KeeperException
|
||||||
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public AssignmentManager(Server master, ServerManager serverManager,
|
public AssignmentManager(Server master, ServerManager serverManager,
|
||||||
CatalogTracker catalogTracker, final ExecutorService service)
|
CatalogTracker catalogTracker, final ExecutorService service)
|
||||||
throws KeeperException {
|
throws KeeperException, IOException {
|
||||||
super(master.getZooKeeper());
|
super(master.getZooKeeper());
|
||||||
this.master = master;
|
this.master = master;
|
||||||
this.serverManager = serverManager;
|
this.serverManager = serverManager;
|
||||||
|
@ -172,7 +166,6 @@ public class AssignmentManager extends ZooKeeperListener {
|
||||||
this.zkTable = new ZKTable(this.master.getZooKeeper());
|
this.zkTable = new ZKTable(this.master.getZooKeeper());
|
||||||
this.maximumAssignmentAttempts =
|
this.maximumAssignmentAttempts =
|
||||||
this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
|
this.master.getConfiguration().getInt("hbase.assignment.maximum.attempts", 10);
|
||||||
initHTableDescriptorMap();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1070,10 +1063,6 @@ public class AssignmentManager extends ZooKeeperListener {
|
||||||
}
|
}
|
||||||
// Move on to open regions.
|
// Move on to open regions.
|
||||||
try {
|
try {
|
||||||
// Update the tableDesc map.
|
|
||||||
for (HRegionInfo region : regions) {
|
|
||||||
updateDescMap(region.getTableNameAsString());
|
|
||||||
}
|
|
||||||
// Send OPEN RPC. This can fail if the server on other end is is not up.
|
// Send OPEN RPC. This can fail if the server on other end is is not up.
|
||||||
// If we fail, fail the startup by aborting the server. There is one
|
// If we fail, fail the startup by aborting the server. There is one
|
||||||
// exception we will tolerate: ServerNotRunningException. This is thrown
|
// exception we will tolerate: ServerNotRunningException. This is thrown
|
||||||
|
@ -2257,140 +2246,6 @@ public class AssignmentManager extends ZooKeeperListener {
|
||||||
LOG.info("Bulk assigning done");
|
LOG.info("Bulk assigning done");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void initHTableDescriptorMap() {
|
|
||||||
try {
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
this.tableDescMap =
|
|
||||||
FSUtils.getTableDescriptors(this.master.getConfiguration());
|
|
||||||
}
|
|
||||||
} catch (IOException e) {
|
|
||||||
LOG.info("IOException while initializing HTableDescriptor Map");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private HTableDescriptor readTableDescriptor(String tableName)
|
|
||||||
throws IOException {
|
|
||||||
return FSUtils.getHTableDescriptor(
|
|
||||||
this.master.getConfiguration(), tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
private boolean isRootOrMetaRegion(String tableName) {
|
|
||||||
return (
|
|
||||||
tableName.equals(
|
|
||||||
HRegionInfo.ROOT_REGIONINFO.getTableNameAsString())
|
|
||||||
||
|
|
||||||
tableName.equals(
|
|
||||||
HRegionInfo.FIRST_META_REGIONINFO.getTableNameAsString()));
|
|
||||||
}
|
|
||||||
|
|
||||||
private void updateDescMap(String tableName) throws IOException {
|
|
||||||
|
|
||||||
if (this.tableDescMap == null) {
|
|
||||||
LOG.error("Table Descriptor cache is null. " +
|
|
||||||
"Skipping desc map update for table = " + tableName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (tableName == null || isRootOrMetaRegion(tableName))
|
|
||||||
return;
|
|
||||||
if (!this.tableDescMap.containsKey(tableName)) {
|
|
||||||
HTableDescriptor htd = readTableDescriptor(tableName);
|
|
||||||
if (htd != null) {
|
|
||||||
LOG.info("Updating TableDesc Map for tablename = " + tableName
|
|
||||||
+ "htd == " + htd);
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
this.tableDescMap.put(tableName, htd);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
LOG.info("HTable Descriptor is NULL for table = " + tableName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void updateTableDesc(String tableName, HTableDescriptor htd) {
|
|
||||||
if (this.tableDescMap == null) {
|
|
||||||
LOG.error("Table Descriptor cache is null. " +
|
|
||||||
"Skipping desc map update for table = " + tableName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (tableName == null || isRootOrMetaRegion(tableName))
|
|
||||||
return;
|
|
||||||
if (!this.tableDescMap.containsKey(tableName)) {
|
|
||||||
LOG.error("Table descriptor missing in DescMap. for tablename = " + tableName);
|
|
||||||
}
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
this.tableDescMap.put(tableName, htd);
|
|
||||||
}
|
|
||||||
LOG.info("TableDesc updated successfully for table = " + tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void deleteTableDesc(String tableName) {
|
|
||||||
if (this.tableDescMap == null) {
|
|
||||||
LOG.error("Table Descriptor cache is null. " +
|
|
||||||
"Skipping desc map update for table = " + tableName);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (tableName == null || isRootOrMetaRegion(tableName))
|
|
||||||
return;
|
|
||||||
if (!this.tableDescMap.containsKey(tableName)) {
|
|
||||||
LOG.error("Table descriptor missing in DescMap. for tablename = " + tableName);
|
|
||||||
}
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
this.tableDescMap.remove(tableName);
|
|
||||||
}
|
|
||||||
LOG.info("TableDesc removed successfully for table = " + tableName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
|
|
||||||
List htdList = null;
|
|
||||||
HTableDescriptor[] htd = null;
|
|
||||||
if (tableNames != null && tableNames.size() > 0) {
|
|
||||||
if (this.tableDescMap != null) {
|
|
||||||
htd = new HTableDescriptor[tableNames.size()];
|
|
||||||
htdList = new ArrayList();
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
int index = 0;
|
|
||||||
for (String tableName : tableNames) {
|
|
||||||
HTableDescriptor htdesc = this.tableDescMap.get(tableName);
|
|
||||||
htd[index++] = this.tableDescMap.get(tableName);
|
|
||||||
if (htdesc != null) {
|
|
||||||
htdList.add(htdesc);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (htdList != null && htdList.size() > 0 ) {
|
|
||||||
return (HTableDescriptor[]) htdList.toArray(new HTableDescriptor[htdList.size()]);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public HTableDescriptor[] getHTableDescriptors() {
|
|
||||||
if (this.tableDescMap != null) {
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
Collection<HTableDescriptor> htdc = this.tableDescMap.values();
|
|
||||||
if (htdc != null) {
|
|
||||||
return htdc.toArray(new HTableDescriptor[htdc.size()]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public HTableDescriptor getTableDescriptor(String tableName) {
|
|
||||||
HTableDescriptor htd = null;
|
|
||||||
if (tableName != null) {
|
|
||||||
synchronized (this.tableDescMap) {
|
|
||||||
htd = this.tableDescMap.get(tableName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return htd;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* State of a Region while undergoing transitions.
|
* State of a Region while undergoing transitions.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.master;
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
|
@ -36,6 +37,7 @@ import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
|
import org.apache.hadoop.hbase.TableExistsException;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaReader;
|
import org.apache.hadoop.hbase.catalog.MetaReader;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
|
@ -283,9 +285,8 @@ class CatalogJanitor extends Chore {
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
private HTableDescriptor getTableDescriptor(byte[] tableName) {
|
private HTableDescriptor getTableDescriptor(byte[] tableName)
|
||||||
return this.services.getAssignmentManager().getTableDescriptor(
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
Bytes.toString(tableName));
|
return this.services.getTableDescriptors().get(Bytes.toString(tableName));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
|
||||||
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
|
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
import org.apache.hadoop.hbase.TableExistsException;
|
import org.apache.hadoop.hbase.TableExistsException;
|
||||||
import org.apache.hadoop.hbase.TableNotDisabledException;
|
import org.apache.hadoop.hbase.TableNotDisabledException;
|
||||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||||
|
@ -78,6 +79,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
import org.apache.hadoop.hbase.replication.regionserver.Replication;
|
import org.apache.hadoop.hbase.replication.regionserver.Replication;
|
||||||
import org.apache.hadoop.hbase.security.User;
|
import org.apache.hadoop.hbase.security.User;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.FSTableDescriptors;
|
||||||
import org.apache.hadoop.hbase.util.InfoServer;
|
import org.apache.hadoop.hbase.util.InfoServer;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
import org.apache.hadoop.hbase.util.Sleeper;
|
import org.apache.hadoop.hbase.util.Sleeper;
|
||||||
|
@ -178,6 +180,8 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
private MasterCoprocessorHost cpHost;
|
private MasterCoprocessorHost cpHost;
|
||||||
private final ServerName serverName;
|
private final ServerName serverName;
|
||||||
|
|
||||||
|
private TableDescriptors tableDescriptors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initializes the HMaster. The steps are as follows:
|
* Initializes the HMaster. The steps are as follows:
|
||||||
* <p>
|
* <p>
|
||||||
|
@ -410,7 +414,11 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
|
|
||||||
status.setStatus("Initializing Master file system");
|
status.setStatus("Initializing Master file system");
|
||||||
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
|
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
|
||||||
this.fileSystemManager = new MasterFileSystem(this, metrics);
|
this.fileSystemManager = new MasterFileSystem(this, this, metrics);
|
||||||
|
|
||||||
|
this.tableDescriptors =
|
||||||
|
new FSTableDescriptors(this.fileSystemManager.getFileSystem(),
|
||||||
|
this.fileSystemManager.getRootDir());
|
||||||
|
|
||||||
// publish cluster ID
|
// publish cluster ID
|
||||||
status.setStatus("Publishing Cluster ID in ZooKeeper");
|
status.setStatus("Publishing Cluster ID in ZooKeeper");
|
||||||
|
@ -579,6 +587,11 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TableDescriptors getTableDescriptors() {
|
||||||
|
return this.tableDescriptors;
|
||||||
|
}
|
||||||
|
|
||||||
/** @return InfoServer object. Maybe null.*/
|
/** @return InfoServer object. Maybe null.*/
|
||||||
public InfoServer getInfoServer() {
|
public InfoServer getInfoServer() {
|
||||||
return this.infoServer;
|
return this.infoServer;
|
||||||
|
@ -948,11 +961,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
return hRegionInfos;
|
return hRegionInfos;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void storeTableDescriptor(HTableDescriptor hTableDescriptor)
|
|
||||||
throws IOException {
|
|
||||||
FSUtils.createTableDescriptor(hTableDescriptor, conf);
|
|
||||||
}
|
|
||||||
|
|
||||||
private synchronized void createTable(final HTableDescriptor hTableDescriptor,
|
private synchronized void createTable(final HTableDescriptor hTableDescriptor,
|
||||||
final HRegionInfo [] newRegions,
|
final HRegionInfo [] newRegions,
|
||||||
final boolean sync)
|
final boolean sync)
|
||||||
|
@ -961,7 +969,11 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
if (MetaReader.tableExists(catalogTracker, tableName)) {
|
if (MetaReader.tableExists(catalogTracker, tableName)) {
|
||||||
throw new TableExistsException(tableName);
|
throw new TableExistsException(tableName);
|
||||||
}
|
}
|
||||||
storeTableDescriptor(hTableDescriptor);
|
// TODO: Currently we make the table descriptor and as side-effect the
|
||||||
|
// tableDir is created. Should we change below method to be createTable
|
||||||
|
// where we create table in tmp dir with its table descriptor file and then
|
||||||
|
// do rename to move it into place?
|
||||||
|
FSUtils.createTableDescriptor(hTableDescriptor, conf);
|
||||||
|
|
||||||
for (HRegionInfo newRegion : newRegions) {
|
for (HRegionInfo newRegion : newRegions) {
|
||||||
// 1. Set table enabling flag up in zk.
|
// 1. Set table enabling flag up in zk.
|
||||||
|
@ -1378,32 +1390,35 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server {
|
||||||
* @return HTableDescriptor[]
|
* @return HTableDescriptor[]
|
||||||
*/
|
*/
|
||||||
public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
|
public HTableDescriptor[] getHTableDescriptors(List<String> tableNames) {
|
||||||
return this.assignmentManager.getHTableDescriptors(tableNames);
|
List<HTableDescriptor> list =
|
||||||
|
new ArrayList<HTableDescriptor>(tableNames.size());
|
||||||
|
for (String s: tableNames) {
|
||||||
|
HTableDescriptor htd = null;
|
||||||
|
try {
|
||||||
|
htd = this.tableDescriptors.get(s);
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.warn("Failed getting descriptor for " + s, e);
|
||||||
|
}
|
||||||
|
if (htd == null) continue;
|
||||||
|
list.add(htd);
|
||||||
|
}
|
||||||
|
return list.toArray(new HTableDescriptor [] {});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all table descriptors
|
* Get all table descriptors
|
||||||
* @return HTableDescriptor[]
|
* @return All descriptors or null if none.
|
||||||
*/
|
*/
|
||||||
public HTableDescriptor [] getHTableDescriptors() {
|
public HTableDescriptor [] getHTableDescriptors() {
|
||||||
return this.assignmentManager.getHTableDescriptors();
|
Map<String, HTableDescriptor> descriptors = null;
|
||||||
|
try {
|
||||||
|
descriptors = this.tableDescriptors.getAll();
|
||||||
|
} catch (IOException e) {
|
||||||
|
LOG.warn("Failed getting all descriptors", e);
|
||||||
}
|
}
|
||||||
|
return descriptors == null?
|
||||||
/**
|
null: descriptors.values().toArray(new HTableDescriptor [] {});
|
||||||
* Get a HTD for a given table name
|
|
||||||
* @param tableName
|
|
||||||
* @return HTableDescriptor
|
|
||||||
*/
|
|
||||||
/*
|
|
||||||
public HTableDescriptor getHTableDescriptor(byte[] tableName) {
|
|
||||||
if (tableName != null && tableName.length > 0) {
|
|
||||||
return this.assignmentManager.getTableDescriptor(
|
|
||||||
Bytes.toString(tableName));
|
|
||||||
}
|
}
|
||||||
return null;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compute the average load across all region servers.
|
* Compute the average load across all region servers.
|
||||||
|
|
|
@ -33,19 +33,16 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileStatus;
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.FSDataInputStream;
|
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
||||||
import org.apache.hadoop.hbase.RemoteExceptionHandler;
|
import org.apache.hadoop.hbase.RemoteExceptionHandler;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
|
import org.apache.hadoop.hbase.master.metrics.MasterMetrics;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
import org.apache.hadoop.hbase.regionserver.Store;
|
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
|
import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
|
import org.apache.hadoop.hbase.regionserver.wal.OrphanHLogAfterSplitException;
|
||||||
|
@ -80,11 +77,14 @@ public class MasterFileSystem {
|
||||||
final Lock splitLogLock = new ReentrantLock();
|
final Lock splitLogLock = new ReentrantLock();
|
||||||
final boolean distributedLogSplitting;
|
final boolean distributedLogSplitting;
|
||||||
final SplitLogManager splitLogManager;
|
final SplitLogManager splitLogManager;
|
||||||
|
private final MasterServices services;
|
||||||
|
|
||||||
public MasterFileSystem(Server master, MasterMetrics metrics)
|
public MasterFileSystem(Server master, MasterServices services,
|
||||||
|
MasterMetrics metrics)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this.conf = master.getConfiguration();
|
this.conf = master.getConfiguration();
|
||||||
this.master = master;
|
this.master = master;
|
||||||
|
this.services = services;
|
||||||
this.metrics = metrics;
|
this.metrics = metrics;
|
||||||
// Set filesystem to be that of this.rootdir else we get complaints about
|
// Set filesystem to be that of this.rootdir else we get complaints about
|
||||||
// mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
|
// mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
|
||||||
|
@ -410,28 +410,6 @@ public class MasterFileSystem {
|
||||||
return tableInfoPath;
|
return tableInfoPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get table info path for a table.
|
|
||||||
* @param tableName
|
|
||||||
* @return Table info path
|
|
||||||
*/
|
|
||||||
private Path getTablePath(byte[] tableName) {
|
|
||||||
return new Path(this.rootdir, Bytes.toString(tableName));
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Get a HTableDescriptor of a table.
|
|
||||||
* @param tableName
|
|
||||||
* @return HTableDescriptor
|
|
||||||
*/
|
|
||||||
public HTableDescriptor getTableDescriptor(byte[] tableName) {
|
|
||||||
try {
|
|
||||||
return FSUtils.getTableDescriptor(fs, this.rootdir, tableName);
|
|
||||||
} catch (IOException ioe) {
|
|
||||||
LOG.info("Exception during readTableDecriptor ", ioe);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create new HTableDescriptor in HDFS.
|
* Create new HTableDescriptor in HDFS.
|
||||||
* @param htableDescriptor
|
* @param htableDescriptor
|
||||||
|
@ -440,19 +418,6 @@ public class MasterFileSystem {
|
||||||
FSUtils.createTableDescriptor(htableDescriptor, conf);
|
FSUtils.createTableDescriptor(htableDescriptor, conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Update a table descriptor.
|
|
||||||
* @param htableDescriptor
|
|
||||||
* @return updated HTableDescriptor
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
public HTableDescriptor updateTableDescriptor(HTableDescriptor htableDescriptor)
|
|
||||||
throws IOException {
|
|
||||||
LOG.info("Update Table Descriptor. Current HTD = " + htableDescriptor);
|
|
||||||
FSUtils.updateHTableDescriptor(fs, conf, htableDescriptor);
|
|
||||||
return htableDescriptor;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete column of a table
|
* Delete column of a table
|
||||||
* @param tableName
|
* @param tableName
|
||||||
|
@ -464,9 +429,9 @@ public class MasterFileSystem {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
|
LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
|
||||||
+ " family = " + Bytes.toString(familyName));
|
+ " family = " + Bytes.toString(familyName));
|
||||||
HTableDescriptor htd = getTableDescriptor(tableName);
|
HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
|
||||||
htd.removeFamily(familyName);
|
htd.removeFamily(familyName);
|
||||||
updateTableDescriptor(htd);
|
this.services.getTableDescriptors().add(htd);
|
||||||
return htd;
|
return htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -482,14 +447,14 @@ public class MasterFileSystem {
|
||||||
LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
|
LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
|
||||||
+ " HCD = " + hcd.toString());
|
+ " HCD = " + hcd.toString());
|
||||||
|
|
||||||
HTableDescriptor htd = getTableDescriptor(tableName);
|
HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
|
||||||
byte [] familyName = hcd.getName();
|
byte [] familyName = hcd.getName();
|
||||||
if(!htd.hasFamily(familyName)) {
|
if(!htd.hasFamily(familyName)) {
|
||||||
throw new InvalidFamilyOperationException("Family '" +
|
throw new InvalidFamilyOperationException("Family '" +
|
||||||
Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
|
Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
|
||||||
}
|
}
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
updateTableDescriptor(htd);
|
this.services.getTableDescriptors().add(htd);
|
||||||
return htd;
|
return htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -502,17 +467,15 @@ public class MasterFileSystem {
|
||||||
*/
|
*/
|
||||||
public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
|
public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
LOG.info("AddColumn. Table = " + Bytes.toString(tableName)
|
LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
|
||||||
+ " HCD = " + hcd.toString());
|
hcd.toString());
|
||||||
|
HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
|
||||||
HTableDescriptor htd = getTableDescriptor(tableName);
|
|
||||||
if (htd == null) {
|
if (htd == null) {
|
||||||
throw new InvalidFamilyOperationException("Family '" +
|
throw new InvalidFamilyOperationException("Family '" +
|
||||||
hcd.getNameAsString() + "' cannot be modified as HTD is null");
|
hcd.getNameAsString() + "' cannot be modified as HTD is null");
|
||||||
}
|
}
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
updateTableDescriptor(htd);
|
this.services.getTableDescriptors().add(htd);
|
||||||
return htd;
|
return htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -22,11 +22,10 @@ package org.apache.hadoop.hbase.master;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
import org.apache.hadoop.hbase.TableNotDisabledException;
|
import org.apache.hadoop.hbase.TableNotDisabledException;
|
||||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||||
import org.apache.hadoop.hbase.catalog.CatalogTracker;
|
|
||||||
import org.apache.hadoop.hbase.executor.ExecutorService;
|
import org.apache.hadoop.hbase.executor.ExecutorService;
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Services Master supplies
|
* Services Master supplies
|
||||||
|
@ -60,4 +59,8 @@ public interface MasterServices extends Server {
|
||||||
*/
|
*/
|
||||||
public void checkTableModifiable(final byte [] tableName) throws IOException;
|
public void checkTableModifiable(final byte [] tableName) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return Return table descriptors implementation.
|
||||||
|
*/
|
||||||
|
public TableDescriptors getTableDescriptors();
|
||||||
}
|
}
|
||||||
|
|
|
@ -231,7 +231,7 @@ public class SplitLogManager extends ZooKeeperListener {
|
||||||
for (FileStatus lf : logfiles) {
|
for (FileStatus lf : logfiles) {
|
||||||
// TODO If the log file is still being written to - which is most likely
|
// TODO If the log file is still being written to - which is most likely
|
||||||
// the case for the last log file - then its length will show up here
|
// the case for the last log file - then its length will show up here
|
||||||
// as zero. The size of such a file can only be retrieved after after
|
// as zero. The size of such a file can only be retrieved after
|
||||||
// recover-lease is done. totalSize will be under in most cases and the
|
// recover-lease is done. totalSize will be under in most cases and the
|
||||||
// metrics that it drives will also be under-reported.
|
// metrics that it drives will also be under-reported.
|
||||||
totalSize += lf.getLen();
|
totalSize += lf.getLen();
|
||||||
|
|
|
@ -71,7 +71,7 @@ public class DeleteTableHandler extends TableEventHandler {
|
||||||
// Delete table from FS
|
// Delete table from FS
|
||||||
this.masterServices.getMasterFileSystem().deleteTable(tableName);
|
this.masterServices.getMasterFileSystem().deleteTable(tableName);
|
||||||
// Update table descriptor cache
|
// Update table descriptor cache
|
||||||
am.deleteTableDesc(Bytes.toString(tableName));
|
this.masterServices.getTableDescriptors().remove(Bytes.toString(tableName));
|
||||||
|
|
||||||
// If entry for this table in zk, and up in AssignmentManager, remove it.
|
// If entry for this table in zk, and up in AssignmentManager, remove it.
|
||||||
// Call to undisableTable does this. TODO: Make a more formal purge table.
|
// Call to undisableTable does this. TODO: Make a more formal purge table.
|
||||||
|
|
|
@ -25,10 +25,7 @@ import java.util.List;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
|
||||||
import org.apache.hadoop.hbase.master.AssignmentManager;
|
|
||||||
import org.apache.hadoop.hbase.master.MasterServices;
|
import org.apache.hadoop.hbase.master.MasterServices;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
|
||||||
|
|
||||||
public class ModifyTableHandler extends TableEventHandler {
|
public class ModifyTableHandler extends TableEventHandler {
|
||||||
private final HTableDescriptor htd;
|
private final HTableDescriptor htd;
|
||||||
|
@ -43,20 +40,10 @@ public class ModifyTableHandler extends TableEventHandler {
|
||||||
@Override
|
@Override
|
||||||
protected void handleTableOperation(List<HRegionInfo> hris)
|
protected void handleTableOperation(List<HRegionInfo> hris)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
AssignmentManager am = this.masterServices.getAssignmentManager();
|
// Update descriptor
|
||||||
HTableDescriptor htd = am.getTableDescriptor(Bytes.toString(tableName));
|
this.masterServices.getTableDescriptors().add(this.htd);
|
||||||
if (htd == null) {
|
|
||||||
throw new IOException("Modify Table operation could not be completed as " +
|
|
||||||
"HTableDescritor is missing for table = "
|
|
||||||
+ Bytes.toString(tableName));
|
|
||||||
}
|
}
|
||||||
// Update table descriptor in HDFS
|
|
||||||
|
|
||||||
HTableDescriptor updatedHTD = this.masterServices.getMasterFileSystem()
|
|
||||||
.updateTableDescriptor(this.htd);
|
|
||||||
// Update in-memory descriptor cache
|
|
||||||
am.updateTableDesc(Bytes.toString(tableName), updatedHTD);
|
|
||||||
}
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
String name = "UnknownServerName";
|
String name = "UnknownServerName";
|
||||||
|
|
|
@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
|
||||||
import org.apache.hadoop.hbase.master.AssignmentManager;
|
import org.apache.hadoop.hbase.master.AssignmentManager;
|
||||||
import org.apache.hadoop.hbase.master.MasterServices;
|
import org.apache.hadoop.hbase.master.MasterServices;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
@ -49,7 +48,7 @@ public class TableAddFamilyHandler extends TableEventHandler {
|
||||||
protected void handleTableOperation(List<HRegionInfo> hris)
|
protected void handleTableOperation(List<HRegionInfo> hris)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
AssignmentManager am = this.masterServices.getAssignmentManager();
|
AssignmentManager am = this.masterServices.getAssignmentManager();
|
||||||
HTableDescriptor htd = am.getTableDescriptor(Bytes.toString(tableName));
|
HTableDescriptor htd = this.masterServices.getTableDescriptors().get(Bytes.toString(tableName));
|
||||||
byte [] familyName = familyDesc.getName();
|
byte [] familyName = familyDesc.getName();
|
||||||
if (htd == null) {
|
if (htd == null) {
|
||||||
throw new IOException("Add Family operation could not be completed as " +
|
throw new IOException("Add Family operation could not be completed as " +
|
||||||
|
@ -65,7 +64,7 @@ public class TableAddFamilyHandler extends TableEventHandler {
|
||||||
htd = this.masterServices.getMasterFileSystem()
|
htd = this.masterServices.getMasterFileSystem()
|
||||||
.addColumn(tableName, familyDesc);
|
.addColumn(tableName, familyDesc);
|
||||||
// Update in-memory descriptor cache
|
// Update in-memory descriptor cache
|
||||||
am.updateTableDesc(Bytes.toString(tableName), htd);
|
this.masterServices.getTableDescriptors().add(htd);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
|
|
|
@ -26,9 +26,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
|
||||||
import org.apache.hadoop.hbase.master.AssignmentManager;
|
import org.apache.hadoop.hbase.master.AssignmentManager;
|
||||||
import org.apache.hadoop.hbase.master.MasterFileSystem;
|
|
||||||
import org.apache.hadoop.hbase.master.MasterServices;
|
import org.apache.hadoop.hbase.master.MasterServices;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
|
||||||
|
@ -48,7 +46,7 @@ public class TableDeleteFamilyHandler extends TableEventHandler {
|
||||||
@Override
|
@Override
|
||||||
protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
|
protected void handleTableOperation(List<HRegionInfo> hris) throws IOException {
|
||||||
AssignmentManager am = this.masterServices.getAssignmentManager();
|
AssignmentManager am = this.masterServices.getAssignmentManager();
|
||||||
HTableDescriptor htd = am.getTableDescriptor(Bytes.toString(tableName));
|
HTableDescriptor htd = this.masterServices.getTableDescriptors().get(Bytes.toString(tableName));
|
||||||
if (htd == null) {
|
if (htd == null) {
|
||||||
throw new IOException("Add Family operation could not be completed as " +
|
throw new IOException("Add Family operation could not be completed as " +
|
||||||
"HTableDescritor is missing for table = "
|
"HTableDescritor is missing for table = "
|
||||||
|
@ -63,7 +61,7 @@ public class TableDeleteFamilyHandler extends TableEventHandler {
|
||||||
htd = this.masterServices.getMasterFileSystem()
|
htd = this.masterServices.getMasterFileSystem()
|
||||||
.deleteColumn(tableName, familyName);
|
.deleteColumn(tableName, familyName);
|
||||||
// Update in-memory descriptor cache
|
// Update in-memory descriptor cache
|
||||||
am.updateTableDesc(Bytes.toString(tableName), htd);
|
this.masterServices.getTableDescriptors().add(htd);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
import org.apache.hadoop.hbase.InvalidFamilyOperationException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.catalog.MetaEditor;
|
|
||||||
import org.apache.hadoop.hbase.master.AssignmentManager;
|
import org.apache.hadoop.hbase.master.AssignmentManager;
|
||||||
import org.apache.hadoop.hbase.master.MasterServices;
|
import org.apache.hadoop.hbase.master.MasterServices;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
@ -49,7 +48,7 @@ public class TableModifyFamilyHandler extends TableEventHandler {
|
||||||
@Override
|
@Override
|
||||||
protected void handleTableOperation(List<HRegionInfo> regions) throws IOException {
|
protected void handleTableOperation(List<HRegionInfo> regions) throws IOException {
|
||||||
AssignmentManager am = this.masterServices.getAssignmentManager();
|
AssignmentManager am = this.masterServices.getAssignmentManager();
|
||||||
HTableDescriptor htd = am.getTableDescriptor(Bytes.toString(tableName));
|
HTableDescriptor htd = this.masterServices.getTableDescriptors().get(Bytes.toString(tableName));
|
||||||
byte [] familyName = familyDesc.getName();
|
byte [] familyName = familyDesc.getName();
|
||||||
if (htd == null) {
|
if (htd == null) {
|
||||||
throw new IOException("Modify Family operation could not be completed as " +
|
throw new IOException("Modify Family operation could not be completed as " +
|
||||||
|
@ -61,10 +60,9 @@ public class TableModifyFamilyHandler extends TableEventHandler {
|
||||||
Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
|
Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
|
||||||
}
|
}
|
||||||
// Update table descriptor in HDFS
|
// Update table descriptor in HDFS
|
||||||
htd = this.masterServices.getMasterFileSystem()
|
htd = this.masterServices.getMasterFileSystem().modifyColumn(tableName, familyDesc);
|
||||||
.modifyColumn(tableName, familyDesc);
|
|
||||||
// Update in-memory descriptor cache
|
// Update in-memory descriptor cache
|
||||||
am.updateTableDesc(Bytes.toString(tableName), htd);
|
this.masterServices.getTableDescriptors().add(htd);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
|
|
|
@ -199,8 +199,6 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
final Path regiondir;
|
final Path regiondir;
|
||||||
KeyValue.KVComparator comparator;
|
KeyValue.KVComparator comparator;
|
||||||
|
|
||||||
private Pair<Long,Long> lastCompactInfo = null;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Data structure of write state flags used coordinating flushes,
|
* Data structure of write state flags used coordinating flushes,
|
||||||
* compactions and closes.
|
* compactions and closes.
|
||||||
|
@ -282,6 +280,7 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
this.log = null;
|
this.log = null;
|
||||||
this.regiondir = null;
|
this.regiondir = null;
|
||||||
this.regionInfo = null;
|
this.regionInfo = null;
|
||||||
|
this.htableDescriptor = null;
|
||||||
this.threadWakeFrequency = 0L;
|
this.threadWakeFrequency = 0L;
|
||||||
this.coprocessorHost = null;
|
this.coprocessorHost = null;
|
||||||
}
|
}
|
||||||
|
@ -310,26 +309,22 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
* @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, org.apache.hadoop.hbase.HRegionInfo, FlushRequester)
|
* @see HRegion#newHRegion(Path, HLog, FileSystem, Configuration, org.apache.hadoop.hbase.HRegionInfo, FlushRequester)
|
||||||
*/
|
*/
|
||||||
public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
|
public HRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
|
||||||
HRegionInfo regionInfo, RegionServerServices rsServices) {
|
HRegionInfo regionInfo, final HTableDescriptor htd,
|
||||||
|
RegionServerServices rsServices) {
|
||||||
this.tableDir = tableDir;
|
this.tableDir = tableDir;
|
||||||
this.comparator = regionInfo.getComparator();
|
this.comparator = regionInfo.getComparator();
|
||||||
this.log = log;
|
this.log = log;
|
||||||
this.fs = fs;
|
this.fs = fs;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.regionInfo = regionInfo;
|
this.regionInfo = regionInfo;
|
||||||
|
this.htableDescriptor = htd;
|
||||||
this.rsServices = rsServices;
|
this.rsServices = rsServices;
|
||||||
this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
|
this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY,
|
||||||
10 * 1000);
|
10 * 1000);
|
||||||
String encodedNameStr = this.regionInfo.getEncodedName();
|
String encodedNameStr = this.regionInfo.getEncodedName();
|
||||||
|
setHTableSpecificConf();
|
||||||
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
|
this.regiondir = getRegionDir(this.tableDir, encodedNameStr);
|
||||||
try {
|
|
||||||
LOG.info("Setting table desc from HDFS. Region = "
|
|
||||||
+ this.regionInfo.getTableNameAsString());
|
|
||||||
loadHTableDescriptor(tableDir);
|
|
||||||
LOG.info(" This HTD from HDFS == " + this.htableDescriptor);
|
|
||||||
} catch (IOException ioe) {
|
|
||||||
LOG.error("Could not instantiate region as error loading HTableDescriptor");
|
|
||||||
}
|
|
||||||
// don't initialize coprocessors if not running within a regionserver
|
// don't initialize coprocessors if not running within a regionserver
|
||||||
// TODO: revisit if coprocessors should load in other cases
|
// TODO: revisit if coprocessors should load in other cases
|
||||||
if (rsServices != null) {
|
if (rsServices != null) {
|
||||||
|
@ -341,24 +336,8 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void loadHTableDescriptor(Path tableDir) throws IOException {
|
void setHTableSpecificConf() {
|
||||||
LOG.debug("Assigning tabledesc from .tableinfo for region = "
|
if (this.htableDescriptor == null) return;
|
||||||
+ this.regionInfo.getRegionNameAsString());
|
|
||||||
// load HTableDescriptor
|
|
||||||
this.htableDescriptor = FSUtils.getTableDescriptor(tableDir, fs);
|
|
||||||
|
|
||||||
if (this.htableDescriptor != null) {
|
|
||||||
setHTableSpecificConf();
|
|
||||||
} else {
|
|
||||||
throw new IOException("Table description missing in " +
|
|
||||||
".tableinfo. Cannot create new region."
|
|
||||||
+ " current region is == " + this.regionInfo.toString());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setHTableSpecificConf() {
|
|
||||||
if (this.htableDescriptor != null) {
|
|
||||||
LOG.info("Setting up tabledescriptor config now ...");
|
LOG.info("Setting up tabledescriptor config now ...");
|
||||||
long flushSize = this.htableDescriptor.getMemStoreFlushSize();
|
long flushSize = this.htableDescriptor.getMemStoreFlushSize();
|
||||||
if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
|
if (flushSize == HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE) {
|
||||||
|
@ -369,11 +348,6 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
this.blockingMemStoreSize = this.memstoreFlushSize *
|
this.blockingMemStoreSize = this.memstoreFlushSize *
|
||||||
conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
|
conf.getLong("hbase.hregion.memstore.block.multiplier", 2);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
public void setHtableDescriptor(HTableDescriptor htableDescriptor) {
|
|
||||||
this.htableDescriptor = htableDescriptor;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize this region.
|
* Initialize this region.
|
||||||
|
@ -2763,11 +2737,12 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
* @param conf is global configuration settings.
|
* @param conf is global configuration settings.
|
||||||
* @param regionInfo - HRegionInfo that describes the region
|
* @param regionInfo - HRegionInfo that describes the region
|
||||||
* is new), then read them from the supplied path.
|
* is new), then read them from the supplied path.
|
||||||
|
* @param htd
|
||||||
* @param rsServices
|
* @param rsServices
|
||||||
* @return the new instance
|
* @return the new instance
|
||||||
*/
|
*/
|
||||||
public static HRegion newHRegion(Path tableDir, HLog log, FileSystem fs,
|
public static HRegion newHRegion(Path tableDir, HLog log, FileSystem fs,
|
||||||
Configuration conf, HRegionInfo regionInfo,
|
Configuration conf, HRegionInfo regionInfo, final HTableDescriptor htd,
|
||||||
RegionServerServices rsServices) {
|
RegionServerServices rsServices) {
|
||||||
try {
|
try {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@ -2776,9 +2751,10 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
|
|
||||||
Constructor<? extends HRegion> c =
|
Constructor<? extends HRegion> c =
|
||||||
regionClass.getConstructor(Path.class, HLog.class, FileSystem.class,
|
regionClass.getConstructor(Path.class, HLog.class, FileSystem.class,
|
||||||
Configuration.class, HRegionInfo.class, RegionServerServices.class);
|
Configuration.class, HRegionInfo.class, HTableDescriptor.class,
|
||||||
|
RegionServerServices.class);
|
||||||
|
|
||||||
return c.newInstance(tableDir, log, fs, conf, regionInfo, rsServices);
|
return c.newInstance(tableDir, log, fs, conf, regionInfo, htd, rsServices);
|
||||||
} catch (Throwable e) {
|
} catch (Throwable e) {
|
||||||
// todo: what should I throw here?
|
// todo: what should I throw here?
|
||||||
throw new IllegalStateException("Could not instantiate a region instance.", e);
|
throw new IllegalStateException("Could not instantiate a region instance.", e);
|
||||||
|
@ -2800,8 +2776,7 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
|
public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
|
||||||
final Configuration conf,
|
final Configuration conf, final HTableDescriptor hTableDescriptor)
|
||||||
final HTableDescriptor hTableDescriptor)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
LOG.info("creating HRegion " + info.getTableNameAsString()
|
LOG.info("creating HRegion " + info.getTableNameAsString()
|
||||||
+ " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
|
+ " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
|
||||||
|
@ -2812,11 +2787,10 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
|
Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
fs.mkdirs(regionDir);
|
fs.mkdirs(regionDir);
|
||||||
FSUtils.createTableDescriptor(fs, hTableDescriptor, tableDir);
|
|
||||||
HRegion region = HRegion.newHRegion(tableDir,
|
HRegion region = HRegion.newHRegion(tableDir,
|
||||||
new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
|
new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
|
||||||
new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
|
new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
|
||||||
fs, conf, info, null);
|
fs, conf, info, hTableDescriptor, null);
|
||||||
region.initialize();
|
region.initialize();
|
||||||
return region;
|
return region;
|
||||||
}
|
}
|
||||||
|
@ -2833,10 +2807,11 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
*
|
*
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HRegion openHRegion(final HRegionInfo info, final HLog wal,
|
public static HRegion openHRegion(final HRegionInfo info,
|
||||||
|
final HTableDescriptor htd, final HLog wal,
|
||||||
final Configuration conf)
|
final Configuration conf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return openHRegion(info, wal, conf, null, null);
|
return openHRegion(info, htd, wal, conf, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -2853,8 +2828,9 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
*
|
*
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HRegion openHRegion(final HRegionInfo info, final HLog wal,
|
public static HRegion openHRegion(final HRegionInfo info,
|
||||||
final Configuration conf, final RegionServerServices rsServices,
|
final HTableDescriptor htd, final HLog wal, final Configuration conf,
|
||||||
|
final RegionServerServices rsServices,
|
||||||
final CancelableProgressable reporter)
|
final CancelableProgressable reporter)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
|
@ -2866,14 +2842,14 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
|
Path dir = HTableDescriptor.getTableDir(FSUtils.getRootDir(conf),
|
||||||
info.getTableName());
|
info.getTableName());
|
||||||
HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
|
HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
|
||||||
rsServices);
|
htd, rsServices);
|
||||||
return r.openHRegion(reporter);
|
return r.openHRegion(reporter);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static HRegion openHRegion(Path tableDir, final HRegionInfo info,
|
public static HRegion openHRegion(Path tableDir, final HRegionInfo info,
|
||||||
final HLog wal, final Configuration conf)
|
final HTableDescriptor htd, final HLog wal, final Configuration conf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return openHRegion(tableDir, info, wal, conf, null, null);
|
return openHRegion(tableDir, info, htd, wal, conf, null, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -2891,21 +2867,19 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HRegion openHRegion(final Path tableDir, final HRegionInfo info,
|
public static HRegion openHRegion(final Path tableDir, final HRegionInfo info,
|
||||||
final HLog wal, final Configuration conf,
|
final HTableDescriptor htd, final HLog wal, final Configuration conf,
|
||||||
final RegionServerServices rsServices,
|
final RegionServerServices rsServices,
|
||||||
final CancelableProgressable reporter)
|
final CancelableProgressable reporter)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
if (info == null) throw new NullPointerException("Passed region info is null");
|
||||||
LOG.info("HRegion.openHRegion Region name ==" + info.getRegionNameAsString());
|
LOG.info("HRegion.openHRegion Region name ==" + info.getRegionNameAsString());
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Opening region: " + info);
|
LOG.debug("Opening region: " + info);
|
||||||
}
|
}
|
||||||
if (info == null) {
|
|
||||||
throw new NullPointerException("Passed region info is null");
|
|
||||||
}
|
|
||||||
Path dir = HTableDescriptor.getTableDir(tableDir,
|
Path dir = HTableDescriptor.getTableDir(tableDir,
|
||||||
info.getTableName());
|
info.getTableName());
|
||||||
HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
|
HRegion r = HRegion.newHRegion(dir, wal, FileSystem.get(conf), conf, info,
|
||||||
rsServices);
|
htd, rsServices);
|
||||||
return r.openHRegion(reporter);
|
return r.openHRegion(reporter);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3077,7 +3051,8 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
* @return new merged region
|
* @return new merged region
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HRegion merge(HRegion a, HRegion b) throws IOException {
|
public static HRegion merge(HRegion a, HRegion b)
|
||||||
|
throws IOException {
|
||||||
if (!a.getRegionInfo().getTableNameAsString().equals(
|
if (!a.getRegionInfo().getTableNameAsString().equals(
|
||||||
b.getRegionInfo().getTableNameAsString())) {
|
b.getRegionInfo().getTableNameAsString())) {
|
||||||
throw new IOException("Regions do not belong to the same table");
|
throw new IOException("Regions do not belong to the same table");
|
||||||
|
@ -3179,7 +3154,8 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
LOG.debug("Files for new region");
|
LOG.debug("Files for new region");
|
||||||
listPaths(fs, newRegionDir);
|
listPaths(fs, newRegionDir);
|
||||||
}
|
}
|
||||||
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf, newRegionInfo, null);
|
HRegion dstRegion = HRegion.newHRegion(tableDir, log, fs, conf,
|
||||||
|
newRegionInfo, a.getTableDesc(), null);
|
||||||
dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get());
|
dstRegion.readRequestsCount.set(a.readRequestsCount.get() + b.readRequestsCount.get());
|
||||||
dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get());
|
dstRegion.writeRequestsCount.set(a.writeRequestsCount.get() + b.writeRequestsCount.get());
|
||||||
dstRegion.initialize();
|
dstRegion.initialize();
|
||||||
|
@ -3592,7 +3568,7 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
|
|
||||||
public static final long FIXED_OVERHEAD = ClassSize.align(
|
public static final long FIXED_OVERHEAD = ClassSize.align(
|
||||||
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
|
(4 * Bytes.SIZEOF_LONG) + ClassSize.ARRAY +
|
||||||
ClassSize.align(28 * ClassSize.REFERENCE) + ClassSize.OBJECT +
|
ClassSize.align(27 * ClassSize.REFERENCE) + ClassSize.OBJECT +
|
||||||
ClassSize.align(Bytes.SIZEOF_INT));
|
ClassSize.align(Bytes.SIZEOF_INT));
|
||||||
|
|
||||||
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
|
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
|
||||||
|
@ -3745,10 +3721,11 @@ public class HRegion implements HeapSize { // , Writable{
|
||||||
String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
|
String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
|
||||||
// Currently expects tables have one region only.
|
// Currently expects tables have one region only.
|
||||||
if (p.getName().startsWith(rootStr)) {
|
if (p.getName().startsWith(rootStr)) {
|
||||||
region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, null);
|
region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO,
|
||||||
|
HTableDescriptor.ROOT_TABLEDESC, null);
|
||||||
} else if (p.getName().startsWith(metaStr)) {
|
} else if (p.getName().startsWith(metaStr)) {
|
||||||
region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO,
|
region = HRegion.newHRegion(p, log, fs, c,
|
||||||
null);
|
HRegionInfo.FIRST_META_REGIONINFO, HTableDescriptor.META_TABLEDESC, null);
|
||||||
} else {
|
} else {
|
||||||
throw new IOException("Not a known catalog table: " + p.toString());
|
throw new IOException("Not a known catalog table: " + p.toString());
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,12 +63,14 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.HServerAddress;
|
import org.apache.hadoop.hbase.HServerAddress;
|
||||||
import org.apache.hadoop.hbase.HServerInfo;
|
import org.apache.hadoop.hbase.HServerInfo;
|
||||||
import org.apache.hadoop.hbase.HServerLoad;
|
import org.apache.hadoop.hbase.HServerLoad;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.KeyValue;
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.MasterAddressTracker;
|
import org.apache.hadoop.hbase.MasterAddressTracker;
|
||||||
import org.apache.hadoop.hbase.NotServingRegionException;
|
import org.apache.hadoop.hbase.NotServingRegionException;
|
||||||
import org.apache.hadoop.hbase.RemoteExceptionHandler;
|
import org.apache.hadoop.hbase.RemoteExceptionHandler;
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
import org.apache.hadoop.hbase.Stoppable;
|
import org.apache.hadoop.hbase.Stoppable;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
import org.apache.hadoop.hbase.UnknownRowLockException;
|
import org.apache.hadoop.hbase.UnknownRowLockException;
|
||||||
import org.apache.hadoop.hbase.UnknownScannerException;
|
import org.apache.hadoop.hbase.UnknownScannerException;
|
||||||
import org.apache.hadoop.hbase.YouAreDeadException;
|
import org.apache.hadoop.hbase.YouAreDeadException;
|
||||||
|
@ -120,6 +122,7 @@ import org.apache.hadoop.hbase.security.User;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.CompressionTest;
|
import org.apache.hadoop.hbase.util.CompressionTest;
|
||||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||||
|
import org.apache.hadoop.hbase.util.FSTableDescriptors;
|
||||||
import org.apache.hadoop.hbase.util.FSUtils;
|
import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
import org.apache.hadoop.hbase.util.InfoServer;
|
import org.apache.hadoop.hbase.util.InfoServer;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
|
@ -293,6 +296,11 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
|
||||||
*/
|
*/
|
||||||
private final long startcode;
|
private final long startcode;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Go here to get table descriptors.
|
||||||
|
*/
|
||||||
|
private TableDescriptors tableDescriptors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Starts a HRegionServer at the default location
|
* Starts a HRegionServer at the default location
|
||||||
*
|
*
|
||||||
|
@ -863,6 +871,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
|
||||||
// Get fs instance used by this RS
|
// Get fs instance used by this RS
|
||||||
this.fs = FileSystem.get(this.conf);
|
this.fs = FileSystem.get(this.conf);
|
||||||
this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
|
this.rootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
|
||||||
|
this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true);
|
||||||
this.hlog = setupWALAndReplication();
|
this.hlog = setupWALAndReplication();
|
||||||
// Init in here rather than in constructor after thread name has been set
|
// Init in here rather than in constructor after thread name has been set
|
||||||
this.metrics = new RegionServerMetrics();
|
this.metrics = new RegionServerMetrics();
|
||||||
|
@ -2268,12 +2277,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler,
|
||||||
LOG.info("Received request to open region: " +
|
LOG.info("Received request to open region: " +
|
||||||
region.getRegionNameAsString());
|
region.getRegionNameAsString());
|
||||||
if (this.stopped) throw new RegionServerStoppedException();
|
if (this.stopped) throw new RegionServerStoppedException();
|
||||||
|
HTableDescriptor htd = this.tableDescriptors.get(region.getTableName());
|
||||||
if (region.isRootRegion()) {
|
if (region.isRootRegion()) {
|
||||||
this.service.submit(new OpenRootHandler(this, this, region));
|
this.service.submit(new OpenRootHandler(this, this, region, htd));
|
||||||
} else if(region.isMetaRegion()) {
|
} else if(region.isMetaRegion()) {
|
||||||
this.service.submit(new OpenMetaHandler(this, this, region));
|
this.service.submit(new OpenMetaHandler(this, this, region, htd));
|
||||||
} else {
|
} else {
|
||||||
this.service.submit(new OpenRegionHandler(this, this, region));
|
this.service.submit(new OpenRegionHandler(this, this, region, htd));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -567,7 +567,7 @@ public class SplitTransaction {
|
||||||
this.splitdir, hri);
|
this.splitdir, hri);
|
||||||
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
|
HRegion r = HRegion.newHRegion(this.parent.getTableDir(),
|
||||||
this.parent.getLog(), fs, this.parent.getConf(),
|
this.parent.getLog(), fs, this.parent.getConf(),
|
||||||
hri, rsServices);
|
hri, this.parent.getTableDesc(), rsServices);
|
||||||
r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2);
|
r.readRequestsCount.set(this.parent.getReadRequestsCount() / 2);
|
||||||
r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2);
|
r.writeRequestsCount.set(this.parent.getWriteRequestsCount() / 2);
|
||||||
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
|
HRegion.moveInitialFilesIntoPlace(fs, regionDir, r.getRegionDir());
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.apache.hadoop.hbase.regionserver.handler;
|
package org.apache.hadoop.hbase.regionserver.handler;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
||||||
|
|
||||||
|
@ -30,7 +31,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
||||||
*/
|
*/
|
||||||
public class OpenMetaHandler extends OpenRegionHandler {
|
public class OpenMetaHandler extends OpenRegionHandler {
|
||||||
public OpenMetaHandler(final Server server,
|
public OpenMetaHandler(final Server server,
|
||||||
final RegionServerServices rsServices, HRegionInfo regionInfo) {
|
final RegionServerServices rsServices, HRegionInfo regionInfo,
|
||||||
super(server,rsServices, regionInfo, EventType.M_RS_OPEN_META);
|
final HTableDescriptor htd) {
|
||||||
|
super(server,rsServices, regionInfo, htd, EventType.M_RS_OPEN_META);
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.executor.EventHandler;
|
import org.apache.hadoop.hbase.executor.EventHandler;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
|
@ -45,6 +46,7 @@ public class OpenRegionHandler extends EventHandler {
|
||||||
private final RegionServerServices rsServices;
|
private final RegionServerServices rsServices;
|
||||||
|
|
||||||
private final HRegionInfo regionInfo;
|
private final HRegionInfo regionInfo;
|
||||||
|
private final HTableDescriptor htd;
|
||||||
|
|
||||||
// We get version of our znode at start of open process and monitor it across
|
// We get version of our znode at start of open process and monitor it across
|
||||||
// the total open. We'll fail the open if someone hijacks our znode; we can
|
// the total open. We'll fail the open if someone hijacks our znode; we can
|
||||||
|
@ -52,16 +54,18 @@ public class OpenRegionHandler extends EventHandler {
|
||||||
private volatile int version = -1;
|
private volatile int version = -1;
|
||||||
|
|
||||||
public OpenRegionHandler(final Server server,
|
public OpenRegionHandler(final Server server,
|
||||||
final RegionServerServices rsServices, HRegionInfo regionInfo) {
|
final RegionServerServices rsServices, HRegionInfo regionInfo,
|
||||||
this(server, rsServices, regionInfo, EventType.M_RS_OPEN_REGION);
|
HTableDescriptor htd) {
|
||||||
|
this (server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected OpenRegionHandler(final Server server,
|
protected OpenRegionHandler(final Server server,
|
||||||
final RegionServerServices rsServices, final HRegionInfo regionInfo,
|
final RegionServerServices rsServices, final HRegionInfo regionInfo,
|
||||||
EventType eventType) {
|
final HTableDescriptor htd, EventType eventType) {
|
||||||
super(server, eventType);
|
super(server, eventType);
|
||||||
this.rsServices = rsServices;
|
this.rsServices = rsServices;
|
||||||
this.regionInfo = regionInfo;
|
this.regionInfo = regionInfo;
|
||||||
|
this.htd = htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
public HRegionInfo getRegionInfo() {
|
public HRegionInfo getRegionInfo() {
|
||||||
|
@ -184,7 +188,7 @@ public class OpenRegionHandler extends EventHandler {
|
||||||
|
|
||||||
// Was there an exception opening the region? This should trigger on
|
// Was there an exception opening the region? This should trigger on
|
||||||
// InterruptedException too. If so, we failed.
|
// InterruptedException too. If so, we failed.
|
||||||
return !t.interrupted() && t.getException() == null;
|
return !Thread.interrupted() && t.getException() == null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -269,8 +273,9 @@ public class OpenRegionHandler extends EventHandler {
|
||||||
try {
|
try {
|
||||||
// Instantiate the region. This also periodically tickles our zk OPENING
|
// Instantiate the region. This also periodically tickles our zk OPENING
|
||||||
// state so master doesn't timeout this region in transition.
|
// state so master doesn't timeout this region in transition.
|
||||||
region = HRegion.openHRegion(tableDir, this.regionInfo, this.rsServices.getWAL(),
|
region = HRegion.openHRegion(tableDir, this.regionInfo, this.htd,
|
||||||
this.server.getConfiguration(), this.rsServices,
|
this.rsServices.getWAL(), this.server.getConfiguration(),
|
||||||
|
this.rsServices,
|
||||||
new CancelableProgressable() {
|
new CancelableProgressable() {
|
||||||
public boolean progress() {
|
public boolean progress() {
|
||||||
// We may lose the znode ownership during the open. Currently its
|
// We may lose the znode ownership during the open. Currently its
|
||||||
|
@ -296,8 +301,9 @@ public class OpenRegionHandler extends EventHandler {
|
||||||
try {
|
try {
|
||||||
// Instantiate the region. This also periodically tickles our zk OPENING
|
// Instantiate the region. This also periodically tickles our zk OPENING
|
||||||
// state so master doesn't timeout this region in transition.
|
// state so master doesn't timeout this region in transition.
|
||||||
region = HRegion.openHRegion(this.regionInfo, this.rsServices.getWAL(),
|
region = HRegion.openHRegion(this.regionInfo, this.htd,
|
||||||
this.server.getConfiguration(), this.rsServices,
|
this.rsServices.getWAL(), this.server.getConfiguration(),
|
||||||
|
this.rsServices,
|
||||||
new CancelableProgressable() {
|
new CancelableProgressable() {
|
||||||
public boolean progress() {
|
public boolean progress() {
|
||||||
// We may lose the znode ownership during the open. Currently its
|
// We may lose the znode ownership during the open. Currently its
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
package org.apache.hadoop.hbase.regionserver.handler;
|
package org.apache.hadoop.hbase.regionserver.handler;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
||||||
|
|
||||||
|
@ -30,7 +31,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices;
|
||||||
*/
|
*/
|
||||||
public class OpenRootHandler extends OpenRegionHandler {
|
public class OpenRootHandler extends OpenRegionHandler {
|
||||||
public OpenRootHandler(final Server server,
|
public OpenRootHandler(final Server server,
|
||||||
final RegionServerServices rsServices, HRegionInfo regionInfo) {
|
final RegionServerServices rsServices, HRegionInfo regionInfo,
|
||||||
super(server, rsServices, regionInfo, EventType.M_RS_OPEN_ROOT);
|
final HTableDescriptor htd) {
|
||||||
|
super(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_ROOT);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -173,7 +173,10 @@ public class SequenceFileLogWriter implements HLog.Writer {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() throws IOException {
|
public void close() throws IOException {
|
||||||
|
if (this.writer != null) {
|
||||||
this.writer.close();
|
this.writer.close();
|
||||||
|
this.writer = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -0,0 +1,184 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.util;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
|
import org.apache.hadoop.hbase.TableExistsException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implementation of {@link TableDescriptors} that reads descriptors from the
|
||||||
|
* passed filesystem. It expects descriptors to be in a file under the
|
||||||
|
* table's directory in FS. Can be read-only -- i.e. does not modify
|
||||||
|
* the filesystem or can be read and write.
|
||||||
|
*/
|
||||||
|
public class FSTableDescriptors implements TableDescriptors {
|
||||||
|
private final FileSystem fs;
|
||||||
|
private final Path rootdir;
|
||||||
|
private final boolean fsreadonly;
|
||||||
|
long cachehits = 0;
|
||||||
|
long invocations = 0;
|
||||||
|
|
||||||
|
// This cache does not age out the old stuff. Thinking is that the amount
|
||||||
|
// of data we keep up in here is so small, no need to do occasional purge.
|
||||||
|
// TODO.
|
||||||
|
private final Map<String, TableDescriptorModtime> cache =
|
||||||
|
new ConcurrentHashMap<String, TableDescriptorModtime>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Data structure to hold modification time and table descriptor.
|
||||||
|
*/
|
||||||
|
static class TableDescriptorModtime {
|
||||||
|
private final HTableDescriptor descriptor;
|
||||||
|
private final long modtime;
|
||||||
|
|
||||||
|
TableDescriptorModtime(final long modtime, final HTableDescriptor htd) {
|
||||||
|
this.descriptor = htd;
|
||||||
|
this.modtime = modtime;
|
||||||
|
}
|
||||||
|
|
||||||
|
long getModtime() {
|
||||||
|
return this.modtime;
|
||||||
|
}
|
||||||
|
|
||||||
|
HTableDescriptor getTableDescriptor() {
|
||||||
|
return this.descriptor;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public FSTableDescriptors(final FileSystem fs, final Path rootdir) {
|
||||||
|
this(fs, rootdir, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param fs
|
||||||
|
* @param rootdir
|
||||||
|
* @param fsreadOnly True if we are read-only when it comes to filesystem
|
||||||
|
* operations; i.e. on remove, we do not do delete in fs.
|
||||||
|
*/
|
||||||
|
public FSTableDescriptors(final FileSystem fs, final Path rootdir,
|
||||||
|
final boolean fsreadOnly) {
|
||||||
|
super();
|
||||||
|
this.fs = fs;
|
||||||
|
this.rootdir = rootdir;
|
||||||
|
this.fsreadonly = fsreadOnly;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see org.apache.hadoop.hbase.TableDescriptors#getHTableDescriptor(java.lang.String)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor get(final byte [] tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
|
return get(Bytes.toString(tablename));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptor(byte[])
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor get(final String tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
|
invocations++;
|
||||||
|
if (HTableDescriptor.ROOT_TABLEDESC.getNameAsString().equals(tablename)) {
|
||||||
|
cachehits++;
|
||||||
|
return HTableDescriptor.ROOT_TABLEDESC;
|
||||||
|
}
|
||||||
|
if (HTableDescriptor.META_TABLEDESC.getNameAsString().equals(tablename)) {
|
||||||
|
cachehits++;
|
||||||
|
return HTableDescriptor.META_TABLEDESC;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look in cache of descriptors.
|
||||||
|
TableDescriptorModtime tdm = this.cache.get(tablename);
|
||||||
|
|
||||||
|
// Check mod time has not changed (this is trip to NN).
|
||||||
|
long modtime =
|
||||||
|
FSUtils.getTableInfoModtime(this.fs, this.rootdir, tablename);
|
||||||
|
if (tdm != null) {
|
||||||
|
if (modtime <= tdm.getModtime()) {
|
||||||
|
cachehits++;
|
||||||
|
return tdm.getTableDescriptor();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
HTableDescriptor htd =
|
||||||
|
FSUtils.getTableDescriptor(this.fs, this.rootdir, tablename);
|
||||||
|
if (htd == null) {
|
||||||
|
// More likely is above will throw a FileNotFoundException
|
||||||
|
throw new TableExistsException("No descriptor for " + tablename);
|
||||||
|
}
|
||||||
|
this.cache.put(tablename, new TableDescriptorModtime(modtime, htd));
|
||||||
|
return htd;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* (non-Javadoc)
|
||||||
|
* @see org.apache.hadoop.hbase.TableDescriptors#getTableDescriptors(org.apache.hadoop.fs.FileSystem, org.apache.hadoop.fs.Path)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public Map<String, HTableDescriptor> getAll()
|
||||||
|
throws IOException {
|
||||||
|
Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
|
||||||
|
List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
|
||||||
|
for (Path d: tableDirs) {
|
||||||
|
HTableDescriptor htd = get(d.getName());
|
||||||
|
if (htd == null) continue;
|
||||||
|
htds.put(d.getName(), htd);
|
||||||
|
}
|
||||||
|
return htds;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void add(HTableDescriptor htd) throws IOException {
|
||||||
|
if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
if (!this.fsreadonly) FSUtils.updateHTableDescriptor(this.fs, this.rootdir, htd);
|
||||||
|
long modtime =
|
||||||
|
FSUtils.getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
|
||||||
|
this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor remove(final String tablename)
|
||||||
|
throws IOException {
|
||||||
|
if (!this.fsreadonly) {
|
||||||
|
Path tabledir = FSUtils.getTablePath(this.rootdir, tablename);
|
||||||
|
if (this.fs.exists(tabledir)) {
|
||||||
|
if (!this.fs.delete(tabledir, true)) {
|
||||||
|
throw new IOException("Failed delete of " + tabledir.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TableDescriptorModtime tdm = this.cache.remove(tablename);
|
||||||
|
return tdm == null? null: tdm.getTableDescriptor();
|
||||||
|
}
|
||||||
|
}
|
|
@ -49,7 +49,9 @@ import java.io.InterruptedIOException;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -835,68 +837,68 @@ public class FSUtils {
|
||||||
LOG.info("Finished lease recover attempt for " + p);
|
LOG.info("Finished lease recover attempt for " + p);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
public static Map<String, HTableDescriptor> getTableDescriptors(
|
* @param fs
|
||||||
final Configuration config)
|
* @param rootdir
|
||||||
|
* @return All the table directories under <code>rootdir</code>
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static List<Path> getTableDirs(final FileSystem fs, final Path rootdir)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path path = getRootDir(config);
|
|
||||||
// since HMaster.getFileSystem() is package private
|
|
||||||
FileSystem fs = path.getFileSystem(config);
|
|
||||||
return getTableDescriptors(fs, path);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Map<String, HTableDescriptor> getTableDescriptors(
|
|
||||||
final FileSystem fs, final Path hbaseRootDir)
|
|
||||||
throws IOException {
|
|
||||||
Map<String, HTableDescriptor> desc =
|
|
||||||
new HashMap<String, HTableDescriptor>();
|
|
||||||
DirFilter df = new DirFilter(fs);
|
|
||||||
// presumes any directory under hbase.rootdir is a table
|
// presumes any directory under hbase.rootdir is a table
|
||||||
FileStatus [] tableDirs = fs.listStatus(hbaseRootDir, df);
|
FileStatus [] dirs = fs.listStatus(rootdir, new DirFilter(fs));
|
||||||
for (FileStatus tableDir : tableDirs) {
|
List<Path> tabledirs = new ArrayList<Path>(dirs.length);
|
||||||
Path d = tableDir.getPath();
|
for (FileStatus dir: dirs) {
|
||||||
String tableName = d.getName();
|
Path p = dir.getPath();
|
||||||
|
String tableName = p.getName();
|
||||||
if (tableName.equals(HConstants.HREGION_LOGDIR_NAME)
|
if (tableName.equals(HConstants.HREGION_LOGDIR_NAME) ||
|
||||||
|| tableName.equals(Bytes.toString(HConstants.ROOT_TABLE_NAME))
|
tableName.equals(Bytes.toString(HConstants.ROOT_TABLE_NAME)) ||
|
||||||
|| tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME))
|
tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME)) ||
|
||||||
|| tableName.equals(HConstants.HREGION_OLDLOGDIR_NAME)
|
tableName.equals(HConstants.HREGION_OLDLOGDIR_NAME) ) {
|
||||||
) {
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
LOG.info("Adding tabledescriptor for table = " + tableName);
|
tabledirs.add(p);
|
||||||
HTableDescriptor htd = readTableDescriptor(fs, hbaseRootDir,
|
|
||||||
tableName);
|
|
||||||
if (htd != null) {
|
|
||||||
if (!desc.containsKey(tableName)) {
|
|
||||||
desc.put(tableName, htd);
|
|
||||||
}
|
}
|
||||||
}
|
return tabledirs;
|
||||||
}
|
|
||||||
return desc;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Path getTableInfoPath(Path hbaseRootDir, String tableName) {
|
|
||||||
Path tablePath = new Path(hbaseRootDir, tableName);
|
|
||||||
return new Path(tablePath, HConstants.TABLEINFO_NAME);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get table info path for a table.
|
* Get table info path for a table.
|
||||||
|
* @param rootdir
|
||||||
* @param tableName
|
* @param tableName
|
||||||
* @return Table info path
|
* @return Table info path
|
||||||
*/
|
*/
|
||||||
private static Path getTableInfoPath(byte[] tableName, Configuration conf) throws IOException {
|
private static Path getTableInfoPath(Path rootdir, String tablename) {
|
||||||
Path tablePath = new Path(getRootDir(conf), Bytes.toString(tableName));
|
Path tablePath = getTablePath(rootdir, tablename);
|
||||||
Path tableInfoPath = new Path(tablePath, HConstants.TABLEINFO_NAME);
|
return new Path(tablePath, HConstants.TABLEINFO_NAME);
|
||||||
return tableInfoPath;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path getTablePath(byte[] tableName, Configuration conf) throws IOException {
|
/**
|
||||||
return new Path(getRootDir(conf), Bytes.toString(tableName));
|
* @param fs
|
||||||
|
* @param rootdir
|
||||||
|
* @param tablename
|
||||||
|
* @return Modification time for the table {@link HConstants#TABLEINFO_NAME} file.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static long getTableInfoModtime(final FileSystem fs, final Path rootdir,
|
||||||
|
final String tablename)
|
||||||
|
throws IOException {
|
||||||
|
Path p = getTablePath(rootdir, tablename);
|
||||||
|
FileStatus [] status = fs.listStatus(p);
|
||||||
|
if (status.length < 1) throw new FileNotFoundException("No status for " + p.toString());
|
||||||
|
return status[0].getModificationTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
private static FileSystem getCurrentFileSystem(Configuration conf) throws IOException {
|
public static Path getTablePath(Path rootdir, byte [] tableName) {
|
||||||
|
return getTablePath(rootdir, Bytes.toString(tableName));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Path getTablePath(Path rootdir, final String tableName) {
|
||||||
|
return new Path(rootdir, tableName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static FileSystem getCurrentFileSystem(Configuration conf)
|
||||||
|
throws IOException {
|
||||||
return getRootDir(conf).getFileSystem(conf);
|
return getRootDir(conf).getFileSystem(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -912,23 +914,7 @@ public class FSUtils {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path path = getRootDir(config);
|
Path path = getRootDir(config);
|
||||||
FileSystem fs = path.getFileSystem(config);
|
FileSystem fs = path.getFileSystem(config);
|
||||||
return readTableDescriptor(fs, path, tableName);
|
return getTableDescriptor(fs, path, tableName);
|
||||||
}
|
|
||||||
|
|
||||||
private static HTableDescriptor readTableDescriptor(FileSystem fs,
|
|
||||||
Path hbaseRootDir,
|
|
||||||
String tableName) {
|
|
||||||
try {
|
|
||||||
FSDataInputStream fsDataInputStream =
|
|
||||||
fs.open(getTableInfoPath(hbaseRootDir, tableName));
|
|
||||||
HTableDescriptor hTableDescriptor = new HTableDescriptor();
|
|
||||||
hTableDescriptor.readFields(fsDataInputStream);
|
|
||||||
fsDataInputStream.close();
|
|
||||||
return hTableDescriptor;
|
|
||||||
} catch (IOException ioe) {
|
|
||||||
LOG.info("Exception during readTableDecriptor. Current table name = " + tableName , ioe);
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -936,92 +922,99 @@ public class FSUtils {
|
||||||
* @param fs
|
* @param fs
|
||||||
* @param hbaseRootDir
|
* @param hbaseRootDir
|
||||||
* @param tableName
|
* @param tableName
|
||||||
* @return
|
* @return Descriptor or null if none found.
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static HTableDescriptor getTableDescriptor(FileSystem fs,
|
public static HTableDescriptor getTableDescriptor(FileSystem fs,
|
||||||
Path hbaseRootDir,
|
Path hbaseRootDir, byte[] tableName)
|
||||||
byte[] tableName)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
return readTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
|
return getTableDescriptor(fs, hbaseRootDir, Bytes.toString(tableName));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static HTableDescriptor getTableDescriptor(FileSystem fs,
|
||||||
public static HTableDescriptor getTableDescriptor(Path tableDir, FileSystem fs) {
|
Path hbaseRootDir, String tableName) {
|
||||||
|
HTableDescriptor htd = null;
|
||||||
try {
|
try {
|
||||||
LOG.info("Reading table descriptor from .tableinfo. current path = "
|
htd = getTableDescriptor(fs, getTablePath(hbaseRootDir, tableName));
|
||||||
+ tableDir);
|
} catch (IOException ioe) {
|
||||||
if (tableDir == null) {
|
LOG.debug("Exception during readTableDecriptor. Current table name = " +
|
||||||
LOG.info("Reading table descriptor from .tableinfo current tablename is NULL ");
|
tableName , ioe);
|
||||||
return null;
|
}
|
||||||
|
return htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static HTableDescriptor getTableDescriptor(FileSystem fs, Path tableDir)
|
||||||
|
throws IOException {
|
||||||
|
if (tableDir == null) throw new NullPointerException();
|
||||||
FSDataInputStream fsDataInputStream =
|
FSDataInputStream fsDataInputStream =
|
||||||
fs.open(new Path(tableDir, HConstants.TABLEINFO_NAME));
|
fs.open(new Path(tableDir, HConstants.TABLEINFO_NAME));
|
||||||
HTableDescriptor hTableDescriptor = new HTableDescriptor();
|
HTableDescriptor hTableDescriptor = null;
|
||||||
|
try {
|
||||||
|
hTableDescriptor = new HTableDescriptor();
|
||||||
hTableDescriptor.readFields(fsDataInputStream);
|
hTableDescriptor.readFields(fsDataInputStream);
|
||||||
LOG.info("Current tabledescriptor from .tableinfo is " + hTableDescriptor.toString());
|
} finally {
|
||||||
fsDataInputStream.close();
|
fsDataInputStream.close();
|
||||||
return hTableDescriptor;
|
|
||||||
} catch (IOException ioe) {
|
|
||||||
LOG.info("Exception during getTableDescriptor ", ioe);
|
|
||||||
}
|
}
|
||||||
return null;
|
return hTableDescriptor;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create new HTableDescriptor in HDFS.
|
* Create new HTableDescriptor in HDFS. Happens when we are creating table.
|
||||||
|
/**
|
||||||
* @param htableDescriptor
|
* @param htableDescriptor
|
||||||
|
* @param conf
|
||||||
*/
|
*/
|
||||||
public static void createTableDescriptor(HTableDescriptor htableDescriptor,
|
public static void createTableDescriptor(HTableDescriptor htableDescriptor,
|
||||||
Configuration conf) {
|
Configuration conf) {
|
||||||
try {
|
try {
|
||||||
Path tableDir = getTablePath(htableDescriptor.getName(), conf);
|
|
||||||
FileSystem fs = getCurrentFileSystem(conf);
|
FileSystem fs = getCurrentFileSystem(conf);
|
||||||
createTableDescriptor(fs, htableDescriptor, tableDir);
|
createTableDescriptor(fs, getRootDir(conf), htableDescriptor);
|
||||||
} catch(IOException ioe) {
|
} catch(IOException ioe) {
|
||||||
LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
|
LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param fs
|
||||||
|
* @param htableDescriptor
|
||||||
|
* @param rootdir
|
||||||
|
*/
|
||||||
public static void createTableDescriptor(FileSystem fs,
|
public static void createTableDescriptor(FileSystem fs,
|
||||||
HTableDescriptor htableDescriptor,
|
Path rootdir, HTableDescriptor htableDescriptor) {
|
||||||
Path tableDir) {
|
|
||||||
try {
|
try {
|
||||||
Path tableInfoPath = new Path(tableDir, HConstants.TABLEINFO_NAME);
|
Path tableInfoPath =
|
||||||
LOG.info("Current tableInfoPath = " + tableInfoPath
|
getTableInfoPath(rootdir, htableDescriptor.getNameAsString());
|
||||||
+ " tableDir = " + tableDir) ;
|
LOG.info("Current tableInfoPath = " + tableInfoPath) ;
|
||||||
if (fs.exists(tableInfoPath) &&
|
if (fs.exists(tableInfoPath) &&
|
||||||
fs.getFileStatus(tableInfoPath).getLen() > 0) {
|
fs.getFileStatus(tableInfoPath).getLen() > 0) {
|
||||||
LOG.info("TableInfo already exists.. Skipping creation");
|
LOG.info("TableInfo already exists.. Skipping creation");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
writeTableDescriptor(fs, htableDescriptor, tableDir);
|
writeTableDescriptor(fs, htableDescriptor,
|
||||||
|
getTablePath(rootdir, htableDescriptor.getNameAsString()));
|
||||||
} catch(IOException ioe) {
|
} catch(IOException ioe) {
|
||||||
LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
|
LOG.info("IOException while trying to create tableInfo in HDFS", ioe);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when we are creating a table to write out the tables' descriptor.
|
||||||
|
* @param fs
|
||||||
|
* @param hTableDescriptor
|
||||||
|
* @param tableDir
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
private static void writeTableDescriptor(FileSystem fs,
|
private static void writeTableDescriptor(FileSystem fs,
|
||||||
HTableDescriptor hTableDescriptor,
|
HTableDescriptor hTableDescriptor, Path tableDir)
|
||||||
Path tableDir) throws IOException {
|
throws IOException {
|
||||||
// Create in tmpdir and then move into place in case we crash after
|
// Create in tmpdir and then move into place in case we crash after
|
||||||
// create but before close. If we don't successfully close the file,
|
// create but before close. If we don't successfully close the file,
|
||||||
// subsequent region reopens will fail the below because create is
|
// subsequent region reopens will fail the below because create is
|
||||||
// registered in NN.
|
// registered in NN.
|
||||||
Path tableInfoPath = new Path(tableDir, HConstants.TABLEINFO_NAME);
|
Path tableInfoPath = new Path(tableDir, HConstants.TABLEINFO_NAME);
|
||||||
Path tmpPath = new Path(new Path(tableDir,".tmp"),
|
Path tmpPath = new Path(new Path(tableDir,".tmp"), HConstants.TABLEINFO_NAME);
|
||||||
HConstants.TABLEINFO_NAME);
|
|
||||||
LOG.info("TableInfoPath = " + tableInfoPath + " tmpPath = " + tmpPath);
|
LOG.info("TableInfoPath = " + tableInfoPath + " tmpPath = " + tmpPath);
|
||||||
FSDataOutputStream out = fs.create(tmpPath, true);
|
writeHTD(fs, tmpPath, hTableDescriptor);
|
||||||
try {
|
|
||||||
hTableDescriptor.write(out);
|
|
||||||
out.write('\n');
|
|
||||||
out.write('\n');
|
|
||||||
out.write(Bytes.toBytes(hTableDescriptor.toString()));
|
|
||||||
} finally {
|
|
||||||
out.close();
|
|
||||||
}
|
|
||||||
if (!fs.rename(tmpPath, tableInfoPath)) {
|
if (!fs.rename(tmpPath, tableInfoPath)) {
|
||||||
throw new IOException("Unable to rename " + tmpPath + " to " +
|
throw new IOException("Unable to rename " + tmpPath + " to " +
|
||||||
tableInfoPath);
|
tableInfoPath);
|
||||||
|
@ -1030,29 +1023,34 @@ public class FSUtils {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update table descriptor
|
||||||
|
* @param fs
|
||||||
|
* @param conf
|
||||||
|
* @param hTableDescriptor
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static void updateHTableDescriptor(FileSystem fs, Path rootdir,
|
||||||
|
HTableDescriptor hTableDescriptor)
|
||||||
|
throws IOException {
|
||||||
|
Path tableInfoPath =
|
||||||
|
getTableInfoPath(rootdir, hTableDescriptor.getNameAsString());
|
||||||
|
writeHTD(fs, tableInfoPath, hTableDescriptor);
|
||||||
|
LOG.info("updateHTableDescriptor. Updated tableinfo in HDFS under " +
|
||||||
|
tableInfoPath + " For HTD => " + hTableDescriptor.toString());
|
||||||
|
}
|
||||||
|
|
||||||
public static void updateHTableDescriptor(FileSystem fs,
|
private static void writeHTD(final FileSystem fs, final Path p,
|
||||||
Configuration conf,
|
final HTableDescriptor htd)
|
||||||
HTableDescriptor hTableDescriptor) throws IOException
|
throws IOException {
|
||||||
{
|
FSDataOutputStream out = fs.create(p, true);
|
||||||
Path tableInfoPath = getTableInfoPath(hTableDescriptor.getName(), conf);
|
|
||||||
FSDataOutputStream out = fs.create(tableInfoPath, true);
|
|
||||||
try {
|
try {
|
||||||
hTableDescriptor.write(out);
|
htd.write(out);
|
||||||
out.write('\n');
|
out.write('\n');
|
||||||
out.write('\n');
|
out.write('\n');
|
||||||
out.write(Bytes.toBytes(hTableDescriptor.toString()));
|
out.write(Bytes.toBytes(htd.toString()));
|
||||||
LOG.info("updateHTableDescriptor. Updated tableinfo in HDFS under "
|
|
||||||
+ tableInfoPath + " For HTD => "
|
|
||||||
+ hTableDescriptor.toString());
|
|
||||||
} finally {
|
} finally {
|
||||||
out.close();
|
out.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path getTmpDir(HTableDescriptor htableDescriptor, Configuration configuration)
|
|
||||||
throws IOException {
|
|
||||||
return new Path(getTablePath(htableDescriptor.getName(), configuration), ".tmp");
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
|
@ -55,6 +55,7 @@ import java.util.Random;
|
||||||
* a table by merging adjacent regions.
|
* a table by merging adjacent regions.
|
||||||
*/
|
*/
|
||||||
class HMerge {
|
class HMerge {
|
||||||
|
// TODO: Where is this class used? How does it relate to Merge in same package?
|
||||||
static final Log LOG = LogFactory.getLog(HMerge.class);
|
static final Log LOG = LogFactory.getLog(HMerge.class);
|
||||||
static final Random rand = new Random();
|
static final Random rand = new Random();
|
||||||
|
|
||||||
|
@ -135,12 +136,12 @@ class HMerge {
|
||||||
protected final Configuration conf;
|
protected final Configuration conf;
|
||||||
protected final FileSystem fs;
|
protected final FileSystem fs;
|
||||||
protected final Path tabledir;
|
protected final Path tabledir;
|
||||||
|
protected final HTableDescriptor htd;
|
||||||
protected final HLog hlog;
|
protected final HLog hlog;
|
||||||
private final long maxFilesize;
|
private final long maxFilesize;
|
||||||
|
|
||||||
|
|
||||||
protected Merger(Configuration conf, FileSystem fs,
|
protected Merger(Configuration conf, FileSystem fs, final byte [] tableName)
|
||||||
final byte [] tableName)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.fs = fs;
|
this.fs = fs;
|
||||||
|
@ -151,6 +152,7 @@ class HMerge {
|
||||||
fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
|
fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
|
||||||
Bytes.toString(tableName)
|
Bytes.toString(tableName)
|
||||||
);
|
);
|
||||||
|
this.htd = FSUtils.getTableDescriptor(this.fs, this.tabledir);
|
||||||
Path logdir = new Path(tabledir, "merge_" + System.currentTimeMillis() +
|
Path logdir = new Path(tabledir, "merge_" + System.currentTimeMillis() +
|
||||||
HConstants.HREGION_LOGDIR_NAME);
|
HConstants.HREGION_LOGDIR_NAME);
|
||||||
Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME);
|
Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME);
|
||||||
|
@ -188,13 +190,13 @@ class HMerge {
|
||||||
long nextSize = 0;
|
long nextSize = 0;
|
||||||
for (int i = 0; i < info.length - 1; i++) {
|
for (int i = 0; i < info.length - 1; i++) {
|
||||||
if (currentRegion == null) {
|
if (currentRegion == null) {
|
||||||
currentRegion =
|
currentRegion = HRegion.newHRegion(tabledir, hlog, fs, conf, info[i],
|
||||||
HRegion.newHRegion(tabledir, hlog, fs, conf, info[i], null);
|
this.htd, null);
|
||||||
currentRegion.initialize();
|
currentRegion.initialize();
|
||||||
currentSize = currentRegion.getLargestHStoreSize();
|
currentSize = currentRegion.getLargestHStoreSize();
|
||||||
}
|
}
|
||||||
nextRegion =
|
nextRegion = HRegion.newHRegion(tabledir, hlog, fs, conf, info[i + 1],
|
||||||
HRegion.newHRegion(tabledir, hlog, fs, conf, info[i + 1], null);
|
this.htd, null);
|
||||||
nextRegion.initialize();
|
nextRegion.initialize();
|
||||||
nextSize = nextRegion.getLargestHStoreSize();
|
nextSize = nextRegion.getLargestHStoreSize();
|
||||||
|
|
||||||
|
@ -357,7 +359,7 @@ class HMerge {
|
||||||
// Scan root region to find all the meta regions
|
// Scan root region to find all the meta regions
|
||||||
|
|
||||||
root = HRegion.newHRegion(rootTableDir, hlog, fs, conf,
|
root = HRegion.newHRegion(rootTableDir, hlog, fs, conf,
|
||||||
HRegionInfo.ROOT_REGIONINFO, null);
|
HRegionInfo.ROOT_REGIONINFO, HTableDescriptor.ROOT_TABLEDESC, null);
|
||||||
root.initialize();
|
root.initialize();
|
||||||
|
|
||||||
Scan scan = new Scan();
|
Scan scan = new Scan();
|
||||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
import org.apache.hadoop.hbase.KeyValue;
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.MasterNotRunningException;
|
import org.apache.hadoop.hbase.MasterNotRunningException;
|
||||||
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
||||||
|
@ -153,7 +154,7 @@ public class Merge extends Configured implements Tool {
|
||||||
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
||||||
List<KeyValue> cells2 = rootRegion.get(get, null).list();
|
List<KeyValue> cells2 = rootRegion.get(get, null).list();
|
||||||
HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
|
HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
|
||||||
HRegion merged = merge(info1, rootRegion, info2, rootRegion);
|
HRegion merged = merge(HTableDescriptor.META_TABLEDESC, info1, rootRegion, info2, rootRegion);
|
||||||
LOG.info("Adding " + merged.getRegionInfo() + " to " +
|
LOG.info("Adding " + merged.getRegionInfo() + " to " +
|
||||||
rootRegion.getRegionInfo());
|
rootRegion.getRegionInfo());
|
||||||
HRegion.addRegionToMETA(rootRegion, merged);
|
HRegion.addRegionToMETA(rootRegion, merged);
|
||||||
|
@ -216,7 +217,8 @@ public class Merge extends Configured implements Tool {
|
||||||
Get get = new Get(region1);
|
Get get = new Get(region1);
|
||||||
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
|
||||||
List<KeyValue> cells1 = metaRegion1.get(get, null).list();
|
List<KeyValue> cells1 = metaRegion1.get(get, null).list();
|
||||||
HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
|
HRegionInfo info1 =
|
||||||
|
Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
|
||||||
if (info1 == null) {
|
if (info1 == null) {
|
||||||
throw new NullPointerException("info1 is null using key " +
|
throw new NullPointerException("info1 is null using key " +
|
||||||
Bytes.toStringBinary(region1) + " in " + meta1);
|
Bytes.toStringBinary(region1) + " in " + meta1);
|
||||||
|
@ -235,7 +237,9 @@ public class Merge extends Configured implements Tool {
|
||||||
if (info2 == null) {
|
if (info2 == null) {
|
||||||
throw new NullPointerException("info2 is null using key " + meta2);
|
throw new NullPointerException("info2 is null using key " + meta2);
|
||||||
}
|
}
|
||||||
HRegion merged = merge(info1, metaRegion1, info2, metaRegion2);
|
HTableDescriptor htd = FSUtils.getTableDescriptor(FileSystem.get(getConf()),
|
||||||
|
this.rootdir, this.tableName);
|
||||||
|
HRegion merged = merge(htd, info1, metaRegion1, info2, metaRegion2);
|
||||||
|
|
||||||
// Now find the meta region which will contain the newly merged region
|
// Now find the meta region which will contain the newly merged region
|
||||||
|
|
||||||
|
@ -267,8 +271,8 @@ public class Merge extends Configured implements Tool {
|
||||||
* to scan the meta if the resulting merged region does not go in either)
|
* to scan the meta if the resulting merged region does not go in either)
|
||||||
* Returns HRegion object for newly merged region
|
* Returns HRegion object for newly merged region
|
||||||
*/
|
*/
|
||||||
private HRegion merge(HRegionInfo info1, HRegion meta1, HRegionInfo info2,
|
private HRegion merge(final HTableDescriptor htd, HRegionInfo info1,
|
||||||
HRegion meta2)
|
HRegion meta1, HRegionInfo info2, HRegion meta2)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (info1 == null) {
|
if (info1 == null) {
|
||||||
throw new IOException("Could not find " + Bytes.toStringBinary(region1) + " in " +
|
throw new IOException("Could not find " + Bytes.toStringBinary(region1) + " in " +
|
||||||
|
@ -280,9 +284,9 @@ public class Merge extends Configured implements Tool {
|
||||||
}
|
}
|
||||||
HRegion merged = null;
|
HRegion merged = null;
|
||||||
HLog log = utils.getLog();
|
HLog log = utils.getLog();
|
||||||
HRegion r1 = HRegion.openHRegion(info1, log, getConf());
|
HRegion r1 = HRegion.openHRegion(info1, htd, log, getConf());
|
||||||
try {
|
try {
|
||||||
HRegion r2 = HRegion.openHRegion(info2, log, getConf());
|
HRegion r2 = HRegion.openHRegion(info2, htd, log, getConf());
|
||||||
try {
|
try {
|
||||||
merged = HRegion.merge(r1, r2);
|
merged = HRegion.merge(r1, r2);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
@ -20,14 +20,23 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.util;
|
package org.apache.hadoop.hbase.util;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.client.Delete;
|
import org.apache.hadoop.hbase.client.Delete;
|
||||||
import org.apache.hadoop.hbase.client.Get;
|
import org.apache.hadoop.hbase.client.Get;
|
||||||
import org.apache.hadoop.hbase.client.HTable;
|
import org.apache.hadoop.hbase.client.HTable;
|
||||||
|
@ -36,17 +45,7 @@ import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
import org.apache.hadoop.hbase.regionserver.InternalScanner;
|
import org.apache.hadoop.hbase.regionserver.InternalScanner;
|
||||||
import org.apache.hadoop.hbase.regionserver.Store;
|
|
||||||
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
|
||||||
import org.apache.hadoop.hbase.KeyValue;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.TreeMap;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Contains utility methods for manipulating HBase meta tables.
|
* Contains utility methods for manipulating HBase meta tables.
|
||||||
|
@ -59,7 +58,6 @@ public class MetaUtils {
|
||||||
private static final Log LOG = LogFactory.getLog(MetaUtils.class);
|
private static final Log LOG = LogFactory.getLog(MetaUtils.class);
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private FileSystem fs;
|
private FileSystem fs;
|
||||||
private Path rootdir;
|
|
||||||
private HLog log;
|
private HLog log;
|
||||||
private HRegion rootRegion;
|
private HRegion rootRegion;
|
||||||
private Map<byte [], HRegion> metaRegions = Collections.synchronizedSortedMap(
|
private Map<byte [], HRegion> metaRegions = Collections.synchronizedSortedMap(
|
||||||
|
@ -89,8 +87,6 @@ public class MetaUtils {
|
||||||
*/
|
*/
|
||||||
private void initialize() throws IOException {
|
private void initialize() throws IOException {
|
||||||
this.fs = FileSystem.get(this.conf);
|
this.fs = FileSystem.get(this.conf);
|
||||||
// Get root directory of HBase installation
|
|
||||||
this.rootdir = FSUtils.getRootDir(this.conf);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -266,14 +262,16 @@ public class MetaUtils {
|
||||||
if (this.rootRegion != null) {
|
if (this.rootRegion != null) {
|
||||||
return this.rootRegion;
|
return this.rootRegion;
|
||||||
}
|
}
|
||||||
this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO, getLog(),
|
this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO,
|
||||||
|
HTableDescriptor.ROOT_TABLEDESC, getLog(),
|
||||||
this.conf);
|
this.conf);
|
||||||
this.rootRegion.compactStores();
|
this.rootRegion.compactStores();
|
||||||
return this.rootRegion;
|
return this.rootRegion;
|
||||||
}
|
}
|
||||||
|
|
||||||
private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException {
|
private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException {
|
||||||
HRegion meta = HRegion.openHRegion(metaInfo, getLog(), this.conf);
|
HRegion meta = HRegion.openHRegion(metaInfo, HTableDescriptor.META_TABLEDESC,
|
||||||
|
getLog(), this.conf);
|
||||||
meta.compactStores();
|
meta.compactStores();
|
||||||
return meta;
|
return meta;
|
||||||
}
|
}
|
||||||
|
|
|
@ -173,7 +173,7 @@ public abstract class HBaseTestCase extends TestCase {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
|
HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
|
||||||
closedRegion.getFilesystem(), closedRegion.getConf(),
|
closedRegion.getFilesystem(), closedRegion.getConf(),
|
||||||
closedRegion.getRegionInfo(), null);
|
closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
|
||||||
r.initialize();
|
r.initialize();
|
||||||
return r;
|
return r;
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Put;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
import org.apache.hadoop.hbase.util.JVMClusterUtil;
|
import org.apache.hadoop.hbase.util.JVMClusterUtil;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -91,6 +92,9 @@ public class TestRegionRebalancing extends HBaseClusterTestCase {
|
||||||
HRegion.addRegionToMETA(meta, region);
|
HRegion.addRegionToMETA(meta, region);
|
||||||
}
|
}
|
||||||
closeRootAndMeta();
|
closeRootAndMeta();
|
||||||
|
|
||||||
|
// Add new table descriptor file
|
||||||
|
FSUtils.createTableDescriptor(this.desc, this.conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -183,7 +183,7 @@ public class TestCoprocessorInterface extends HBaseTestCase {
|
||||||
//HRegionInfo info = new HRegionInfo(tableName, null, null, false);
|
//HRegionInfo info = new HRegionInfo(tableName, null, null, false);
|
||||||
HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
|
HRegion r = new HRegion(closedRegion.getTableDir(), closedRegion.getLog(),
|
||||||
closedRegion.getFilesystem(), closedRegion.getConf(),
|
closedRegion.getFilesystem(), closedRegion.getConf(),
|
||||||
closedRegion.getRegionInfo(), null);
|
closedRegion.getRegionInfo(), closedRegion.getTableDesc(), null);
|
||||||
r.initialize();
|
r.initialize();
|
||||||
|
|
||||||
// this following piece is a hack. currently a coprocessorHost
|
// this following piece is a hack. currently a coprocessorHost
|
||||||
|
|
|
@ -275,7 +275,7 @@ public class TestWALObserver {
|
||||||
Path tableDir =
|
Path tableDir =
|
||||||
HTableDescriptor.getTableDir(hbaseRootDir, hri.getTableName());
|
HTableDescriptor.getTableDir(hbaseRootDir, hri.getTableName());
|
||||||
HRegion region = new HRegion(tableDir, wal2, FileSystem.get(newConf),
|
HRegion region = new HRegion(tableDir, wal2, FileSystem.get(newConf),
|
||||||
newConf, hri, TEST_UTIL.getHBaseCluster().getRegionServer(0));
|
newConf, hri, htd, TEST_UTIL.getHBaseCluster().getRegionServer(0));
|
||||||
|
|
||||||
long seqid2 = region.initialize();
|
long seqid2 = region.initialize();
|
||||||
SampleRegionWALObserver cp2 =
|
SampleRegionWALObserver cp2 =
|
||||||
|
|
|
@ -24,9 +24,11 @@ import static org.junit.Assert.assertNotNull;
|
||||||
import static org.junit.Assert.assertNull;
|
import static org.junit.Assert.assertNull;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
@ -40,6 +42,8 @@ import org.apache.hadoop.hbase.KeyValue;
|
||||||
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
|
import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
|
||||||
import org.apache.hadoop.hbase.Server;
|
import org.apache.hadoop.hbase.Server;
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
|
import org.apache.hadoop.hbase.TableExistsException;
|
||||||
import org.apache.hadoop.hbase.catalog.CatalogTracker;
|
import org.apache.hadoop.hbase.catalog.CatalogTracker;
|
||||||
import org.apache.hadoop.hbase.client.Result;
|
import org.apache.hadoop.hbase.client.Result;
|
||||||
import org.apache.hadoop.hbase.executor.ExecutorService;
|
import org.apache.hadoop.hbase.executor.ExecutorService;
|
||||||
|
@ -119,11 +123,8 @@ public class TestCatalogJanitor {
|
||||||
private final AssignmentManager asm;
|
private final AssignmentManager asm;
|
||||||
|
|
||||||
MockMasterServices(final Server server) throws IOException {
|
MockMasterServices(final Server server) throws IOException {
|
||||||
this.mfs = new MasterFileSystem(server, null);
|
this.mfs = new MasterFileSystem(server, this, null);
|
||||||
HTableDescriptor htd = new HTableDescriptor("table");
|
|
||||||
htd.addFamily(new HColumnDescriptor("family"));
|
|
||||||
this.asm = Mockito.mock(AssignmentManager.class);
|
this.asm = Mockito.mock(AssignmentManager.class);
|
||||||
Mockito.when(asm.getTableDescriptor("table")).thenReturn(htd);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -185,6 +186,43 @@ public class TestCatalogJanitor {
|
||||||
public boolean isStopped() {
|
public boolean isStopped() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public TableDescriptors getTableDescriptors() {
|
||||||
|
return new TableDescriptors() {
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor remove(String tablename) throws IOException {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String, HTableDescriptor> getAll() throws IOException {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor get(byte[] tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
|
return get(Bytes.toString(tablename));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor get(String tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
|
HTableDescriptor htd = new HTableDescriptor("table");
|
||||||
|
htd.addFamily(new HColumnDescriptor("family"));
|
||||||
|
return htd;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void add(HTableDescriptor htd) throws IOException {
|
||||||
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -87,7 +87,7 @@ public class TestCompactSelection extends TestCase {
|
||||||
HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
|
HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
|
||||||
HRegion.createHRegion(info, basedir, conf, htd);
|
HRegion.createHRegion(info, basedir, conf, htd);
|
||||||
Path tableDir = new Path(basedir, Bytes.toString(htd.getName()));
|
Path tableDir = new Path(basedir, Bytes.toString(htd.getName()));
|
||||||
HRegion region = new HRegion(tableDir, hlog, fs, conf, info, null);
|
HRegion region = new HRegion(tableDir, hlog, fs, conf, info, htd, null);
|
||||||
|
|
||||||
store = new Store(basedir, region, hcd, fs, conf);
|
store = new Store(basedir, region, hcd, fs, conf);
|
||||||
TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir());
|
TEST_FILE = StoreFile.getRandomFilename(fs, store.getHomedir());
|
||||||
|
|
|
@ -154,7 +154,7 @@ public class TestSplitTransaction {
|
||||||
for (HRegion r: daughters) {
|
for (HRegion r: daughters) {
|
||||||
// Open so can count its content.
|
// Open so can count its content.
|
||||||
HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
|
HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
|
||||||
r.getLog(), r.getConf());
|
r.getTableDesc(), r.getLog(), r.getConf());
|
||||||
try {
|
try {
|
||||||
int count = countRows(openRegion);
|
int count = countRows(openRegion);
|
||||||
assertTrue(count > 0 && count != rowcount);
|
assertTrue(count > 0 && count != rowcount);
|
||||||
|
@ -209,7 +209,7 @@ public class TestSplitTransaction {
|
||||||
for (HRegion r: daughters) {
|
for (HRegion r: daughters) {
|
||||||
// Open so can count its content.
|
// Open so can count its content.
|
||||||
HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
|
HRegion openRegion = HRegion.openHRegion(this.testdir, r.getRegionInfo(),
|
||||||
r.getLog(), r.getConf());
|
r.getTableDesc(), r.getLog(), r.getConf());
|
||||||
try {
|
try {
|
||||||
int count = countRows(openRegion);
|
int count = countRows(openRegion);
|
||||||
assertTrue(count > 0 && count != rowcount);
|
assertTrue(count > 0 && count != rowcount);
|
||||||
|
@ -254,6 +254,7 @@ public class TestSplitTransaction {
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
HRegionInfo hri = new HRegionInfo(htd.getName(), STARTROW, ENDROW);
|
HRegionInfo hri = new HRegionInfo(htd.getName(), STARTROW, ENDROW);
|
||||||
HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
|
HRegion.createHRegion(hri, testdir, TEST_UTIL.getConfiguration(), htd);
|
||||||
return HRegion.openHRegion(testdir, hri, wal, TEST_UTIL.getConfiguration());
|
return HRegion.openHRegion(testdir, hri, htd, wal,
|
||||||
|
TEST_UTIL.getConfiguration());
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -56,7 +56,6 @@ import org.apache.hadoop.hbase.security.User;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
|
||||||
import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
|
|
||||||
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
|
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
|
|
||||||
|
@ -129,7 +128,7 @@ public class TestStore extends TestCase {
|
||||||
htd.addFamily(hcd);
|
htd.addFamily(hcd);
|
||||||
HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
|
HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
|
||||||
HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
|
HLog hlog = new HLog(fs, logdir, oldLogDir, conf);
|
||||||
HRegion region = new HRegion(basedir, hlog, fs, conf, info, null);
|
HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null);
|
||||||
|
|
||||||
store = new Store(basedir, region, hcd, fs, conf);
|
store = new Store(basedir, region, hcd, fs, conf);
|
||||||
}
|
}
|
||||||
|
|
|
@ -238,7 +238,7 @@ public class TestOpenRegionHandler {
|
||||||
HRegion region =
|
HRegion region =
|
||||||
HRegion.createHRegion(hri, HBaseTestingUtility.getTestDir(), HTU
|
HRegion.createHRegion(hri, HBaseTestingUtility.getTestDir(), HTU
|
||||||
.getConfiguration(), htd);
|
.getConfiguration(), htd);
|
||||||
OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri) {
|
OpenRegionHandler handler = new OpenRegionHandler(server, rss, hri, htd) {
|
||||||
HRegion openRegion() {
|
HRegion openRegion() {
|
||||||
// Open region first, then remove znode as though it'd been hijacked.
|
// Open region first, then remove znode as though it'd been hijacked.
|
||||||
//HRegion region = super.openRegion();
|
//HRegion region = super.openRegion();
|
||||||
|
|
|
@ -165,7 +165,7 @@ public class TestWALReplay {
|
||||||
wal3.setSequenceNumber(wal2.getSequenceNumber());
|
wal3.setSequenceNumber(wal2.getSequenceNumber());
|
||||||
try {
|
try {
|
||||||
final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
|
final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
|
||||||
null);
|
htd, null);
|
||||||
long seqid = region.initialize();
|
long seqid = region.initialize();
|
||||||
assertTrue(seqid > wal3.getSequenceNumber());
|
assertTrue(seqid > wal3.getSequenceNumber());
|
||||||
|
|
||||||
|
@ -193,11 +193,11 @@ public class TestWALReplay {
|
||||||
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
|
final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
|
||||||
final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
|
final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
|
||||||
deleteDir(basedir);
|
deleteDir(basedir);
|
||||||
HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
|
final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
|
||||||
HRegion region2 = HRegion.createHRegion(hri,
|
HRegion region2 = HRegion.createHRegion(hri,
|
||||||
hbaseRootDir, this.conf, htd);
|
hbaseRootDir, this.conf, htd);
|
||||||
HLog wal = createWAL(this.conf);
|
HLog wal = createWAL(this.conf);
|
||||||
HRegion region = HRegion.openHRegion(hri, wal, this.conf);
|
HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
|
||||||
Path f = new Path(basedir, "hfile");
|
Path f = new Path(basedir, "hfile");
|
||||||
HFile.Writer writer = new HFile.Writer(this.fs, f);
|
HFile.Writer writer = new HFile.Writer(this.fs, f);
|
||||||
byte [] family = htd.getFamilies().iterator().next().getName();
|
byte [] family = htd.getFamilies().iterator().next().getName();
|
||||||
|
@ -218,7 +218,7 @@ public class TestWALReplay {
|
||||||
runWALSplit(newConf);
|
runWALSplit(newConf);
|
||||||
HLog wal2 = createWAL(newConf);
|
HLog wal2 = createWAL(newConf);
|
||||||
HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
|
HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
|
||||||
newConf, hri, null);
|
newConf, hri, htd, null);
|
||||||
long seqid2 = region2.initialize();
|
long seqid2 = region2.initialize();
|
||||||
assertTrue(seqid2 > -1);
|
assertTrue(seqid2 > -1);
|
||||||
|
|
||||||
|
@ -257,7 +257,7 @@ public class TestWALReplay {
|
||||||
// of the families during the load of edits so its seqid is not same as
|
// of the families during the load of edits so its seqid is not same as
|
||||||
// others to test we do right thing when different seqids.
|
// others to test we do right thing when different seqids.
|
||||||
HLog wal = createWAL(this.conf);
|
HLog wal = createWAL(this.conf);
|
||||||
HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, null);
|
HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
|
||||||
long seqid = region.initialize();
|
long seqid = region.initialize();
|
||||||
// HRegionServer usually does this. It knows the largest seqid across all regions.
|
// HRegionServer usually does this. It knows the largest seqid across all regions.
|
||||||
wal.setSequenceNumber(seqid);
|
wal.setSequenceNumber(seqid);
|
||||||
|
@ -282,7 +282,7 @@ public class TestWALReplay {
|
||||||
wal.close();
|
wal.close();
|
||||||
runWALSplit(this.conf);
|
runWALSplit(this.conf);
|
||||||
HLog wal2 = createWAL(this.conf);
|
HLog wal2 = createWAL(this.conf);
|
||||||
HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, null) {
|
HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null) {
|
||||||
@Override
|
@Override
|
||||||
protected boolean restoreEdit(Store s, KeyValue kv) {
|
protected boolean restoreEdit(Store s, KeyValue kv) {
|
||||||
super.restoreEdit(s, kv);
|
super.restoreEdit(s, kv);
|
||||||
|
@ -317,7 +317,7 @@ public class TestWALReplay {
|
||||||
// Make a new wal for new region open.
|
// Make a new wal for new region open.
|
||||||
HLog wal3 = createWAL(newConf);
|
HLog wal3 = createWAL(newConf);
|
||||||
final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
|
final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
|
||||||
HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, null) {
|
HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
|
||||||
@Override
|
@Override
|
||||||
protected boolean restoreEdit(Store s, KeyValue kv) {
|
protected boolean restoreEdit(Store s, KeyValue kv) {
|
||||||
boolean b = super.restoreEdit(s, kv);
|
boolean b = super.restoreEdit(s, kv);
|
||||||
|
@ -409,7 +409,7 @@ public class TestWALReplay {
|
||||||
final AtomicInteger flushcount = new AtomicInteger(0);
|
final AtomicInteger flushcount = new AtomicInteger(0);
|
||||||
try {
|
try {
|
||||||
final HRegion region =
|
final HRegion region =
|
||||||
new HRegion(basedir, newWal, newFS, newConf, hri, null) {
|
new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
|
||||||
protected boolean internalFlushcache(
|
protected boolean internalFlushcache(
|
||||||
final HLog wal, final long myseqid, MonitoredTask status)
|
final HLog wal, final long myseqid, MonitoredTask status)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
|
@ -0,0 +1,133 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.util;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
|
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.TableDescriptors;
|
||||||
|
import org.apache.hadoop.hbase.TableExistsException;
|
||||||
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
|
import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests for {@link FSTableDescriptors}.
|
||||||
|
*/
|
||||||
|
public class TestFSTableDescriptors {
|
||||||
|
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRemoves() throws IOException {
|
||||||
|
final String name = "testRemoves";
|
||||||
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
|
// Cleanup old tests if any detrius laying around.
|
||||||
|
Path rootdir = new Path(HBaseTestingUtility.getTestDir(), name);
|
||||||
|
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
||||||
|
HTableDescriptor htd = new HTableDescriptor(name);
|
||||||
|
htds.add(htd);
|
||||||
|
assertNotNull(htds.remove(htd.getNameAsString()));
|
||||||
|
assertNull(htds.remove(htd.getNameAsString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test public void testReadingHTDFromFS() throws IOException {
|
||||||
|
final String name = "testReadingHTDFromFS";
|
||||||
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
|
HTableDescriptor htd = new HTableDescriptor(name);
|
||||||
|
Path rootdir = HBaseTestingUtility.getTestDir(name);
|
||||||
|
createHTDInFS(fs, rootdir, htd);
|
||||||
|
HTableDescriptor htd2 =
|
||||||
|
FSUtils.getTableDescriptor(fs, rootdir, htd.getNameAsString());
|
||||||
|
assertTrue(htd.equals(htd2));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createHTDInFS(final FileSystem fs, Path rootdir,
|
||||||
|
final HTableDescriptor htd)
|
||||||
|
throws IOException {
|
||||||
|
FSUtils.createTableDescriptor(fs, rootdir, htd);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test public void testHTableDescriptors()
|
||||||
|
throws IOException, InterruptedException {
|
||||||
|
final String name = "testHTableDescriptors";
|
||||||
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
|
// Cleanup old tests if any detrius laying around.
|
||||||
|
Path rootdir = new Path(HBaseTestingUtility.getTestDir(), name);
|
||||||
|
final int count = 10;
|
||||||
|
// Write out table infos.
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
HTableDescriptor htd = new HTableDescriptor(name + i);
|
||||||
|
createHTDInFS(fs, rootdir, htd);
|
||||||
|
}
|
||||||
|
FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
|
||||||
|
}
|
||||||
|
// Update the table infos
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
HTableDescriptor htd = new HTableDescriptor(name + i);
|
||||||
|
htd.addFamily(new HColumnDescriptor("" + i));
|
||||||
|
FSUtils.updateHTableDescriptor(fs, rootdir, htd);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
|
||||||
|
}
|
||||||
|
assertEquals(htds.invocations, count * 4);
|
||||||
|
assertEquals(htds.cachehits, count * 2);
|
||||||
|
assertTrue(htds.get(HConstants.ROOT_TABLE_NAME) != null);
|
||||||
|
assertEquals(htds.invocations, count * 4 + 1);
|
||||||
|
assertEquals(htds.cachehits, count * 2 + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test (expected=java.io.FileNotFoundException.class)
|
||||||
|
public void testNoSuchTable() throws IOException {
|
||||||
|
final String name = "testNoSuchTable";
|
||||||
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
|
// Cleanup old tests if any detrius laying around.
|
||||||
|
Path rootdir = new Path(HBaseTestingUtility.getTestDir(), name);
|
||||||
|
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
||||||
|
htds.get("NoSuchTable");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUpdates() throws IOException {
|
||||||
|
final String name = "testUpdates";
|
||||||
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
|
// Cleanup old tests if any detrius laying around.
|
||||||
|
Path rootdir = new Path(HBaseTestingUtility.getTestDir(), name);
|
||||||
|
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
||||||
|
HTableDescriptor htd = new HTableDescriptor(name);
|
||||||
|
htds.add(htd);
|
||||||
|
htds.add(htd);
|
||||||
|
htds.add(htd);
|
||||||
|
}
|
||||||
|
}
|
|
@ -190,7 +190,7 @@ public class TestMergeTool extends HBaseTestCase {
|
||||||
|
|
||||||
// Now verify that we can read all the rows from regions 0, 1
|
// Now verify that we can read all the rows from regions 0, 1
|
||||||
// in the new merged region.
|
// in the new merged region.
|
||||||
HRegion merged = HRegion.openHRegion(mergedInfo, log, this.conf);
|
HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf);
|
||||||
verifyMerge(merged, upperbound);
|
verifyMerge(merged, upperbound);
|
||||||
merged.close();
|
merged.close();
|
||||||
LOG.info("Verified " + msg);
|
LOG.info("Verified " + msg);
|
||||||
|
|
Loading…
Reference in New Issue