HDFS-7775. Use consistent naming for NN-internal quota related types and functions. (Contributed bu Xiaoyu Yao)
This commit is contained in:
parent
751b834ad1
commit
c6aa6cbfe5
|
@ -654,6 +654,9 @@ Release 2.7.0 - UNRELEASED
|
||||||
HDFS-7778. Rename FsVolumeListTest to TestFsVolumeList and commit it to
|
HDFS-7778. Rename FsVolumeListTest to TestFsVolumeList and commit it to
|
||||||
branch-2. (Lei (Eddy) Xu via cnauroth)
|
branch-2. (Lei (Eddy) Xu via cnauroth)
|
||||||
|
|
||||||
|
HDFS-4625. BKJM doesn't take advantage of speculative reads. (Rakesh R
|
||||||
|
via aajisaka)
|
||||||
|
|
||||||
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
|
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
|
||||||
|
|
||||||
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
|
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
|
||||||
|
@ -668,8 +671,8 @@ Release 2.7.0 - UNRELEASED
|
||||||
HDFS-7776. Adding additional unit tests for Quota By Storage Type.
|
HDFS-7776. Adding additional unit tests for Quota By Storage Type.
|
||||||
(Xiaoyu Yao via Arpit Agarwal)
|
(Xiaoyu Yao via Arpit Agarwal)
|
||||||
|
|
||||||
HDFS-4625. BKJM doesn't take advantage of speculative reads. (Rakesh R
|
HDFS-7775. Use consistent naming for NN-internal quota related types
|
||||||
via aajisaka)
|
and functions. (Xiaoyu Yao via Arpit Agarwal)
|
||||||
|
|
||||||
Release 2.6.1 - UNRELEASED
|
Release 2.6.1 - UNRELEASED
|
||||||
|
|
||||||
|
|
|
@ -3019,22 +3019,22 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
* Sets or resets quotas for a directory.
|
* Sets or resets quotas for a directory.
|
||||||
* @see ClientProtocol#setQuota(String, long, long, StorageType)
|
* @see ClientProtocol#setQuota(String, long, long, StorageType)
|
||||||
*/
|
*/
|
||||||
void setQuota(String src, long namespaceQuota, long diskspaceQuota)
|
void setQuota(String src, long namespaceQuota, long storagespaceQuota)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
// sanity check
|
// sanity check
|
||||||
if ((namespaceQuota <= 0 && namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
|
if ((namespaceQuota <= 0 && namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
|
||||||
namespaceQuota != HdfsConstants.QUOTA_RESET) ||
|
namespaceQuota != HdfsConstants.QUOTA_RESET) ||
|
||||||
(diskspaceQuota <= 0 && diskspaceQuota != HdfsConstants.QUOTA_DONT_SET &&
|
(storagespaceQuota <= 0 && storagespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
|
||||||
diskspaceQuota != HdfsConstants.QUOTA_RESET)) {
|
storagespaceQuota != HdfsConstants.QUOTA_RESET)) {
|
||||||
throw new IllegalArgumentException("Invalid values for quota : " +
|
throw new IllegalArgumentException("Invalid values for quota : " +
|
||||||
namespaceQuota + " and " +
|
namespaceQuota + " and " +
|
||||||
diskspaceQuota);
|
storagespaceQuota);
|
||||||
|
|
||||||
}
|
}
|
||||||
TraceScope scope = getPathTraceScope("setQuota", src);
|
TraceScope scope = getPathTraceScope("setQuota", src);
|
||||||
try {
|
try {
|
||||||
// Pass null as storage type for traditional space/namespace quota.
|
// Pass null as storage type for traditional namespace/storagespace quota.
|
||||||
namenode.setQuota(src, namespaceQuota, diskspaceQuota, null);
|
namenode.setQuota(src, namespaceQuota, storagespaceQuota, null);
|
||||||
} catch(RemoteException re) {
|
} catch(RemoteException re) {
|
||||||
throw re.unwrapRemoteException(AccessControlException.class,
|
throw re.unwrapRemoteException(AccessControlException.class,
|
||||||
FileNotFoundException.class,
|
FileNotFoundException.class,
|
||||||
|
@ -3051,12 +3051,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
* Sets or resets quotas by storage type for a directory.
|
* Sets or resets quotas by storage type for a directory.
|
||||||
* @see ClientProtocol#setQuota(String, long, long, StorageType)
|
* @see ClientProtocol#setQuota(String, long, long, StorageType)
|
||||||
*/
|
*/
|
||||||
void setQuotaByStorageType(String src, StorageType type, long spaceQuota)
|
void setQuotaByStorageType(String src, StorageType type, long quota)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (spaceQuota <= 0 && spaceQuota != HdfsConstants.QUOTA_DONT_SET &&
|
if (quota <= 0 && quota != HdfsConstants.QUOTA_DONT_SET &&
|
||||||
spaceQuota != HdfsConstants.QUOTA_RESET) {
|
quota != HdfsConstants.QUOTA_RESET) {
|
||||||
throw new IllegalArgumentException("Invalid values for quota :" +
|
throw new IllegalArgumentException("Invalid values for quota :" +
|
||||||
spaceQuota);
|
quota);
|
||||||
}
|
}
|
||||||
if (type == null) {
|
if (type == null) {
|
||||||
throw new IllegalArgumentException("Invalid storage type(null)");
|
throw new IllegalArgumentException("Invalid storage type(null)");
|
||||||
|
@ -3066,7 +3066,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
||||||
+ type.toString());
|
+ type.toString());
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, spaceQuota, type);
|
namenode.setQuota(src, HdfsConstants.QUOTA_DONT_SET, quota, type);
|
||||||
} catch (RemoteException re) {
|
} catch (RemoteException re) {
|
||||||
throw re.unwrapRemoteException(AccessControlException.class,
|
throw re.unwrapRemoteException(AccessControlException.class,
|
||||||
FileNotFoundException.class,
|
FileNotFoundException.class,
|
||||||
|
|
|
@ -692,13 +692,13 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long, StorageType)
|
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long, StorageType)
|
||||||
*/
|
*/
|
||||||
public void setQuota(Path src, final long namespaceQuota,
|
public void setQuota(Path src, final long namespaceQuota,
|
||||||
final long diskspaceQuota) throws IOException {
|
final long storagespaceQuota) throws IOException {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
dfs.setQuota(getPathName(p), namespaceQuota, diskspaceQuota);
|
dfs.setQuota(getPathName(p), namespaceQuota, storagespaceQuota);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
@ -716,18 +716,18 @@ public class DistributedFileSystem extends FileSystem {
|
||||||
*
|
*
|
||||||
* @param src target directory whose quota is to be modified.
|
* @param src target directory whose quota is to be modified.
|
||||||
* @param type storage type of the specific storage type quota to be modified.
|
* @param type storage type of the specific storage type quota to be modified.
|
||||||
* @param spaceQuota value of the specific storage type quota to be modified.
|
* @param quota value of the specific storage type quota to be modified.
|
||||||
* Maybe {@link HdfsConstants#QUOTA_RESET} to clear quota by storage type.
|
* Maybe {@link HdfsConstants#QUOTA_RESET} to clear quota by storage type.
|
||||||
*/
|
*/
|
||||||
public void setQuotaByStorageType(
|
public void setQuotaByStorageType(
|
||||||
Path src, final StorageType type, final long spaceQuota)
|
Path src, final StorageType type, final long quota)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Path absF = fixRelativePart(src);
|
Path absF = fixRelativePart(src);
|
||||||
new FileSystemLinkResolver<Void>() {
|
new FileSystemLinkResolver<Void>() {
|
||||||
@Override
|
@Override
|
||||||
public Void doCall(final Path p)
|
public Void doCall(final Path p)
|
||||||
throws IOException, UnresolvedLinkException {
|
throws IOException, UnresolvedLinkException {
|
||||||
dfs.setQuotaByStorageType(getPathName(p), type, spaceQuota);
|
dfs.setQuotaByStorageType(getPathName(p), type, quota);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -96,8 +96,8 @@ public class HdfsAdmin {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the disk space quota (size of files) for a directory. Note that
|
* Set the storage space quota (size of files) for a directory. Note that
|
||||||
* directories and sym links do not occupy disk space.
|
* directories and sym links do not occupy storage space.
|
||||||
*
|
*
|
||||||
* @param src the path to set the space quota of
|
* @param src the path to set the space quota of
|
||||||
* @param spaceQuota the value to set for the space quota
|
* @param spaceQuota the value to set for the space quota
|
||||||
|
@ -108,8 +108,8 @@ public class HdfsAdmin {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear the disk space quota (size of files) for a directory. Note that
|
* Clear the storage space quota (size of files) for a directory. Note that
|
||||||
* directories and sym links do not occupy disk space.
|
* directories and sym links do not occupy storage space.
|
||||||
*
|
*
|
||||||
* @param src the path to clear the space quota of
|
* @param src the path to clear the space quota of
|
||||||
* @throws IOException in the event of error
|
* @throws IOException in the event of error
|
||||||
|
@ -120,21 +120,21 @@ public class HdfsAdmin {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the quota by storage type for a directory. Note that
|
* Set the quota by storage type for a directory. Note that
|
||||||
* directories and sym links do not occupy disk space.
|
* directories and sym links do not occupy storage type quota.
|
||||||
*
|
*
|
||||||
* @param src the target directory to set the quota by storage type
|
* @param src the target directory to set the quota by storage type
|
||||||
* @param type the storage type to set for quota by storage type
|
* @param type the storage type to set for quota by storage type
|
||||||
* @param spaceQuota the value to set for quota by storage type
|
* @param quota the value to set for quota by storage type
|
||||||
* @throws IOException in the event of error
|
* @throws IOException in the event of error
|
||||||
*/
|
*/
|
||||||
public void setQuotaByStorageType(Path src, StorageType type, long spaceQuota)
|
public void setQuotaByStorageType(Path src, StorageType type, long quota)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
dfs.setQuotaByStorageType(src, type, spaceQuota);
|
dfs.setQuotaByStorageType(src, type, quota);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clear the space quota by storage type for a directory. Note that
|
* Clear the space quota by storage type for a directory. Note that
|
||||||
* directories and sym links do not occupy disk space.
|
* directories and sym links do not occupy storage type quota.
|
||||||
*
|
*
|
||||||
* @param src the target directory to clear the quota by storage type
|
* @param src the target directory to clear the quota by storage type
|
||||||
* @param type the storage type to clear for quota by storage type
|
* @param type the storage type to clear for quota by storage type
|
||||||
|
|
|
@ -951,10 +951,14 @@ public interface ClientProtocol {
|
||||||
* @param path The string representation of the path to the directory
|
* @param path The string representation of the path to the directory
|
||||||
* @param namespaceQuota Limit on the number of names in the tree rooted
|
* @param namespaceQuota Limit on the number of names in the tree rooted
|
||||||
* at the directory
|
* at the directory
|
||||||
* @param diskspaceQuota Limit on disk space occupied all the files under
|
* @param storagespaceQuota Limit on storage space occupied all the files under
|
||||||
* this directory.
|
* this directory.
|
||||||
* @param type StorageType that the space quota is intended to be set on.
|
* @param type StorageType that the space quota is intended to be set on.
|
||||||
* It may be null when called by traditional space/namespace quota.
|
* It may be null when called by traditional space/namespace quota.
|
||||||
|
* When type is is not null, the storagespaceQuota parameter is for
|
||||||
|
* type specified and namespaceQuota must be
|
||||||
|
* {@link HdfsConstants#QUOTA_DONT_SET}.
|
||||||
|
*
|
||||||
* <br><br>
|
* <br><br>
|
||||||
*
|
*
|
||||||
* The quota can have three types of values : (1) 0 or more will set
|
* The quota can have three types of values : (1) 0 or more will set
|
||||||
|
@ -971,7 +975,7 @@ public interface ClientProtocol {
|
||||||
* @throws IOException If an I/O error occurred
|
* @throws IOException If an I/O error occurred
|
||||||
*/
|
*/
|
||||||
@Idempotent
|
@Idempotent
|
||||||
public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
|
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
|
||||||
StorageType type) throws AccessControlException, FileNotFoundException,
|
StorageType type) throws AccessControlException, FileNotFoundException,
|
||||||
UnresolvedLinkException, SnapshotAccessControlException, IOException;
|
UnresolvedLinkException, SnapshotAccessControlException, IOException;
|
||||||
|
|
||||||
|
|
|
@ -887,7 +887,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
|
||||||
SetQuotaRequestProto req) throws ServiceException {
|
SetQuotaRequestProto req) throws ServiceException {
|
||||||
try {
|
try {
|
||||||
server.setQuota(req.getPath(), req.getNamespaceQuota(),
|
server.setQuota(req.getPath(), req.getNamespaceQuota(),
|
||||||
req.getDiskspaceQuota(),
|
req.getStoragespaceQuota(),
|
||||||
req.hasStorageType() ?
|
req.hasStorageType() ?
|
||||||
PBHelper.convertStorageType(req.getStorageType()): null);
|
PBHelper.convertStorageType(req.getStorageType()): null);
|
||||||
return VOID_SETQUOTA_RESPONSE;
|
return VOID_SETQUOTA_RESPONSE;
|
||||||
|
|
|
@ -806,7 +806,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
|
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
|
||||||
StorageType type)
|
StorageType type)
|
||||||
throws AccessControlException, FileNotFoundException,
|
throws AccessControlException, FileNotFoundException,
|
||||||
UnresolvedLinkException, IOException {
|
UnresolvedLinkException, IOException {
|
||||||
|
@ -814,7 +814,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
||||||
= SetQuotaRequestProto.newBuilder()
|
= SetQuotaRequestProto.newBuilder()
|
||||||
.setPath(path)
|
.setPath(path)
|
||||||
.setNamespaceQuota(namespaceQuota)
|
.setNamespaceQuota(namespaceQuota)
|
||||||
.setDiskspaceQuota(diskspaceQuota);
|
.setStoragespaceQuota(storagespaceQuota);
|
||||||
if (type != null) {
|
if (type != null) {
|
||||||
builder.setStorageType(PBHelper.convertStorageType(type));
|
builder.setStorageType(PBHelper.convertStorageType(type));
|
||||||
}
|
}
|
||||||
|
|
|
@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.util.EnumCounters;
|
||||||
*/
|
*/
|
||||||
public final class DirectoryWithQuotaFeature implements INode.Feature {
|
public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE;
|
public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE;
|
||||||
public static final long DEFAULT_SPACE_QUOTA = HdfsConstants.QUOTA_RESET;
|
public static final long DEFAULT_STORAGE_SPACE_QUOTA = HdfsConstants.QUOTA_RESET;
|
||||||
|
|
||||||
private QuotaCounts quota;
|
private QuotaCounts quota;
|
||||||
private QuotaCounts usage;
|
private QuotaCounts usage;
|
||||||
|
@ -41,9 +41,10 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
private QuotaCounts usage;
|
private QuotaCounts usage;
|
||||||
|
|
||||||
public Builder() {
|
public Builder() {
|
||||||
this.quota = new QuotaCounts.Builder().nameCount(DEFAULT_NAMESPACE_QUOTA).
|
this.quota = new QuotaCounts.Builder().nameSpace(DEFAULT_NAMESPACE_QUOTA).
|
||||||
spaceCount(DEFAULT_SPACE_QUOTA).typeCounts(DEFAULT_SPACE_QUOTA).build();
|
storageSpace(DEFAULT_STORAGE_SPACE_QUOTA).
|
||||||
this.usage = new QuotaCounts.Builder().nameCount(1).build();
|
typeSpaces(DEFAULT_STORAGE_SPACE_QUOTA).build();
|
||||||
|
this.usage = new QuotaCounts.Builder().nameSpace(1).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder nameSpaceQuota(long nameSpaceQuota) {
|
public Builder nameSpaceQuota(long nameSpaceQuota) {
|
||||||
|
@ -51,8 +52,8 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder spaceQuota(long spaceQuota) {
|
public Builder storageSpaceQuota(long spaceQuota) {
|
||||||
this.quota.setDiskSpace(spaceQuota);
|
this.quota.setStorageSpace(spaceQuota);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -84,33 +85,41 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
/** Set this directory's quota
|
/** Set this directory's quota
|
||||||
*
|
*
|
||||||
* @param nsQuota Namespace quota to be set
|
* @param nsQuota Namespace quota to be set
|
||||||
* @param dsQuota Diskspace quota to be set
|
* @param ssQuota Storagespace quota to be set
|
||||||
* @param type Storage type quota to be set
|
* @param type Storage type of the storage space quota to be set.
|
||||||
* * To set traditional space/namespace quota, type must be null
|
* To set storagespace/namespace quota, type must be null.
|
||||||
*/
|
*/
|
||||||
void setQuota(long nsQuota, long dsQuota, StorageType type) {
|
void setQuota(long nsQuota, long ssQuota, StorageType type) {
|
||||||
if (type != null) {
|
if (type != null) {
|
||||||
this.quota.setTypeSpace(type, dsQuota);
|
this.quota.setTypeSpace(type, ssQuota);
|
||||||
} else {
|
} else {
|
||||||
setQuota(nsQuota, dsQuota);
|
setQuota(nsQuota, ssQuota);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void setQuota(long nsQuota, long dsQuota) {
|
void setQuota(long nsQuota, long ssQuota) {
|
||||||
this.quota.setNameSpace(nsQuota);
|
this.quota.setNameSpace(nsQuota);
|
||||||
this.quota.setDiskSpace(dsQuota);
|
this.quota.setStorageSpace(ssQuota);
|
||||||
}
|
}
|
||||||
|
|
||||||
void setQuota(long dsQuota, StorageType type) {
|
void setQuota(long quota, StorageType type) {
|
||||||
this.quota.setTypeSpace(type, dsQuota);
|
this.quota.setTypeSpace(type, quota);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set in a batch only during FSImage load
|
/** Set storage type quota in a batch. (Only used by FSImage load)
|
||||||
void setQuota(EnumCounters<StorageType> typeQuotas) {
|
*
|
||||||
this.quota.setTypeSpaces(typeQuotas);
|
* @param tsQuotas type space counts for all storage types supporting quota
|
||||||
|
*/
|
||||||
|
void setQuota(EnumCounters<StorageType> tsQuotas) {
|
||||||
|
this.quota.setTypeSpaces(tsQuotas);
|
||||||
}
|
}
|
||||||
|
|
||||||
QuotaCounts addNamespaceDiskspace(QuotaCounts counts) {
|
/**
|
||||||
|
* Add current quota usage to counts and return the updated counts
|
||||||
|
* @param counts counts to be added with current quota usage
|
||||||
|
* @return counts that have been added with the current qutoa usage
|
||||||
|
*/
|
||||||
|
QuotaCounts AddCurrentSpaceUsage(QuotaCounts counts) {
|
||||||
counts.add(this.usage);
|
counts.add(this.usage);
|
||||||
return counts;
|
return counts;
|
||||||
}
|
}
|
||||||
|
@ -122,15 +131,15 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
dir.computeDirectoryContentSummary(summary, Snapshot.CURRENT_STATE_ID);
|
dir.computeDirectoryContentSummary(summary, Snapshot.CURRENT_STATE_ID);
|
||||||
// Check only when the content has not changed in the middle.
|
// Check only when the content has not changed in the middle.
|
||||||
if (oldYieldCount == summary.getYieldCount()) {
|
if (oldYieldCount == summary.getYieldCount()) {
|
||||||
checkDiskspace(dir, summary.getCounts().get(Content.DISKSPACE) - original);
|
checkStoragespace(dir, summary.getCounts().get(Content.DISKSPACE) - original);
|
||||||
}
|
}
|
||||||
return summary;
|
return summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkDiskspace(final INodeDirectory dir, final long computed) {
|
private void checkStoragespace(final INodeDirectory dir, final long computed) {
|
||||||
if (-1 != quota.getDiskSpace() && usage.getDiskSpace() != computed) {
|
if (-1 != quota.getStorageSpace() && usage.getStorageSpace() != computed) {
|
||||||
NameNode.LOG.error("BUG: Inconsistent diskspace for directory "
|
NameNode.LOG.error("BUG: Inconsistent storagespace for directory "
|
||||||
+ dir.getFullPathName() + ". Cached = " + usage.getDiskSpace()
|
+ dir.getFullPathName() + ". Cached = " + usage.getStorageSpace()
|
||||||
+ " != Computed = " + computed);
|
+ " != Computed = " + computed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -163,28 +172,28 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sets namespace and diskspace take by the directory rooted
|
* Sets namespace and storagespace take by the directory rooted
|
||||||
* at this INode. This should be used carefully. It does not check
|
* at this INode. This should be used carefully. It does not check
|
||||||
* for quota violations.
|
* for quota violations.
|
||||||
*
|
*
|
||||||
* @param namespace size of the directory to be set
|
* @param namespace size of the directory to be set
|
||||||
* @param diskspace disk space take by all the nodes under this directory
|
* @param storagespace storage space take by all the nodes under this directory
|
||||||
* @param typeUsed counters of storage type usage
|
* @param typespaces counters of storage type usage
|
||||||
*/
|
*/
|
||||||
void setSpaceConsumed(long namespace, long diskspace,
|
void setSpaceConsumed(long namespace, long storagespace,
|
||||||
EnumCounters<StorageType> typeUsed) {
|
EnumCounters<StorageType> typespaces) {
|
||||||
usage.setNameSpace(namespace);
|
usage.setNameSpace(namespace);
|
||||||
usage.setDiskSpace(diskspace);
|
usage.setStorageSpace(storagespace);
|
||||||
usage.setTypeSpaces(typeUsed);
|
usage.setTypeSpaces(typespaces);
|
||||||
}
|
}
|
||||||
|
|
||||||
void setSpaceConsumed(QuotaCounts c) {
|
void setSpaceConsumed(QuotaCounts c) {
|
||||||
usage.setNameSpace(c.getNameSpace());
|
usage.setNameSpace(c.getNameSpace());
|
||||||
usage.setDiskSpace(c.getDiskSpace());
|
usage.setStorageSpace(c.getStorageSpace());
|
||||||
usage.setTypeSpaces(c.getTypeSpaces());
|
usage.setTypeSpaces(c.getTypeSpaces());
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return the namespace and diskspace consumed. */
|
/** @return the namespace and storagespace and typespace consumed. */
|
||||||
public QuotaCounts getSpaceConsumed() {
|
public QuotaCounts getSpaceConsumed() {
|
||||||
return new QuotaCounts.Builder().quotaCount(usage).build();
|
return new QuotaCounts.Builder().quotaCount(usage).build();
|
||||||
}
|
}
|
||||||
|
@ -196,11 +205,11 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
usage.getNameSpace() + delta);
|
usage.getNameSpace() + delta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/** Verify if the diskspace quota is violated after applying delta. */
|
/** Verify if the storagespace quota is violated after applying delta. */
|
||||||
private void verifyDiskspaceQuota(long delta) throws DSQuotaExceededException {
|
private void verifyStoragespaceQuota(long delta) throws DSQuotaExceededException {
|
||||||
if (Quota.isViolated(quota.getDiskSpace(), usage.getDiskSpace(), delta)) {
|
if (Quota.isViolated(quota.getStorageSpace(), usage.getStorageSpace(), delta)) {
|
||||||
throw new DSQuotaExceededException(quota.getDiskSpace(),
|
throw new DSQuotaExceededException(quota.getStorageSpace(),
|
||||||
usage.getDiskSpace() + delta);
|
usage.getStorageSpace() + delta);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -222,22 +231,22 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @throws QuotaExceededException if namespace, diskspace or storage type quotas
|
* @throws QuotaExceededException if namespace, storagespace or storage type
|
||||||
* is violated after applying the deltas.
|
* space quota is violated after applying the deltas.
|
||||||
*/
|
*/
|
||||||
void verifyQuota(QuotaCounts counts) throws QuotaExceededException {
|
void verifyQuota(QuotaCounts counts) throws QuotaExceededException {
|
||||||
verifyNamespaceQuota(counts.getNameSpace());
|
verifyNamespaceQuota(counts.getNameSpace());
|
||||||
verifyDiskspaceQuota(counts.getDiskSpace());
|
verifyStoragespaceQuota(counts.getStorageSpace());
|
||||||
verifyQuotaByStorageType(counts.getTypeSpaces());
|
verifyQuotaByStorageType(counts.getTypeSpaces());
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean isQuotaSet() {
|
boolean isQuotaSet() {
|
||||||
return quota.anyNsSpCountGreaterOrEqual(0) ||
|
return quota.anyNsSsCountGreaterOrEqual(0) ||
|
||||||
quota.anyTypeCountGreaterOrEqual(0);
|
quota.anyTypeSpaceCountGreaterOrEqual(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean isQuotaByStorageTypeSet() {
|
boolean isQuotaByStorageTypeSet() {
|
||||||
return quota.anyTypeCountGreaterOrEqual(0);
|
return quota.anyTypeSpaceCountGreaterOrEqual(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean isQuotaByStorageTypeSet(StorageType t) {
|
boolean isQuotaByStorageTypeSet(StorageType t) {
|
||||||
|
@ -248,12 +257,12 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
return "namespace: " + (quota.getNameSpace() < 0? "-":
|
return "namespace: " + (quota.getNameSpace() < 0? "-":
|
||||||
usage.getNameSpace() + "/" + quota.getNameSpace());
|
usage.getNameSpace() + "/" + quota.getNameSpace());
|
||||||
}
|
}
|
||||||
private String diskspaceString() {
|
private String storagespaceString() {
|
||||||
return "diskspace: " + (quota.getDiskSpace() < 0? "-":
|
return "storagespace: " + (quota.getStorageSpace() < 0? "-":
|
||||||
usage.getDiskSpace() + "/" + quota.getDiskSpace());
|
usage.getStorageSpace() + "/" + quota.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
private String quotaByStorageTypeString() {
|
private String typeSpaceString() {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
for (StorageType t : StorageType.getTypesSupportingQuota()) {
|
for (StorageType t : StorageType.getTypesSupportingQuota()) {
|
||||||
sb.append("StorageType: " + t +
|
sb.append("StorageType: " + t +
|
||||||
|
@ -265,7 +274,7 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "Quota[" + namespaceString() + ", " + diskspaceString() +
|
return "Quota[" + namespaceString() + ", " + storagespaceString() +
|
||||||
", " + quotaByStorageTypeString() + "]";
|
", " + typeSpaceString() + "]";
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -219,12 +219,12 @@ public class FSDirAttrOp {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the namespace quota, diskspace and typeSpace quota for a directory.
|
* Set the namespace, storagespace and typespace quota for a directory.
|
||||||
*
|
*
|
||||||
* Note: This does not support ".inodes" relative path.
|
* Note: This does not support ".inodes" relative path.
|
||||||
*/
|
*/
|
||||||
static void setQuota(FSDirectory fsd, String src, long nsQuota, long dsQuota, StorageType type)
|
static void setQuota(FSDirectory fsd, String src, long nsQuota, long ssQuota,
|
||||||
throws IOException {
|
StorageType type) throws IOException {
|
||||||
if (fsd.isPermissionEnabled()) {
|
if (fsd.isPermissionEnabled()) {
|
||||||
FSPermissionChecker pc = fsd.getPermissionChecker();
|
FSPermissionChecker pc = fsd.getPermissionChecker();
|
||||||
pc.checkSuperuserPrivilege();
|
pc.checkSuperuserPrivilege();
|
||||||
|
@ -232,11 +232,11 @@ public class FSDirAttrOp {
|
||||||
|
|
||||||
fsd.writeLock();
|
fsd.writeLock();
|
||||||
try {
|
try {
|
||||||
INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, dsQuota, type);
|
INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, ssQuota, type);
|
||||||
if (changed != null) {
|
if (changed != null) {
|
||||||
final QuotaCounts q = changed.getQuotaCounts();
|
final QuotaCounts q = changed.getQuotaCounts();
|
||||||
if (type == null) {
|
if (type == null) {
|
||||||
fsd.getEditLog().logSetQuota(src, q.getNameSpace(), q.getDiskSpace());
|
fsd.getEditLog().logSetQuota(src, q.getNameSpace(), q.getStorageSpace());
|
||||||
} else {
|
} else {
|
||||||
fsd.getEditLog().logSetQuotaByStorageType(
|
fsd.getEditLog().logSetQuotaByStorageType(
|
||||||
src, q.getTypeSpaces().get(type), type);
|
src, q.getTypeSpaces().get(type), type);
|
||||||
|
@ -314,7 +314,7 @@ public class FSDirAttrOp {
|
||||||
* @throws SnapshotAccessControlException if path is in RO snapshot
|
* @throws SnapshotAccessControlException if path is in RO snapshot
|
||||||
*/
|
*/
|
||||||
static INodeDirectory unprotectedSetQuota(
|
static INodeDirectory unprotectedSetQuota(
|
||||||
FSDirectory fsd, String src, long nsQuota, long dsQuota, StorageType type)
|
FSDirectory fsd, String src, long nsQuota, long ssQuota, StorageType type)
|
||||||
throws FileNotFoundException, PathIsNotDirectoryException,
|
throws FileNotFoundException, PathIsNotDirectoryException,
|
||||||
QuotaExceededException, UnresolvedLinkException,
|
QuotaExceededException, UnresolvedLinkException,
|
||||||
SnapshotAccessControlException, UnsupportedActionException {
|
SnapshotAccessControlException, UnsupportedActionException {
|
||||||
|
@ -322,11 +322,11 @@ public class FSDirAttrOp {
|
||||||
// sanity check
|
// sanity check
|
||||||
if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
|
if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
|
||||||
nsQuota != HdfsConstants.QUOTA_RESET) ||
|
nsQuota != HdfsConstants.QUOTA_RESET) ||
|
||||||
(dsQuota < 0 && dsQuota != HdfsConstants.QUOTA_DONT_SET &&
|
(ssQuota < 0 && ssQuota != HdfsConstants.QUOTA_DONT_SET &&
|
||||||
dsQuota != HdfsConstants.QUOTA_RESET)) {
|
ssQuota != HdfsConstants.QUOTA_RESET)) {
|
||||||
throw new IllegalArgumentException("Illegal value for nsQuota or " +
|
throw new IllegalArgumentException("Illegal value for nsQuota or " +
|
||||||
"dsQuota : " + nsQuota + " and " +
|
"ssQuota : " + nsQuota + " and " +
|
||||||
dsQuota);
|
ssQuota);
|
||||||
}
|
}
|
||||||
// sanity check for quota by storage type
|
// sanity check for quota by storage type
|
||||||
if ((type != null) && (!fsd.isQuotaByStorageTypeEnabled() ||
|
if ((type != null) && (!fsd.isQuotaByStorageTypeEnabled() ||
|
||||||
|
@ -346,31 +346,31 @@ public class FSDirAttrOp {
|
||||||
} else { // a directory inode
|
} else { // a directory inode
|
||||||
final QuotaCounts oldQuota = dirNode.getQuotaCounts();
|
final QuotaCounts oldQuota = dirNode.getQuotaCounts();
|
||||||
final long oldNsQuota = oldQuota.getNameSpace();
|
final long oldNsQuota = oldQuota.getNameSpace();
|
||||||
final long oldDsQuota = oldQuota.getDiskSpace();
|
final long oldSsQuota = oldQuota.getStorageSpace();
|
||||||
|
|
||||||
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
|
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
|
||||||
nsQuota = oldNsQuota;
|
nsQuota = oldNsQuota;
|
||||||
}
|
}
|
||||||
if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
|
if (ssQuota == HdfsConstants.QUOTA_DONT_SET) {
|
||||||
dsQuota = oldDsQuota;
|
ssQuota = oldSsQuota;
|
||||||
}
|
}
|
||||||
|
|
||||||
// unchanged space/namespace quota
|
// unchanged space/namespace quota
|
||||||
if (type == null && oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
|
if (type == null && oldNsQuota == nsQuota && oldSsQuota == ssQuota) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// unchanged type quota
|
// unchanged type quota
|
||||||
if (type != null) {
|
if (type != null) {
|
||||||
EnumCounters<StorageType> oldTypeQuotas = oldQuota.getTypeSpaces();
|
EnumCounters<StorageType> oldTypeQuotas = oldQuota.getTypeSpaces();
|
||||||
if (oldTypeQuotas != null && oldTypeQuotas.get(type) == dsQuota) {
|
if (oldTypeQuotas != null && oldTypeQuotas.get(type) == ssQuota) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
final int latest = iip.getLatestSnapshotId();
|
final int latest = iip.getLatestSnapshotId();
|
||||||
dirNode.recordModification(latest);
|
dirNode.recordModification(latest);
|
||||||
dirNode.setQuota(fsd.getBlockStoragePolicySuite(), nsQuota, dsQuota, type);
|
dirNode.setQuota(fsd.getBlockStoragePolicySuite(), nsQuota, ssQuota, type);
|
||||||
return dirNode;
|
return dirNode;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -393,7 +393,7 @@ public class FSDirAttrOp {
|
||||||
// if replication > oldBR, then newBR == replication.
|
// if replication > oldBR, then newBR == replication.
|
||||||
// if replication < oldBR, we don't know newBR yet.
|
// if replication < oldBR, we don't know newBR yet.
|
||||||
if (replication > oldBR) {
|
if (replication > oldBR) {
|
||||||
long dsDelta = file.diskspaceConsumed()/oldBR;
|
long dsDelta = file.storagespaceConsumed()/oldBR;
|
||||||
fsd.updateCount(iip, 0L, dsDelta, oldBR, replication, true);
|
fsd.updateCount(iip, 0L, dsDelta, oldBR, replication, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -402,7 +402,7 @@ public class FSDirAttrOp {
|
||||||
final short newBR = file.getBlockReplication();
|
final short newBR = file.getBlockReplication();
|
||||||
// check newBR < oldBR case.
|
// check newBR < oldBR case.
|
||||||
if (newBR < oldBR) {
|
if (newBR < oldBR) {
|
||||||
long dsDelta = file.diskspaceConsumed()/newBR;
|
long dsDelta = file.storagespaceConsumed()/newBR;
|
||||||
fsd.updateCount(iip, 0L, dsDelta, oldBR, newBR, true);
|
fsd.updateCount(iip, 0L, dsDelta, oldBR, newBR, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -150,7 +150,7 @@ class FSDirConcatOp {
|
||||||
short srcRepl = src.getBlockReplication();
|
short srcRepl = src.getBlockReplication();
|
||||||
long fileSize = src.computeFileSize();
|
long fileSize = src.computeFileSize();
|
||||||
if (targetRepl != srcRepl) {
|
if (targetRepl != srcRepl) {
|
||||||
deltas.addDiskSpace(fileSize * (targetRepl - srcRepl));
|
deltas.addStorageSpace(fileSize * (targetRepl - srcRepl));
|
||||||
BlockStoragePolicy bsp =
|
BlockStoragePolicy bsp =
|
||||||
fsd.getBlockStoragePolicySuite().getPolicy(src.getStoragePolicyID());
|
fsd.getBlockStoragePolicySuite().getPolicy(src.getStoragePolicyID());
|
||||||
if (bsp != null) {
|
if (bsp != null) {
|
||||||
|
|
|
@ -108,7 +108,8 @@ public class FSDirectory implements Closeable {
|
||||||
r.addDirectoryWithQuotaFeature(
|
r.addDirectoryWithQuotaFeature(
|
||||||
new DirectoryWithQuotaFeature.Builder().
|
new DirectoryWithQuotaFeature.Builder().
|
||||||
nameSpaceQuota(DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA).
|
nameSpaceQuota(DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA).
|
||||||
spaceQuota(DirectoryWithQuotaFeature.DEFAULT_SPACE_QUOTA).build());
|
storageSpaceQuota(DirectoryWithQuotaFeature.DEFAULT_STORAGE_SPACE_QUOTA).
|
||||||
|
build());
|
||||||
r.addSnapshottableFeature();
|
r.addSnapshottableFeature();
|
||||||
r.setSnapshotQuota(0);
|
r.setSnapshotQuota(0);
|
||||||
return r;
|
return r;
|
||||||
|
@ -600,18 +601,18 @@ public class FSDirectory implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Updates namespace and diskspace consumed for all
|
/** Updates namespace, storagespace and typespaces consumed for all
|
||||||
* directories until the parent directory of file represented by path.
|
* directories until the parent directory of file represented by path.
|
||||||
*
|
*
|
||||||
* @param iip the INodesInPath instance containing all the INodes for
|
* @param iip the INodesInPath instance containing all the INodes for
|
||||||
* updating quota usage
|
* updating quota usage
|
||||||
* @param nsDelta the delta change of namespace
|
* @param nsDelta the delta change of namespace
|
||||||
* @param dsDelta the delta change of space consumed without replication
|
* @param ssDelta the delta change of storage space consumed without replication
|
||||||
* @param replication the replication factor of the block consumption change
|
* @param replication the replication factor of the block consumption change
|
||||||
* @throws QuotaExceededException if the new count violates any quota limit
|
* @throws QuotaExceededException if the new count violates any quota limit
|
||||||
* @throws FileNotFoundException if path does not exist.
|
* @throws FileNotFoundException if path does not exist.
|
||||||
*/
|
*/
|
||||||
void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta, short replication)
|
void updateSpaceConsumed(INodesInPath iip, long nsDelta, long ssDelta, short replication)
|
||||||
throws QuotaExceededException, FileNotFoundException,
|
throws QuotaExceededException, FileNotFoundException,
|
||||||
UnresolvedLinkException, SnapshotAccessControlException {
|
UnresolvedLinkException, SnapshotAccessControlException {
|
||||||
writeLock();
|
writeLock();
|
||||||
|
@ -619,7 +620,7 @@ public class FSDirectory implements Closeable {
|
||||||
if (iip.getLastINode() == null) {
|
if (iip.getLastINode() == null) {
|
||||||
throw new FileNotFoundException("Path not found: " + iip.getPath());
|
throw new FileNotFoundException("Path not found: " + iip.getPath());
|
||||||
}
|
}
|
||||||
updateCount(iip, nsDelta, dsDelta, replication, true);
|
updateCount(iip, nsDelta, ssDelta, replication, true);
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock();
|
writeUnlock();
|
||||||
}
|
}
|
||||||
|
@ -641,30 +642,30 @@ public class FSDirectory implements Closeable {
|
||||||
/**
|
/**
|
||||||
* Update usage count without replication factor change
|
* Update usage count without replication factor change
|
||||||
*/
|
*/
|
||||||
void updateCount(INodesInPath iip, long nsDelta, long dsDelta, short replication,
|
void updateCount(INodesInPath iip, long nsDelta, long ssDelta, short replication,
|
||||||
boolean checkQuota) throws QuotaExceededException {
|
boolean checkQuota) throws QuotaExceededException {
|
||||||
final INodeFile fileINode = iip.getLastINode().asFile();
|
final INodeFile fileINode = iip.getLastINode().asFile();
|
||||||
EnumCounters<StorageType> typeSpaceDeltas =
|
EnumCounters<StorageType> typeSpaceDeltas =
|
||||||
getStorageTypeDeltas(fileINode.getStoragePolicyID(), dsDelta,
|
getStorageTypeDeltas(fileINode.getStoragePolicyID(), ssDelta,
|
||||||
replication, replication);;
|
replication, replication);;
|
||||||
updateCount(iip, iip.length() - 1,
|
updateCount(iip, iip.length() - 1,
|
||||||
new QuotaCounts.Builder().nameCount(nsDelta).spaceCount(dsDelta * replication).
|
new QuotaCounts.Builder().nameSpace(nsDelta).storageSpace(ssDelta * replication).
|
||||||
typeCounts(typeSpaceDeltas).build(),
|
typeSpaces(typeSpaceDeltas).build(),
|
||||||
checkQuota);
|
checkQuota);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Update usage count with replication factor change due to setReplication
|
* Update usage count with replication factor change due to setReplication
|
||||||
*/
|
*/
|
||||||
void updateCount(INodesInPath iip, long nsDelta, long dsDelta, short oldRep,
|
void updateCount(INodesInPath iip, long nsDelta, long ssDelta, short oldRep,
|
||||||
short newRep, boolean checkQuota) throws QuotaExceededException {
|
short newRep, boolean checkQuota) throws QuotaExceededException {
|
||||||
final INodeFile fileINode = iip.getLastINode().asFile();
|
final INodeFile fileINode = iip.getLastINode().asFile();
|
||||||
EnumCounters<StorageType> typeSpaceDeltas =
|
EnumCounters<StorageType> typeSpaceDeltas =
|
||||||
getStorageTypeDeltas(fileINode.getStoragePolicyID(), dsDelta, oldRep, newRep);
|
getStorageTypeDeltas(fileINode.getStoragePolicyID(), ssDelta, oldRep, newRep);
|
||||||
updateCount(iip, iip.length() - 1,
|
updateCount(iip, iip.length() - 1,
|
||||||
new QuotaCounts.Builder().nameCount(nsDelta).
|
new QuotaCounts.Builder().nameSpace(nsDelta).
|
||||||
spaceCount(dsDelta * (newRep - oldRep)).
|
storageSpace(ssDelta * (newRep - oldRep)).
|
||||||
typeCounts(typeSpaceDeltas).build(),
|
typeSpaces(typeSpaceDeltas).build(),
|
||||||
checkQuota);
|
checkQuota);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -827,11 +828,11 @@ public class FSDirectory implements Closeable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verify quota for adding or moving a new INode with required
|
* Verify quota for adding or moving a new INode with required
|
||||||
* namespace and diskspace to a given position.
|
* namespace and storagespace to a given position.
|
||||||
*
|
*
|
||||||
* @param iip INodes corresponding to a path
|
* @param iip INodes corresponding to a path
|
||||||
* @param pos position where a new INode will be added
|
* @param pos position where a new INode will be added
|
||||||
* @param deltas needed namespace, diskspace and storage types
|
* @param deltas needed namespace, storagespace and storage types
|
||||||
* @param commonAncestor Last node in inodes array that is a common ancestor
|
* @param commonAncestor Last node in inodes array that is a common ancestor
|
||||||
* for a INode that is being moved from one location to the other.
|
* for a INode that is being moved from one location to the other.
|
||||||
* Pass null if a node is not being moved.
|
* Pass null if a node is not being moved.
|
||||||
|
@ -839,7 +840,7 @@ public class FSDirectory implements Closeable {
|
||||||
*/
|
*/
|
||||||
static void verifyQuota(INodesInPath iip, int pos, QuotaCounts deltas,
|
static void verifyQuota(INodesInPath iip, int pos, QuotaCounts deltas,
|
||||||
INode commonAncestor) throws QuotaExceededException {
|
INode commonAncestor) throws QuotaExceededException {
|
||||||
if (deltas.getNameSpace() <= 0 && deltas.getDiskSpace() <= 0
|
if (deltas.getNameSpace() <= 0 && deltas.getStorageSpace() <= 0
|
||||||
&& deltas.getTypeSpaces().allLessOrEqual(0L)) {
|
&& deltas.getTypeSpaces().allLessOrEqual(0L)) {
|
||||||
// if quota is being freed or not being consumed
|
// if quota is being freed or not being consumed
|
||||||
return;
|
return;
|
||||||
|
@ -1101,12 +1102,12 @@ public class FSDirectory implements Closeable {
|
||||||
INodeFile file = iip.getLastINode().asFile();
|
INodeFile file = iip.getLastINode().asFile();
|
||||||
int latestSnapshot = iip.getLatestSnapshotId();
|
int latestSnapshot = iip.getLatestSnapshotId();
|
||||||
file.recordModification(latestSnapshot, true);
|
file.recordModification(latestSnapshot, true);
|
||||||
long oldDiskspaceNoRep = file.diskspaceConsumedNoReplication();
|
long oldDiskspaceNoRep = file.storagespaceConsumedNoReplication();
|
||||||
long remainingLength =
|
long remainingLength =
|
||||||
file.collectBlocksBeyondMax(newLength, collectedBlocks);
|
file.collectBlocksBeyondMax(newLength, collectedBlocks);
|
||||||
file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
|
file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
|
||||||
file.setModificationTime(mtime);
|
file.setModificationTime(mtime);
|
||||||
updateCount(iip, 0, file.diskspaceConsumedNoReplication() - oldDiskspaceNoRep,
|
updateCount(iip, 0, file.storagespaceConsumedNoReplication() - oldDiskspaceNoRep,
|
||||||
file.getBlockReplication(), true);
|
file.getBlockReplication(), true);
|
||||||
// return whether on a block boundary
|
// return whether on a block boundary
|
||||||
return (remainingLength - newLength) == 0;
|
return (remainingLength - newLength) == 0;
|
||||||
|
|
|
@ -866,7 +866,7 @@ public class FSImage implements Closeable {
|
||||||
private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
|
private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
|
||||||
INodeDirectory dir, QuotaCounts counts) {
|
INodeDirectory dir, QuotaCounts counts) {
|
||||||
final long parentNamespace = counts.getNameSpace();
|
final long parentNamespace = counts.getNameSpace();
|
||||||
final long parentDiskspace = counts.getDiskSpace();
|
final long parentStoragespace = counts.getStorageSpace();
|
||||||
final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
|
final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
|
||||||
|
|
||||||
dir.computeQuotaUsage4CurrentDirectory(bsps, counts);
|
dir.computeQuotaUsage4CurrentDirectory(bsps, counts);
|
||||||
|
@ -892,12 +892,12 @@ public class FSImage implements Closeable {
|
||||||
+ " quota = " + nsQuota + " < consumed = " + namespace);
|
+ " quota = " + nsQuota + " < consumed = " + namespace);
|
||||||
}
|
}
|
||||||
|
|
||||||
final long diskspace = counts.getDiskSpace() - parentDiskspace;
|
final long ssConsumed = counts.getStorageSpace() - parentStoragespace;
|
||||||
final long dsQuota = q.getDiskSpace();
|
final long ssQuota = q.getStorageSpace();
|
||||||
if (Quota.isViolated(dsQuota, diskspace)) {
|
if (Quota.isViolated(ssQuota, ssConsumed)) {
|
||||||
LOG.error("BUG: Diskspace quota violation in image for "
|
LOG.error("BUG: Storagespace quota violation in image for "
|
||||||
+ dir.getFullPathName()
|
+ dir.getFullPathName()
|
||||||
+ " quota = " + dsQuota + " < consumed = " + diskspace);
|
+ " quota = " + ssQuota + " < consumed = " + ssConsumed);
|
||||||
}
|
}
|
||||||
|
|
||||||
final EnumCounters<StorageType> typeSpaces =
|
final EnumCounters<StorageType> typeSpaces =
|
||||||
|
@ -907,14 +907,14 @@ public class FSImage implements Closeable {
|
||||||
parentTypeSpaces.get(t);
|
parentTypeSpaces.get(t);
|
||||||
final long typeQuota = q.getTypeSpaces().get(t);
|
final long typeQuota = q.getTypeSpaces().get(t);
|
||||||
if (Quota.isViolated(typeQuota, typeSpace)) {
|
if (Quota.isViolated(typeQuota, typeSpace)) {
|
||||||
LOG.error("BUG Disk quota by storage type violation in image for "
|
LOG.error("BUG: Storage type quota violation in image for "
|
||||||
+ dir.getFullPathName()
|
+ dir.getFullPathName()
|
||||||
+ " type = " + t.toString() + " quota = "
|
+ " type = " + t.toString() + " quota = "
|
||||||
+ typeQuota + " < consumed " + typeSpace);
|
+ typeQuota + " < consumed " + typeSpace);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, diskspace,
|
dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, ssConsumed,
|
||||||
typeSpaces);
|
typeSpaces);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -450,7 +450,7 @@ public class FSImageFormat {
|
||||||
private void updateRootAttr(INodeWithAdditionalFields root) {
|
private void updateRootAttr(INodeWithAdditionalFields root) {
|
||||||
final QuotaCounts q = root.getQuotaCounts();
|
final QuotaCounts q = root.getQuotaCounts();
|
||||||
final long nsQuota = q.getNameSpace();
|
final long nsQuota = q.getNameSpace();
|
||||||
final long dsQuota = q.getDiskSpace();
|
final long dsQuota = q.getStorageSpace();
|
||||||
FSDirectory fsDir = namesystem.dir;
|
FSDirectory fsDir = namesystem.dir;
|
||||||
if (nsQuota != -1 || dsQuota != -1) {
|
if (nsQuota != -1 || dsQuota != -1) {
|
||||||
fsDir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
|
fsDir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
|
||||||
|
@ -826,7 +826,7 @@ public class FSImageFormat {
|
||||||
permissions, modificationTime);
|
permissions, modificationTime);
|
||||||
if (nsQuota >= 0 || dsQuota >= 0) {
|
if (nsQuota >= 0 || dsQuota >= 0) {
|
||||||
dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
|
dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
|
||||||
nameSpaceQuota(nsQuota).spaceQuota(dsQuota).build());
|
nameSpaceQuota(nsQuota).storageSpaceQuota(dsQuota).build());
|
||||||
}
|
}
|
||||||
if (withSnapshot) {
|
if (withSnapshot) {
|
||||||
dir.addSnapshotFeature(null);
|
dir.addSnapshotFeature(null);
|
||||||
|
@ -910,7 +910,10 @@ public class FSImageFormat {
|
||||||
final PermissionStatus permissions = PermissionStatus.read(in);
|
final PermissionStatus permissions = PermissionStatus.read(in);
|
||||||
final long modificationTime = in.readLong();
|
final long modificationTime = in.readLong();
|
||||||
|
|
||||||
//read quotas
|
// Read quotas: quota by storage type does not need to be processed below.
|
||||||
|
// It is handled only in protobuf based FsImagePBINode class for newer
|
||||||
|
// fsImages. Tools using this class such as legacy-mode of offline image viewer
|
||||||
|
// should only load legacy FSImages without newer features.
|
||||||
final long nsQuota = in.readLong();
|
final long nsQuota = in.readLong();
|
||||||
final long dsQuota = in.readLong();
|
final long dsQuota = in.readLong();
|
||||||
|
|
||||||
|
|
|
@ -169,7 +169,7 @@ public final class FSImageFormatPBINode {
|
||||||
final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
|
final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
|
||||||
if (nsQuota >= 0 || dsQuota >= 0) {
|
if (nsQuota >= 0 || dsQuota >= 0) {
|
||||||
dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
|
dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
|
||||||
nameSpaceQuota(nsQuota).spaceQuota(dsQuota).build());
|
nameSpaceQuota(nsQuota).storageSpaceQuota(dsQuota).build());
|
||||||
}
|
}
|
||||||
EnumCounters<StorageType> typeQuotas = null;
|
EnumCounters<StorageType> typeQuotas = null;
|
||||||
if (d.hasTypeQuotas()) {
|
if (d.hasTypeQuotas()) {
|
||||||
|
@ -374,7 +374,7 @@ public final class FSImageFormatPBINode {
|
||||||
INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
|
INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
|
||||||
final QuotaCounts q = root.getQuotaCounts();
|
final QuotaCounts q = root.getQuotaCounts();
|
||||||
final long nsQuota = q.getNameSpace();
|
final long nsQuota = q.getNameSpace();
|
||||||
final long dsQuota = q.getDiskSpace();
|
final long dsQuota = q.getStorageSpace();
|
||||||
if (nsQuota != -1 || dsQuota != -1) {
|
if (nsQuota != -1 || dsQuota != -1) {
|
||||||
dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
|
dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
|
||||||
}
|
}
|
||||||
|
@ -483,7 +483,7 @@ public final class FSImageFormatPBINode {
|
||||||
INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
|
INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
|
||||||
.newBuilder().setModificationTime(dir.getModificationTime())
|
.newBuilder().setModificationTime(dir.getModificationTime())
|
||||||
.setNsQuota(quota.getNameSpace())
|
.setNsQuota(quota.getNameSpace())
|
||||||
.setDsQuota(quota.getDiskSpace())
|
.setDsQuota(quota.getStorageSpace())
|
||||||
.setPermission(buildPermissionStatus(dir, state.getStringMap()));
|
.setPermission(buildPermissionStatus(dir, state.getStringMap()));
|
||||||
|
|
||||||
if (quota.getTypeSpaces().anyGreaterOrEqual(0)) {
|
if (quota.getTypeSpaces().anyGreaterOrEqual(0)) {
|
||||||
|
|
|
@ -224,7 +224,7 @@ public class FSImageSerialization {
|
||||||
private static void writeQuota(QuotaCounts quota, DataOutput out)
|
private static void writeQuota(QuotaCounts quota, DataOutput out)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
out.writeLong(quota.getNameSpace());
|
out.writeLong(quota.getNameSpace());
|
||||||
out.writeLong(quota.getDiskSpace());
|
out.writeLong(quota.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -3802,20 +3802,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the namespace quota and diskspace quota for a directory.
|
* Set the namespace quota and storage space quota for a directory.
|
||||||
* See {@link ClientProtocol#setQuota(String, long, long, StorageType)} for the
|
* See {@link ClientProtocol#setQuota(String, long, long, StorageType)} for the
|
||||||
* contract.
|
* contract.
|
||||||
*
|
*
|
||||||
* Note: This does not support ".inodes" relative path.
|
* Note: This does not support ".inodes" relative path.
|
||||||
*/
|
*/
|
||||||
void setQuota(String src, long nsQuota, long dsQuota, StorageType type)
|
void setQuota(String src, long nsQuota, long ssQuota, StorageType type)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
checkNameNodeSafeMode("Cannot set quota on " + src);
|
checkNameNodeSafeMode("Cannot set quota on " + src);
|
||||||
FSDirAttrOp.setQuota(dir, src, nsQuota, dsQuota, type);
|
FSDirAttrOp.setQuota(dir, src, nsQuota, ssQuota, type);
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock();
|
writeUnlock();
|
||||||
}
|
}
|
||||||
|
|
|
@ -447,7 +447,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
|
||||||
return new ContentSummary(counts.get(Content.LENGTH),
|
return new ContentSummary(counts.get(Content.LENGTH),
|
||||||
counts.get(Content.FILE) + counts.get(Content.SYMLINK),
|
counts.get(Content.FILE) + counts.get(Content.SYMLINK),
|
||||||
counts.get(Content.DIRECTORY), q.getNameSpace(),
|
counts.get(Content.DIRECTORY), q.getNameSpace(),
|
||||||
counts.get(Content.DISKSPACE), q.getDiskSpace());
|
counts.get(Content.DISKSPACE), q.getStorageSpace());
|
||||||
// TODO: storage type quota reporting HDFS-7701.
|
// TODO: storage type quota reporting HDFS-7701.
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -462,7 +462,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check and add namespace/diskspace/storagetype consumed to itself and the ancestors.
|
* Check and add namespace/storagespace/storagetype consumed to itself and the ancestors.
|
||||||
* @throws QuotaExceededException if quote is violated.
|
* @throws QuotaExceededException if quote is violated.
|
||||||
*/
|
*/
|
||||||
public void addSpaceConsumed(QuotaCounts counts, boolean verify)
|
public void addSpaceConsumed(QuotaCounts counts, boolean verify)
|
||||||
|
@ -471,7 +471,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check and add namespace/diskspace/storagetype consumed to itself and the ancestors.
|
* Check and add namespace/storagespace/storagetype consumed to itself and the ancestors.
|
||||||
* @throws QuotaExceededException if quote is violated.
|
* @throws QuotaExceededException if quote is violated.
|
||||||
*/
|
*/
|
||||||
void addSpaceConsumed2Parent(QuotaCounts counts, boolean verify)
|
void addSpaceConsumed2Parent(QuotaCounts counts, boolean verify)
|
||||||
|
@ -487,26 +487,26 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
|
||||||
*/
|
*/
|
||||||
public QuotaCounts getQuotaCounts() {
|
public QuotaCounts getQuotaCounts() {
|
||||||
return new QuotaCounts.Builder().
|
return new QuotaCounts.Builder().
|
||||||
nameCount(HdfsConstants.QUOTA_RESET).
|
nameSpace(HdfsConstants.QUOTA_RESET).
|
||||||
spaceCount(HdfsConstants.QUOTA_RESET).
|
storageSpace(HdfsConstants.QUOTA_RESET).
|
||||||
typeCounts(HdfsConstants.QUOTA_RESET).
|
typeSpaces(HdfsConstants.QUOTA_RESET).
|
||||||
build();
|
build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public final boolean isQuotaSet() {
|
public final boolean isQuotaSet() {
|
||||||
final QuotaCounts qc = getQuotaCounts();
|
final QuotaCounts qc = getQuotaCounts();
|
||||||
return qc.anyNsSpCountGreaterOrEqual(0) || qc.anyTypeCountGreaterOrEqual(0);
|
return qc.anyNsSsCountGreaterOrEqual(0) || qc.anyTypeSpaceCountGreaterOrEqual(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#DISKSPACE} usages.
|
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#STORAGESPACE} usages.
|
||||||
*/
|
*/
|
||||||
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps) {
|
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps) {
|
||||||
return computeQuotaUsage(bsps, new QuotaCounts.Builder().build(), true);
|
return computeQuotaUsage(bsps, new QuotaCounts.Builder().build(), true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#DISKSPACE} usages.
|
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#STORAGESPACE} usages.
|
||||||
*
|
*
|
||||||
* With the existence of {@link INodeReference}, the same inode and its
|
* With the existence of {@link INodeReference}, the same inode and its
|
||||||
* subtree may be referred by multiple {@link WithName} nodes and a
|
* subtree may be referred by multiple {@link WithName} nodes and a
|
||||||
|
|
|
@ -137,14 +137,14 @@ public class INodeDirectory extends INodeWithAdditionalFields
|
||||||
BlockStoragePolicySuite.ID_UNSPECIFIED;
|
BlockStoragePolicySuite.ID_UNSPECIFIED;
|
||||||
}
|
}
|
||||||
|
|
||||||
void setQuota(BlockStoragePolicySuite bsps, long nsQuota, long dsQuota, StorageType type) {
|
void setQuota(BlockStoragePolicySuite bsps, long nsQuota, long ssQuota, StorageType type) {
|
||||||
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
|
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
|
||||||
if (quota != null) {
|
if (quota != null) {
|
||||||
// already has quota; so set the quota to the new values
|
// already has quota; so set the quota to the new values
|
||||||
if (type != null) {
|
if (type != null) {
|
||||||
quota.setQuota(dsQuota, type);
|
quota.setQuota(ssQuota, type);
|
||||||
} else {
|
} else {
|
||||||
quota.setQuota(nsQuota, dsQuota);
|
quota.setQuota(nsQuota, ssQuota);
|
||||||
}
|
}
|
||||||
if (!isQuotaSet() && !isRoot()) {
|
if (!isQuotaSet() && !isRoot()) {
|
||||||
removeFeature(quota);
|
removeFeature(quota);
|
||||||
|
@ -154,9 +154,9 @@ public class INodeDirectory extends INodeWithAdditionalFields
|
||||||
DirectoryWithQuotaFeature.Builder builder =
|
DirectoryWithQuotaFeature.Builder builder =
|
||||||
new DirectoryWithQuotaFeature.Builder().nameSpaceQuota(nsQuota);
|
new DirectoryWithQuotaFeature.Builder().nameSpaceQuota(nsQuota);
|
||||||
if (type != null) {
|
if (type != null) {
|
||||||
builder.typeQuota(type, dsQuota);
|
builder.typeQuota(type, ssQuota);
|
||||||
} else {
|
} else {
|
||||||
builder.spaceQuota(dsQuota);
|
builder.storageSpaceQuota(ssQuota);
|
||||||
}
|
}
|
||||||
addDirectoryWithQuotaFeature(builder.build()).setSpaceConsumed(c);
|
addDirectoryWithQuotaFeature(builder.build()).setSpaceConsumed(c);
|
||||||
}
|
}
|
||||||
|
@ -588,7 +588,7 @@ public class INodeDirectory extends INodeWithAdditionalFields
|
||||||
// compute the quota usage in the scope of the current directory tree
|
// compute the quota usage in the scope of the current directory tree
|
||||||
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
|
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
|
||||||
if (useCache && q != null && q.isQuotaSet()) { // use the cached quota
|
if (useCache && q != null && q.isQuotaSet()) { // use the cached quota
|
||||||
return q.addNamespaceDiskspace(counts);
|
return q.AddCurrentSpaceUsage(counts);
|
||||||
} else {
|
} else {
|
||||||
useCache = q != null && !q.isQuotaSet() ? false : useCache;
|
useCache = q != null && !q.isQuotaSet() ? false : useCache;
|
||||||
return computeDirectoryQuotaUsage(bsps, counts, useCache, lastSnapshotId);
|
return computeDirectoryQuotaUsage(bsps, counts, useCache, lastSnapshotId);
|
||||||
|
|
|
@ -48,8 +48,8 @@ public interface INodeDirectoryAttributes extends INodeAttributes {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public QuotaCounts getQuotaCounts() {
|
public QuotaCounts getQuotaCounts() {
|
||||||
return new QuotaCounts.Builder().nameCount(-1).
|
return new QuotaCounts.Builder().nameSpace(-1).
|
||||||
spaceCount(-1).typeCounts(-1).build();
|
storageSpace(-1).typeSpaces(-1).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -69,8 +69,8 @@ public interface INodeDirectoryAttributes extends INodeAttributes {
|
||||||
AclFeature aclFeature, long modificationTime, long nsQuota,
|
AclFeature aclFeature, long modificationTime, long nsQuota,
|
||||||
long dsQuota, EnumCounters<StorageType> typeQuotas, XAttrFeature xAttrsFeature) {
|
long dsQuota, EnumCounters<StorageType> typeQuotas, XAttrFeature xAttrsFeature) {
|
||||||
super(name, permissions, aclFeature, modificationTime, xAttrsFeature);
|
super(name, permissions, aclFeature, modificationTime, xAttrsFeature);
|
||||||
this.quota = new QuotaCounts.Builder().nameCount(nsQuota).
|
this.quota = new QuotaCounts.Builder().nameSpace(nsQuota).
|
||||||
spaceCount(dsQuota).typeCounts(typeQuotas).build();
|
storageSpace(dsQuota).typeSpaces(typeQuotas).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public CopyWithQuota(INodeDirectory dir) {
|
public CopyWithQuota(INodeDirectory dir) {
|
||||||
|
|
|
@ -412,8 +412,8 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
return header;
|
return header;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return the diskspace required for a full block. */
|
/** @return the storagespace required for a full block. */
|
||||||
final long getPreferredBlockDiskspace() {
|
final long getPreferredBlockStoragespace() {
|
||||||
return getPreferredBlockSize() * getBlockReplication();
|
return getPreferredBlockSize() * getBlockReplication();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -553,8 +553,8 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
|
BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
|
||||||
int lastSnapshotId) {
|
int lastSnapshotId) {
|
||||||
long nsDelta = 1;
|
long nsDelta = 1;
|
||||||
final long dsDeltaNoReplication;
|
final long ssDeltaNoReplication;
|
||||||
short dsReplication;
|
short replication;
|
||||||
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
|
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
|
||||||
if (sf != null) {
|
if (sf != null) {
|
||||||
FileDiffList fileDiffList = sf.getDiffs();
|
FileDiffList fileDiffList = sf.getDiffs();
|
||||||
|
@ -562,31 +562,31 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
|
|
||||||
if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
|
if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
|
||||||
|| last == Snapshot.CURRENT_STATE_ID) {
|
|| last == Snapshot.CURRENT_STATE_ID) {
|
||||||
dsDeltaNoReplication = diskspaceConsumedNoReplication();
|
ssDeltaNoReplication = storagespaceConsumedNoReplication();
|
||||||
dsReplication = getBlockReplication();
|
replication = getBlockReplication();
|
||||||
} else if (last < lastSnapshotId) {
|
} else if (last < lastSnapshotId) {
|
||||||
dsDeltaNoReplication = computeFileSize(true, false);
|
ssDeltaNoReplication = computeFileSize(true, false);
|
||||||
dsReplication = getFileReplication();
|
replication = getFileReplication();
|
||||||
} else {
|
} else {
|
||||||
int sid = fileDiffList.getSnapshotById(lastSnapshotId);
|
int sid = fileDiffList.getSnapshotById(lastSnapshotId);
|
||||||
dsDeltaNoReplication = diskspaceConsumedNoReplication(sid);
|
ssDeltaNoReplication = storagespaceConsumedNoReplication(sid);
|
||||||
dsReplication = getReplication(sid);
|
replication = getReplication(sid);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
dsDeltaNoReplication = diskspaceConsumedNoReplication();
|
ssDeltaNoReplication = storagespaceConsumedNoReplication();
|
||||||
dsReplication = getBlockReplication();
|
replication = getBlockReplication();
|
||||||
}
|
}
|
||||||
counts.addNameSpace(nsDelta);
|
counts.addNameSpace(nsDelta);
|
||||||
counts.addDiskSpace(dsDeltaNoReplication * dsReplication);
|
counts.addStorageSpace(ssDeltaNoReplication * replication);
|
||||||
|
|
||||||
if (getStoragePolicyID() != BlockStoragePolicySuite.ID_UNSPECIFIED){
|
if (getStoragePolicyID() != BlockStoragePolicySuite.ID_UNSPECIFIED){
|
||||||
BlockStoragePolicy bsp = bsps.getPolicy(getStoragePolicyID());
|
BlockStoragePolicy bsp = bsps.getPolicy(getStoragePolicyID());
|
||||||
List<StorageType> storageTypes = bsp.chooseStorageTypes(dsReplication);
|
List<StorageType> storageTypes = bsp.chooseStorageTypes(replication);
|
||||||
for (StorageType t : storageTypes) {
|
for (StorageType t : storageTypes) {
|
||||||
if (!t.supportTypeQuota()) {
|
if (!t.supportTypeQuota()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
counts.addTypeSpace(t, dsDeltaNoReplication);
|
counts.addTypeSpace(t, ssDeltaNoReplication);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return counts;
|
return counts;
|
||||||
|
@ -610,7 +610,7 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
counts.add(Content.LENGTH, computeFileSize());
|
counts.add(Content.LENGTH, computeFileSize());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
counts.add(Content.DISKSPACE, diskspaceConsumed());
|
counts.add(Content.DISKSPACE, storagespaceConsumed());
|
||||||
return summary;
|
return summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -681,11 +681,11 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
* including blocks in its snapshots.
|
* including blocks in its snapshots.
|
||||||
* Use preferred block size for the last block if it is under construction.
|
* Use preferred block size for the last block if it is under construction.
|
||||||
*/
|
*/
|
||||||
public final long diskspaceConsumed() {
|
public final long storagespaceConsumed() {
|
||||||
return diskspaceConsumedNoReplication() * getBlockReplication();
|
return storagespaceConsumedNoReplication() * getBlockReplication();
|
||||||
}
|
}
|
||||||
|
|
||||||
public final long diskspaceConsumedNoReplication() {
|
public final long storagespaceConsumedNoReplication() {
|
||||||
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
|
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
|
||||||
if(sf == null) {
|
if(sf == null) {
|
||||||
return computeFileSize(true, true);
|
return computeFileSize(true, true);
|
||||||
|
@ -713,12 +713,12 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
public final long diskspaceConsumed(int lastSnapshotId) {
|
public final long storagespaceConsumed(int lastSnapshotId) {
|
||||||
if (lastSnapshotId != CURRENT_STATE_ID) {
|
if (lastSnapshotId != CURRENT_STATE_ID) {
|
||||||
return computeFileSize(lastSnapshotId)
|
return computeFileSize(lastSnapshotId)
|
||||||
* getFileReplication(lastSnapshotId);
|
* getFileReplication(lastSnapshotId);
|
||||||
} else {
|
} else {
|
||||||
return diskspaceConsumed();
|
return storagespaceConsumed();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -730,11 +730,11 @@ public class INodeFile extends INodeWithAdditionalFields
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public final long diskspaceConsumedNoReplication(int lastSnapshotId) {
|
public final long storagespaceConsumedNoReplication(int lastSnapshotId) {
|
||||||
if (lastSnapshotId != CURRENT_STATE_ID) {
|
if (lastSnapshotId != CURRENT_STATE_ID) {
|
||||||
return computeFileSize(lastSnapshotId);
|
return computeFileSize(lastSnapshotId);
|
||||||
} else {
|
} else {
|
||||||
return diskspaceConsumedNoReplication();
|
return storagespaceConsumedNoReplication();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -509,10 +509,10 @@ public abstract class INodeReference extends INode {
|
||||||
@Override
|
@Override
|
||||||
public final ContentSummaryComputationContext computeContentSummary(
|
public final ContentSummaryComputationContext computeContentSummary(
|
||||||
ContentSummaryComputationContext summary) {
|
ContentSummaryComputationContext summary) {
|
||||||
//only count diskspace for WithName
|
//only count storagespace for WithName
|
||||||
final QuotaCounts q = new QuotaCounts.Builder().build();
|
final QuotaCounts q = new QuotaCounts.Builder().build();
|
||||||
computeQuotaUsage(summary.getBlockStoragePolicySuite(), q, false, lastSnapshotId);
|
computeQuotaUsage(summary.getBlockStoragePolicySuite(), q, false, lastSnapshotId);
|
||||||
summary.getCounts().add(Content.DISKSPACE, q.getDiskSpace());
|
summary.getCounts().add(Content.DISKSPACE, q.getStorageSpace());
|
||||||
return summary;
|
return summary;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -81,7 +81,7 @@ public class INodeSymlink extends INodeWithAdditionalFields {
|
||||||
&& priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
|
&& priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
|
||||||
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
|
destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
|
||||||
}
|
}
|
||||||
return new QuotaCounts.Builder().nameCount(1).build();
|
return new QuotaCounts.Builder().nameSpace(1).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -1186,11 +1186,11 @@ class NameNodeRpcServer implements NamenodeProtocols {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // ClientProtocol
|
@Override // ClientProtocol
|
||||||
public void setQuota(String path, long namespaceQuota, long diskspaceQuota,
|
public void setQuota(String path, long namespaceQuota, long storagespaceQuota,
|
||||||
StorageType type)
|
StorageType type)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
checkNNStartup();
|
checkNNStartup();
|
||||||
namesystem.setQuota(path, namespaceQuota, diskspaceQuota, type);
|
namesystem.setQuota(path, namespaceQuota, storagespaceQuota, type);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // ClientProtocol
|
@Override // ClientProtocol
|
||||||
|
|
|
@ -23,16 +23,16 @@ import org.apache.hadoop.hdfs.util.EnumCounters;
|
||||||
public enum Quota {
|
public enum Quota {
|
||||||
/** The namespace usage, i.e. the number of name objects. */
|
/** The namespace usage, i.e. the number of name objects. */
|
||||||
NAMESPACE,
|
NAMESPACE,
|
||||||
/** The diskspace usage in bytes including replication. */
|
/** The storage space usage in bytes including replication. */
|
||||||
DISKSPACE;
|
STORAGESPACE;
|
||||||
|
|
||||||
/** Counters for quota counts. */
|
/** Counters for quota counts. */
|
||||||
public static class Counts extends EnumCounters<Quota> {
|
public static class Counts extends EnumCounters<Quota> {
|
||||||
/** @return a new counter with the given namespace and diskspace usages. */
|
/** @return a new counter with the given namespace and storagespace usages. */
|
||||||
public static Counts newInstance(long namespace, long diskspace) {
|
public static Counts newInstance(long namespace, long storagespace) {
|
||||||
final Counts c = new Counts();
|
final Counts c = new Counts();
|
||||||
c.set(NAMESPACE, namespace);
|
c.set(NAMESPACE, namespace);
|
||||||
c.set(DISKSPACE, diskspace);
|
c.set(STORAGESPACE, storagespace);
|
||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -22,47 +22,49 @@ import org.apache.hadoop.hdfs.StorageType;
|
||||||
import org.apache.hadoop.hdfs.util.EnumCounters;
|
import org.apache.hadoop.hdfs.util.EnumCounters;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Counters for namespace, space and storage type quota and usage.
|
* Counters for namespace, storage space and storage type space quota and usage.
|
||||||
*/
|
*/
|
||||||
public class QuotaCounts {
|
public class QuotaCounts {
|
||||||
|
// Name space and storage space counts (HDFS-7775 refactors the original disk
|
||||||
private EnumCounters<Quota> nsSpCounts;
|
// space count to storage space counts)
|
||||||
private EnumCounters<StorageType> typeCounts;
|
private EnumCounters<Quota> nsSsCounts;
|
||||||
|
// Storage type space counts
|
||||||
|
private EnumCounters<StorageType> tsCounts;
|
||||||
|
|
||||||
public static class Builder {
|
public static class Builder {
|
||||||
private EnumCounters<Quota> nsSpCounts;
|
private EnumCounters<Quota> nsSsCounts;
|
||||||
private EnumCounters<StorageType> typeCounts;
|
private EnumCounters<StorageType> tsCounts;
|
||||||
|
|
||||||
public Builder() {
|
public Builder() {
|
||||||
this.nsSpCounts = new EnumCounters<Quota>(Quota.class);
|
this.nsSsCounts = new EnumCounters<Quota>(Quota.class);
|
||||||
this.typeCounts = new EnumCounters<StorageType>(StorageType.class);
|
this.tsCounts = new EnumCounters<StorageType>(StorageType.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder nameCount(long val) {
|
public Builder nameSpace(long val) {
|
||||||
this.nsSpCounts.set(Quota.NAMESPACE, val);
|
this.nsSsCounts.set(Quota.NAMESPACE, val);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder spaceCount(long val) {
|
public Builder storageSpace(long val) {
|
||||||
this.nsSpCounts.set(Quota.DISKSPACE, val);
|
this.nsSsCounts.set(Quota.STORAGESPACE, val);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder typeCounts(EnumCounters<StorageType> val) {
|
public Builder typeSpaces(EnumCounters<StorageType> val) {
|
||||||
if (val != null) {
|
if (val != null) {
|
||||||
this.typeCounts.set(val);
|
this.tsCounts.set(val);
|
||||||
}
|
}
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder typeCounts(long val) {
|
public Builder typeSpaces(long val) {
|
||||||
this.typeCounts.reset(val);
|
this.tsCounts.reset(val);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Builder quotaCount(QuotaCounts that) {
|
public Builder quotaCount(QuotaCounts that) {
|
||||||
this.nsSpCounts.set(that.nsSpCounts);
|
this.nsSsCounts.set(that.nsSsCounts);
|
||||||
this.typeCounts.set(that.typeCounts);
|
this.tsCounts.set(that.tsCounts);
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -72,18 +74,18 @@ public class QuotaCounts {
|
||||||
}
|
}
|
||||||
|
|
||||||
private QuotaCounts(Builder builder) {
|
private QuotaCounts(Builder builder) {
|
||||||
this.nsSpCounts = builder.nsSpCounts;
|
this.nsSsCounts = builder.nsSsCounts;
|
||||||
this.typeCounts = builder.typeCounts;
|
this.tsCounts = builder.tsCounts;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void add(QuotaCounts that) {
|
public void add(QuotaCounts that) {
|
||||||
this.nsSpCounts.add(that.nsSpCounts);
|
this.nsSsCounts.add(that.nsSsCounts);
|
||||||
this.typeCounts.add(that.typeCounts);
|
this.tsCounts.add(that.tsCounts);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void subtract(QuotaCounts that) {
|
public void subtract(QuotaCounts that) {
|
||||||
this.nsSpCounts.subtract(that.nsSpCounts);
|
this.nsSsCounts.subtract(that.nsSsCounts);
|
||||||
this.typeCounts.subtract(that.typeCounts);
|
this.tsCounts.subtract(that.tsCounts);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -93,70 +95,66 @@ public class QuotaCounts {
|
||||||
*/
|
*/
|
||||||
public QuotaCounts negation() {
|
public QuotaCounts negation() {
|
||||||
QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build();
|
QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build();
|
||||||
ret.nsSpCounts.negation();
|
ret.nsSsCounts.negation();
|
||||||
ret.typeCounts.negation();
|
ret.tsCounts.negation();
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getNameSpace(){
|
public long getNameSpace(){
|
||||||
return nsSpCounts.get(Quota.NAMESPACE);
|
return nsSsCounts.get(Quota.NAMESPACE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setNameSpace(long nameSpaceCount) {
|
public void setNameSpace(long nameSpaceCount) {
|
||||||
this.nsSpCounts.set(Quota.NAMESPACE, nameSpaceCount);
|
this.nsSsCounts.set(Quota.NAMESPACE, nameSpaceCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addNameSpace(long nsDelta) {
|
public void addNameSpace(long nsDelta) {
|
||||||
this.nsSpCounts.add(Quota.NAMESPACE, nsDelta);
|
this.nsSsCounts.add(Quota.NAMESPACE, nsDelta);
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getDiskSpace(){
|
public long getStorageSpace(){
|
||||||
return nsSpCounts.get(Quota.DISKSPACE);
|
return nsSsCounts.get(Quota.STORAGESPACE);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setDiskSpace(long spaceCount) {
|
public void setStorageSpace(long spaceCount) {
|
||||||
this.nsSpCounts.set(Quota.DISKSPACE, spaceCount);
|
this.nsSsCounts.set(Quota.STORAGESPACE, spaceCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addDiskSpace(long dsDelta) {
|
public void addStorageSpace(long dsDelta) {
|
||||||
this.nsSpCounts.add(Quota.DISKSPACE, dsDelta);
|
this.nsSsCounts.add(Quota.STORAGESPACE, dsDelta);
|
||||||
}
|
}
|
||||||
|
|
||||||
public EnumCounters<StorageType> getTypeSpaces() {
|
public EnumCounters<StorageType> getTypeSpaces() {
|
||||||
EnumCounters<StorageType> ret =
|
EnumCounters<StorageType> ret =
|
||||||
new EnumCounters<StorageType>(StorageType.class);
|
new EnumCounters<StorageType>(StorageType.class);
|
||||||
ret.set(typeCounts);
|
ret.set(tsCounts);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
void setTypeSpaces(EnumCounters<StorageType> that) {
|
void setTypeSpaces(EnumCounters<StorageType> that) {
|
||||||
if (that != null) {
|
if (that != null) {
|
||||||
this.typeCounts.set(that);
|
this.tsCounts.set(that);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
long getTypeSpace(StorageType type) {
|
long getTypeSpace(StorageType type) {
|
||||||
return this.typeCounts.get(type);
|
return this.tsCounts.get(type);
|
||||||
}
|
}
|
||||||
|
|
||||||
void setTypeSpace(StorageType type, long spaceCount) {
|
void setTypeSpace(StorageType type, long spaceCount) {
|
||||||
this.typeCounts.set(type, spaceCount);
|
this.tsCounts.set(type, spaceCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addTypeSpace(StorageType type, long delta) {
|
public void addTypeSpace(StorageType type, long delta) {
|
||||||
this.typeCounts.add(type, delta);
|
this.tsCounts.add(type, delta);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void addTypeSpaces(EnumCounters<StorageType> deltas) {
|
public boolean anyNsSsCountGreaterOrEqual(long val) {
|
||||||
this.typeCounts.add(deltas);
|
return nsSsCounts.anyGreaterOrEqual(val);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean anyNsSpCountGreaterOrEqual(long val) {
|
public boolean anyTypeSpaceCountGreaterOrEqual(long val) {
|
||||||
return nsSpCounts.anyGreaterOrEqual(val);
|
return tsCounts.anyGreaterOrEqual(val);
|
||||||
}
|
|
||||||
|
|
||||||
public boolean anyTypeCountGreaterOrEqual(long val) {
|
|
||||||
return typeCounts.anyGreaterOrEqual(val);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -167,8 +165,8 @@ public class QuotaCounts {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
final QuotaCounts that = (QuotaCounts)obj;
|
final QuotaCounts that = (QuotaCounts)obj;
|
||||||
return this.nsSpCounts.equals(that.nsSpCounts)
|
return this.nsSsCounts.equals(that.nsSsCounts)
|
||||||
&& this.typeCounts.equals(that.typeCounts);
|
&& this.tsCounts.equals(that.tsCounts);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -144,7 +144,7 @@ public class FileWithSnapshotFeature implements INode.Feature {
|
||||||
public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile file,
|
public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile file,
|
||||||
FileDiff removed, BlocksMapUpdateInfo collectedBlocks,
|
FileDiff removed, BlocksMapUpdateInfo collectedBlocks,
|
||||||
final List<INode> removedINodes) {
|
final List<INode> removedINodes) {
|
||||||
long oldDiskspace = file.diskspaceConsumed();
|
long oldStoragespace = file.storagespaceConsumed();
|
||||||
|
|
||||||
byte storagePolicyID = file.getStoragePolicyID();
|
byte storagePolicyID = file.getStoragePolicyID();
|
||||||
BlockStoragePolicy bsp = null;
|
BlockStoragePolicy bsp = null;
|
||||||
|
@ -159,7 +159,7 @@ public class FileWithSnapshotFeature implements INode.Feature {
|
||||||
short currentRepl = file.getBlockReplication();
|
short currentRepl = file.getBlockReplication();
|
||||||
if (currentRepl == 0) {
|
if (currentRepl == 0) {
|
||||||
long oldFileSizeNoRep = file.computeFileSize(true, true);
|
long oldFileSizeNoRep = file.computeFileSize(true, true);
|
||||||
oldDiskspace = oldFileSizeNoRep * replication;
|
oldStoragespace = oldFileSizeNoRep * replication;
|
||||||
|
|
||||||
if (bsp != null) {
|
if (bsp != null) {
|
||||||
List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
|
List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
|
||||||
|
@ -170,8 +170,8 @@ public class FileWithSnapshotFeature implements INode.Feature {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (replication > currentRepl) {
|
} else if (replication > currentRepl) {
|
||||||
long oldFileSizeNoRep = file.diskspaceConsumedNoReplication();
|
long oldFileSizeNoRep = file.storagespaceConsumedNoReplication();
|
||||||
oldDiskspace = oldFileSizeNoRep * replication;
|
oldStoragespace = oldFileSizeNoRep * replication;
|
||||||
|
|
||||||
if (bsp != null) {
|
if (bsp != null) {
|
||||||
List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
|
List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
|
||||||
|
@ -197,10 +197,10 @@ public class FileWithSnapshotFeature implements INode.Feature {
|
||||||
getDiffs().combineAndCollectSnapshotBlocks(
|
getDiffs().combineAndCollectSnapshotBlocks(
|
||||||
bsps, file, removed, collectedBlocks, removedINodes);
|
bsps, file, removed, collectedBlocks, removedINodes);
|
||||||
|
|
||||||
long dsDelta = oldDiskspace - file.diskspaceConsumed();
|
long ssDelta = oldStoragespace - file.storagespaceConsumed();
|
||||||
return new QuotaCounts.Builder().
|
return new QuotaCounts.Builder().
|
||||||
spaceCount(dsDelta).
|
storageSpace(ssDelta).
|
||||||
typeCounts(typeSpaces).
|
typeSpaces(typeSpaces).
|
||||||
build();
|
build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -571,7 +571,7 @@ message GetContentSummaryResponseProto {
|
||||||
message SetQuotaRequestProto {
|
message SetQuotaRequestProto {
|
||||||
required string path = 1;
|
required string path = 1;
|
||||||
required uint64 namespaceQuota = 2;
|
required uint64 namespaceQuota = 2;
|
||||||
required uint64 diskspaceQuota = 3;
|
required uint64 storagespaceQuota = 3;
|
||||||
optional StorageTypeProto storageType = 4;
|
optional StorageTypeProto storageType = 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -84,7 +84,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
||||||
.getSpaceConsumed();
|
.getSpaceConsumed();
|
||||||
assertEquals(2, cnt.getNameSpace());
|
assertEquals(2, cnt.getNameSpace());
|
||||||
assertEquals(fileLen * REPLICATION, cnt.getDiskSpace());
|
assertEquals(fileLen * REPLICATION, cnt.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -108,7 +108,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
|
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
|
||||||
.getSpaceConsumed();
|
.getSpaceConsumed();
|
||||||
long ns = quota.getNameSpace();
|
long ns = quota.getNameSpace();
|
||||||
long ds = quota.getDiskSpace();
|
long ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns); // foo and bar
|
assertEquals(2, ns); // foo and bar
|
||||||
assertEquals(currentFileLen * REPLICATION, ds);
|
assertEquals(currentFileLen * REPLICATION, ds);
|
||||||
ContentSummary c = dfs.getContentSummary(foo);
|
ContentSummary c = dfs.getContentSummary(foo);
|
||||||
|
@ -120,7 +120,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
|
|
||||||
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
ns = quota.getNameSpace();
|
ns = quota.getNameSpace();
|
||||||
ds = quota.getDiskSpace();
|
ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns); // foo and bar
|
assertEquals(2, ns); // foo and bar
|
||||||
assertEquals(currentFileLen * REPLICATION, ds);
|
assertEquals(currentFileLen * REPLICATION, ds);
|
||||||
c = dfs.getContentSummary(foo);
|
c = dfs.getContentSummary(foo);
|
||||||
|
@ -132,7 +132,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
|
|
||||||
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
ns = quota.getNameSpace();
|
ns = quota.getNameSpace();
|
||||||
ds = quota.getDiskSpace();
|
ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns); // foo and bar
|
assertEquals(2, ns); // foo and bar
|
||||||
assertEquals(currentFileLen * REPLICATION, ds);
|
assertEquals(currentFileLen * REPLICATION, ds);
|
||||||
c = dfs.getContentSummary(foo);
|
c = dfs.getContentSummary(foo);
|
||||||
|
@ -159,7 +159,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
|
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
|
||||||
.getSpaceConsumed();
|
.getSpaceConsumed();
|
||||||
long ns = quota.getNameSpace();
|
long ns = quota.getNameSpace();
|
||||||
long ds = quota.getDiskSpace();
|
long ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns); // foo and bar
|
assertEquals(2, ns); // foo and bar
|
||||||
assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction
|
assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction
|
||||||
|
|
||||||
|
@ -169,7 +169,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
|
fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
|
||||||
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
ns = quota.getNameSpace();
|
ns = quota.getNameSpace();
|
||||||
ds = quota.getDiskSpace();
|
ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns);
|
assertEquals(2, ns);
|
||||||
assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
|
assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
|
||||||
|
|
||||||
|
@ -178,7 +178,7 @@ public class TestDiskspaceQuotaUpdate {
|
||||||
|
|
||||||
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
ns = quota.getNameSpace();
|
ns = quota.getNameSpace();
|
||||||
ds = quota.getDiskSpace();
|
ds = quota.getStorageSpace();
|
||||||
assertEquals(2, ns); // foo and bar
|
assertEquals(2, ns); // foo and bar
|
||||||
assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
|
assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
|
||||||
}
|
}
|
||||||
|
|
|
@ -424,14 +424,14 @@ public class TestQuotaByStorageType {
|
||||||
QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
||||||
.getSpaceConsumed();
|
.getSpaceConsumed();
|
||||||
assertEquals(2, cnt.getNameSpace());
|
assertEquals(2, cnt.getNameSpace());
|
||||||
assertEquals(fileLen * REPLICATION, cnt.getDiskSpace());
|
assertEquals(fileLen * REPLICATION, cnt.getStorageSpace());
|
||||||
|
|
||||||
dfs.delete(createdFile, true);
|
dfs.delete(createdFile, true);
|
||||||
|
|
||||||
QuotaCounts cntAfterDelete = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
QuotaCounts cntAfterDelete = fnode.asDirectory().getDirectoryWithQuotaFeature()
|
||||||
.getSpaceConsumed();
|
.getSpaceConsumed();
|
||||||
assertEquals(1, cntAfterDelete.getNameSpace());
|
assertEquals(1, cntAfterDelete.getNameSpace());
|
||||||
assertEquals(0, cntAfterDelete.getDiskSpace());
|
assertEquals(0, cntAfterDelete.getStorageSpace());
|
||||||
|
|
||||||
// Validate the computeQuotaUsage()
|
// Validate the computeQuotaUsage()
|
||||||
QuotaCounts counts = new QuotaCounts.Builder().build();
|
QuotaCounts counts = new QuotaCounts.Builder().build();
|
||||||
|
@ -439,7 +439,7 @@ public class TestQuotaByStorageType {
|
||||||
assertEquals(fnode.dumpTreeRecursively().toString(), 1,
|
assertEquals(fnode.dumpTreeRecursively().toString(), 1,
|
||||||
counts.getNameSpace());
|
counts.getNameSpace());
|
||||||
assertEquals(fnode.dumpTreeRecursively().toString(), 0,
|
assertEquals(fnode.dumpTreeRecursively().toString(), 0,
|
||||||
counts.getDiskSpace());
|
counts.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -1200,13 +1200,13 @@ public class TestRenameWithSnapshots {
|
||||||
// make sure the whole referred subtree has been destroyed
|
// make sure the whole referred subtree has been destroyed
|
||||||
QuotaCounts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
|
QuotaCounts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
assertEquals(3, q.getNameSpace());
|
assertEquals(3, q.getNameSpace());
|
||||||
assertEquals(0, q.getDiskSpace());
|
assertEquals(0, q.getStorageSpace());
|
||||||
|
|
||||||
hdfs.deleteSnapshot(sdir1, "s1");
|
hdfs.deleteSnapshot(sdir1, "s1");
|
||||||
restartClusterAndCheckImage(true);
|
restartClusterAndCheckImage(true);
|
||||||
q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
|
q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
|
||||||
assertEquals(3, q.getNameSpace());
|
assertEquals(3, q.getNameSpace());
|
||||||
assertEquals(0, q.getDiskSpace());
|
assertEquals(0, q.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1602,7 +1602,7 @@ public class TestRenameWithSnapshots {
|
||||||
assertTrue(dir2Node.isSnapshottable());
|
assertTrue(dir2Node.isSnapshottable());
|
||||||
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
|
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
|
||||||
assertEquals(2, counts.getNameSpace());
|
assertEquals(2, counts.getNameSpace());
|
||||||
assertEquals(0, counts.getDiskSpace());
|
assertEquals(0, counts.getStorageSpace());
|
||||||
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
|
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
|
||||||
.getChildrenList(Snapshot.CURRENT_STATE_ID));
|
.getChildrenList(Snapshot.CURRENT_STATE_ID));
|
||||||
assertEquals(1, childrenList.size());
|
assertEquals(1, childrenList.size());
|
||||||
|
@ -1676,7 +1676,7 @@ public class TestRenameWithSnapshots {
|
||||||
assertTrue(dir2Node.isSnapshottable());
|
assertTrue(dir2Node.isSnapshottable());
|
||||||
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
|
QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
|
||||||
assertEquals(3, counts.getNameSpace());
|
assertEquals(3, counts.getNameSpace());
|
||||||
assertEquals(0, counts.getDiskSpace());
|
assertEquals(0, counts.getStorageSpace());
|
||||||
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
|
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
|
||||||
.getChildrenList(Snapshot.CURRENT_STATE_ID));
|
.getChildrenList(Snapshot.CURRENT_STATE_ID));
|
||||||
assertEquals(1, childrenList.size());
|
assertEquals(1, childrenList.size());
|
||||||
|
@ -1793,7 +1793,7 @@ public class TestRenameWithSnapshots {
|
||||||
QuotaCounts counts = dir2Node.computeQuotaUsage(
|
QuotaCounts counts = dir2Node.computeQuotaUsage(
|
||||||
fsdir.getBlockStoragePolicySuite());
|
fsdir.getBlockStoragePolicySuite());
|
||||||
assertEquals(4, counts.getNameSpace());
|
assertEquals(4, counts.getNameSpace());
|
||||||
assertEquals(BLOCKSIZE * REPL * 2, counts.getDiskSpace());
|
assertEquals(BLOCKSIZE * REPL * 2, counts.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -207,13 +207,13 @@ public class TestSnapshotDeletion {
|
||||||
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
|
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
|
||||||
q.getNameSpace());
|
q.getNameSpace());
|
||||||
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
|
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
|
||||||
q.getDiskSpace());
|
q.getStorageSpace());
|
||||||
QuotaCounts counts = new QuotaCounts.Builder().build();
|
QuotaCounts counts = new QuotaCounts.Builder().build();
|
||||||
dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
|
dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
|
||||||
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
|
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
|
||||||
counts.getNameSpace());
|
counts.getNameSpace());
|
||||||
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
|
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
|
||||||
counts.getDiskSpace());
|
counts.getStorageSpace());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
Loading…
Reference in New Issue