HDFS-11383. Intern strings in BlockLocation and ExtendedBlock. Contributed by Misha Dmitriev.
This commit is contained in:
parent
219f4c199e
commit
7101477e47
|
@ -22,6 +22,7 @@ import java.io.Serializable;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
import org.apache.hadoop.util.StringInterner;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Represents the network location of a block, information about the hosts
|
* Represents the network location of a block, information about the hosts
|
||||||
|
@ -114,27 +115,27 @@ public class BlockLocation implements Serializable {
|
||||||
if (names == null) {
|
if (names == null) {
|
||||||
this.names = EMPTY_STR_ARRAY;
|
this.names = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.names = names;
|
this.names = StringInterner.internStringsInArray(names);
|
||||||
}
|
}
|
||||||
if (hosts == null) {
|
if (hosts == null) {
|
||||||
this.hosts = EMPTY_STR_ARRAY;
|
this.hosts = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.hosts = hosts;
|
this.hosts = StringInterner.internStringsInArray(hosts);
|
||||||
}
|
}
|
||||||
if (cachedHosts == null) {
|
if (cachedHosts == null) {
|
||||||
this.cachedHosts = EMPTY_STR_ARRAY;
|
this.cachedHosts = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.cachedHosts = cachedHosts;
|
this.cachedHosts = StringInterner.internStringsInArray(cachedHosts);
|
||||||
}
|
}
|
||||||
if (topologyPaths == null) {
|
if (topologyPaths == null) {
|
||||||
this.topologyPaths = EMPTY_STR_ARRAY;
|
this.topologyPaths = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.topologyPaths = topologyPaths;
|
this.topologyPaths = StringInterner.internStringsInArray(topologyPaths);
|
||||||
}
|
}
|
||||||
if (storageIds == null) {
|
if (storageIds == null) {
|
||||||
this.storageIds = EMPTY_STR_ARRAY;
|
this.storageIds = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.storageIds = storageIds;
|
this.storageIds = StringInterner.internStringsInArray(storageIds);
|
||||||
}
|
}
|
||||||
if (storageTypes == null) {
|
if (storageTypes == null) {
|
||||||
this.storageTypes = EMPTY_STORAGE_TYPE_ARRAY;
|
this.storageTypes = EMPTY_STORAGE_TYPE_ARRAY;
|
||||||
|
@ -238,7 +239,7 @@ public class BlockLocation implements Serializable {
|
||||||
if (hosts == null) {
|
if (hosts == null) {
|
||||||
this.hosts = EMPTY_STR_ARRAY;
|
this.hosts = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.hosts = hosts;
|
this.hosts = StringInterner.internStringsInArray(hosts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -249,7 +250,7 @@ public class BlockLocation implements Serializable {
|
||||||
if (cachedHosts == null) {
|
if (cachedHosts == null) {
|
||||||
this.cachedHosts = EMPTY_STR_ARRAY;
|
this.cachedHosts = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.cachedHosts = cachedHosts;
|
this.cachedHosts = StringInterner.internStringsInArray(cachedHosts);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -260,7 +261,7 @@ public class BlockLocation implements Serializable {
|
||||||
if (names == null) {
|
if (names == null) {
|
||||||
this.names = EMPTY_STR_ARRAY;
|
this.names = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.names = names;
|
this.names = StringInterner.internStringsInArray(names);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -271,7 +272,7 @@ public class BlockLocation implements Serializable {
|
||||||
if (topologyPaths == null) {
|
if (topologyPaths == null) {
|
||||||
this.topologyPaths = EMPTY_STR_ARRAY;
|
this.topologyPaths = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.topologyPaths = topologyPaths;
|
this.topologyPaths = StringInterner.internStringsInArray(topologyPaths);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -279,7 +280,7 @@ public class BlockLocation implements Serializable {
|
||||||
if (storageIds == null) {
|
if (storageIds == null) {
|
||||||
this.storageIds = EMPTY_STR_ARRAY;
|
this.storageIds = EMPTY_STR_ARRAY;
|
||||||
} else {
|
} else {
|
||||||
this.storageIds = storageIds;
|
this.storageIds = StringInterner.internStringsInArray(storageIds);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,8 +25,9 @@ import com.google.common.collect.Interner;
|
||||||
import com.google.common.collect.Interners;
|
import com.google.common.collect.Interners;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Provides equivalent behavior to String.intern() to optimize performance,
|
* Provides string interning utility methods. For weak interning,
|
||||||
* whereby does not consume memory in the permanent generation.
|
* we use the standard String.intern() call, that performs very well
|
||||||
|
* (no problems with PermGen overflowing, etc.) starting from JDK 7.
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Public
|
@InterfaceAudience.Public
|
||||||
@InterfaceStability.Stable
|
@InterfaceStability.Stable
|
||||||
|
@ -35,20 +36,9 @@ public class StringInterner {
|
||||||
/**
|
/**
|
||||||
* Retains a strong reference to each string instance it has interned.
|
* Retains a strong reference to each string instance it has interned.
|
||||||
*/
|
*/
|
||||||
private final static Interner<String> strongInterner;
|
private final static Interner<String> STRONG_INTERNER =
|
||||||
|
Interners.newStrongInterner();
|
||||||
/**
|
|
||||||
* Retains a weak reference to each string instance it has interned.
|
|
||||||
*/
|
|
||||||
private final static Interner<String> weakInterner;
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static {
|
|
||||||
strongInterner = Interners.newStrongInterner();
|
|
||||||
weakInterner = Interners.newWeakInterner();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Interns and returns a reference to the representative instance
|
* Interns and returns a reference to the representative instance
|
||||||
* for any of a collection of string instances that are equal to each other.
|
* for any of a collection of string instances that are equal to each other.
|
||||||
|
@ -62,7 +52,7 @@ public class StringInterner {
|
||||||
if (sample == null) {
|
if (sample == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return strongInterner.intern(sample);
|
return STRONG_INTERNER.intern(sample);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -78,7 +68,18 @@ public class StringInterner {
|
||||||
if (sample == null) {
|
if (sample == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return weakInterner.intern(sample);
|
return sample.intern();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interns all the strings in the given array in place,
|
||||||
|
* returning the same array.
|
||||||
|
*/
|
||||||
|
public static String[] internStringsInArray(String[] strings) {
|
||||||
|
for (int i = 0; i < strings.length; i++) {
|
||||||
|
strings[i] = weakIntern(strings[i]);
|
||||||
|
}
|
||||||
|
return strings;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs.protocol;
|
package org.apache.hadoop.hdfs.protocol;
|
||||||
|
|
||||||
|
import org.apache.commons.lang.builder.HashCodeBuilder;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@ -42,13 +43,13 @@ public class ExtendedBlock {
|
||||||
}
|
}
|
||||||
|
|
||||||
public ExtendedBlock(String poolId, Block b) {
|
public ExtendedBlock(String poolId, Block b) {
|
||||||
this.poolId = poolId;
|
this.poolId = poolId != null ? poolId.intern() : null;
|
||||||
this.block = b;
|
this.block = b;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ExtendedBlock(final String poolId, final long blkid, final long len,
|
public ExtendedBlock(final String poolId, final long blkid, final long len,
|
||||||
final long genstamp) {
|
final long genstamp) {
|
||||||
this.poolId = poolId;
|
this.poolId = poolId != null ? poolId.intern() : null;
|
||||||
block = new Block(blkid, len, genstamp);
|
block = new Block(blkid, len, genstamp);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,7 +87,7 @@ public class ExtendedBlock {
|
||||||
}
|
}
|
||||||
|
|
||||||
public void set(String poolId, Block blk) {
|
public void set(String poolId, Block blk) {
|
||||||
this.poolId = poolId;
|
this.poolId = poolId != null ? poolId.intern() : null;
|
||||||
this.block = blk;
|
this.block = blk;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -107,13 +108,16 @@ public class ExtendedBlock {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
ExtendedBlock b = (ExtendedBlock)o;
|
ExtendedBlock b = (ExtendedBlock)o;
|
||||||
return b.block.equals(block) && b.poolId.equals(poolId);
|
return b.block.equals(block) &&
|
||||||
|
(b.poolId != null ? b.poolId.equals(poolId) : poolId == null);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // Object
|
@Override // Object
|
||||||
public int hashCode() {
|
public int hashCode() {
|
||||||
int result = 31 + poolId.hashCode();
|
return new HashCodeBuilder(31, 17).
|
||||||
return (31 * result + block.hashCode());
|
append(poolId).
|
||||||
|
append(block).
|
||||||
|
toHashCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override // Object
|
@Override // Object
|
||||||
|
|
Loading…
Reference in New Issue