pair =
+ rpcRequestQueue.poll(maxIdleTime, TimeUnit.MILLISECONDS);
+ if (pair == null || shouldCloseConnection.get()) {
+ continue;
+ }
+ buf = pair.getRight();
+ synchronized (ipcStreams.out) {
+ if (LOG.isDebugEnabled()) {
+ Call call = pair.getLeft();
+ LOG.debug(getName() + "{} sending #{} {}", getName(), call.id,
+ call.rpcRequest);
+ }
+ // RpcRequestHeader + RpcRequest
+ ipcStreams.sendRequest(buf.toByteArray());
+ ipcStreams.flush();
+ }
+ } catch (InterruptedException ie) {
+ // stop this thread
+ return;
+ } catch (IOException e) {
+ // exception at this point would leave the connection in an
+ // unrecoverable state (eg half a call left on the wire).
+ // So, close the connection, killing any outstanding calls
+ markClosed(e);
+ } finally {
+ //the buffer is just an in-memory buffer, but it is still polite to
+ // close early
+ IOUtils.closeStream(buf);
+ }
+ }
+ }
+ }
+
/** Initiates a rpc call by sending the rpc request to the remote server.
- * Note: this is not called from the Connection thread, but by other
- * threads.
+ * Note: this is not called from the current thread, but by another
+ * thread, so that if the current thread is interrupted that the socket
+ * state isn't corrupted with a partially written message.
* @param call - the rpc request
*/
public void sendRpcRequest(final Call call)
@@ -1185,8 +1164,7 @@ public class Client implements AutoCloseable {
}
// Serialize the call to be sent. This is done from the actual
- // caller thread, rather than the sendParamsExecutor thread,
-
+ // caller thread, rather than the rpcRequestThread in the connection,
// so that if the serialization throws an error, it is reported
// properly. This also parallelizes the serialization.
//
@@ -1203,51 +1181,7 @@ public class Client implements AutoCloseable {
final ResponseBuffer buf = new ResponseBuffer();
header.writeDelimitedTo(buf);
RpcWritable.wrap(call.rpcRequest).writeTo(buf);
-
- synchronized (sendRpcRequestLock) {
- Future senderFuture = sendParamsExecutor.submit(new Runnable() {
- @Override
- public void run() {
- try {
- synchronized (ipcStreams.out) {
- if (shouldCloseConnection.get()) {
- return;
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(getName() + " sending #" + call.id
- + " " + call.rpcRequest);
- }
- // RpcRequestHeader + RpcRequest
- ipcStreams.sendRequest(buf.toByteArray());
- ipcStreams.flush();
- }
- } catch (IOException e) {
- // exception at this point would leave the connection in an
- // unrecoverable state (eg half a call left on the wire).
- // So, close the connection, killing any outstanding calls
- markClosed(e);
- } finally {
- //the buffer is just an in-memory buffer, but it is still polite to
- // close early
- IOUtils.closeStream(buf);
- }
- }
- });
-
- try {
- senderFuture.get();
- } catch (ExecutionException e) {
- Throwable cause = e.getCause();
-
- // cause should only be a RuntimeException as the Runnable above
- // catches IOException
- if (cause instanceof RuntimeException) {
- throw (RuntimeException) cause;
- } else {
- throw new RuntimeException("unexpected checked exception", cause);
- }
- }
- }
+ rpcRequestQueue.put(Pair.of(call, buf));
}
/* Receive a response.
@@ -1396,7 +1330,6 @@ public class Client implements AutoCloseable {
CommonConfigurationKeys.IPC_CLIENT_BIND_WILDCARD_ADDR_DEFAULT);
this.clientId = ClientId.getClientId();
- this.sendParamsExecutor = clientExcecutorFactory.refAndGetInstance();
this.maxAsyncCalls = conf.getInt(
CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_KEY,
CommonConfigurationKeys.IPC_CLIENT_ASYNC_CALLS_MAX_DEFAULT);
@@ -1440,6 +1373,7 @@ public class Client implements AutoCloseable {
// wake up all connections
for (Connection conn : connections.values()) {
conn.interrupt();
+ conn.rpcRequestThread.interrupt();
conn.interruptConnectingThread();
}
@@ -1456,7 +1390,6 @@ public class Client implements AutoCloseable {
}
}
}
- clientExcecutorFactory.unrefAndCleanup();
}
/**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 17366eb9569..a79fc2eeb57 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -123,6 +123,7 @@ import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ProtoUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.tracing.Span;
import org.apache.hadoop.tracing.SpanContext;
import org.apache.hadoop.tracing.TraceScope;
@@ -153,6 +154,13 @@ public abstract class Server {
private ExceptionsHandler exceptionsHandler = new ExceptionsHandler();
private Tracer tracer;
private AlignmentContext alignmentContext;
+
+ /**
+ * Allow server to do force Kerberos re-login once after failure irrespective
+ * of the last login time.
+ */
+ private final AtomicBoolean canTryForceLogin = new AtomicBoolean(true);
+
/**
* Logical name of the server used in metrics and monitor.
*/
@@ -1393,8 +1401,7 @@ public abstract class Server {
bind(acceptChannel.socket(), address, backlogLength, conf, portRangeConfig);
//Could be an ephemeral port
this.listenPort = acceptChannel.socket().getLocalPort();
- Thread.currentThread().setName("Listener at " +
- bindAddress + "/" + this.listenPort);
+ LOG.info("Listener at {}:{}", bindAddress, this.listenPort);
// create a selector;
selector= Selector.open();
readers = new Reader[readThreads];
@@ -2207,7 +2214,23 @@ public abstract class Server {
AUDITLOG.warn(AUTH_FAILED_FOR + this.toString() + ":"
+ attemptingUser + " (" + e.getLocalizedMessage()
+ ") with true cause: (" + tce.getLocalizedMessage() + ")");
- throw tce;
+ if (!UserGroupInformation.getLoginUser().isLoginSuccess()) {
+ doKerberosRelogin();
+ try {
+ // try processing message again
+ LOG.debug("Reprocessing sasl message for {}:{} after re-login",
+ this.toString(), attemptingUser);
+ saslResponse = processSaslMessage(saslMessage);
+ AUDITLOG.info("Retry {}{}:{} after failure", AUTH_SUCCESSFUL_FOR,
+ this.toString(), attemptingUser);
+ canTryForceLogin.set(true);
+ } catch (IOException exp) {
+ tce = (IOException) getTrueCause(e);
+ throw tce;
+ }
+ } else {
+ throw tce;
+ }
}
if (saslServer != null && saslServer.isComplete()) {
@@ -3323,6 +3346,26 @@ public abstract class Server {
metricsUpdaterInterval, metricsUpdaterInterval, TimeUnit.MILLISECONDS);
}
+ private synchronized void doKerberosRelogin() throws IOException {
+ if(UserGroupInformation.getLoginUser().isLoginSuccess()){
+ return;
+ }
+ LOG.warn("Initiating re-login from IPC Server");
+ if (canTryForceLogin.compareAndSet(true, false)) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().forceReloginFromKeytab();
+ } else if (UserGroupInformation.isLoginTicketBased()) {
+ UserGroupInformation.getLoginUser().forceReloginFromTicketCache();
+ }
+ } else {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else if (UserGroupInformation.isLoginTicketBased()) {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
public synchronized void addAuxiliaryListener(int auxiliaryPort)
throws IOException {
if (auxiliaryListenerMap == null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
index af5f8521433..f9ab394771d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
@@ -65,7 +65,7 @@ import org.apache.hadoop.util.Timer;
* This class can also be used to coordinate multiple logging points; see
* {@link #record(String, long, double...)} for more details.
*
- *
This class is not thread-safe.
+ *
This class is thread-safe.
*/
public class LogThrottlingHelper {
@@ -192,7 +192,7 @@ public class LogThrottlingHelper {
* @return A LogAction indicating whether or not the caller should write to
* its log.
*/
- public LogAction record(double... values) {
+ public synchronized LogAction record(double... values) {
return record(DEFAULT_RECORDER_NAME, timer.monotonicNow(), values);
}
@@ -244,7 +244,7 @@ public class LogThrottlingHelper {
*
* @see #record(double...)
*/
- public LogAction record(String recorderName, long currentTimeMs,
+ public synchronized LogAction record(String recorderName, long currentTimeMs,
double... values) {
if (primaryRecorderName == null) {
primaryRecorderName = recorderName;
@@ -262,9 +262,15 @@ public class LogThrottlingHelper {
if (primaryRecorderName.equals(recorderName) &&
currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) {
lastLogTimestampMs = currentTimeMs;
- for (LoggingAction log : currentLogs.values()) {
- log.setShouldLog();
- }
+ currentLogs.replaceAll((key, log) -> {
+ LoggingAction newLog = log;
+ if (log.hasLogged()) {
+ // create a fresh log since the old one has already been logged
+ newLog = new LoggingAction(log.getValueCount());
+ }
+ newLog.setShouldLog();
+ return newLog;
+ });
}
if (currentLog.shouldLog()) {
currentLog.setHasLogged();
@@ -281,7 +287,7 @@ public class LogThrottlingHelper {
* @param idx The index value.
* @return The summary information.
*/
- public SummaryStatistics getCurrentStats(String recorderName, int idx) {
+ public synchronized SummaryStatistics getCurrentStats(String recorderName, int idx) {
LoggingAction currentLog = currentLogs.get(recorderName);
if (currentLog != null) {
return currentLog.getStats(idx);
@@ -308,6 +314,13 @@ public class LogThrottlingHelper {
}
}
+ @VisibleForTesting
+ public synchronized void reset() {
+ primaryRecorderName = null;
+ currentLogs.clear();
+ lastLogTimestampMs = Long.MIN_VALUE;
+ }
+
/**
* A standard log action which keeps track of all of the values which have
* been logged. This is also used for internal bookkeeping via its private
@@ -357,6 +370,10 @@ public class LogThrottlingHelper {
hasLogged = true;
}
+ private int getValueCount() {
+ return stats.length;
+ }
+
private void recordValues(double... values) {
if (values.length != stats.length) {
throw new IllegalArgumentException("received " + values.length +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
index 8837c02b99d..6c5a71a708f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
@@ -280,7 +280,6 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
}
return sink;
}
- allSinks.put(name, sink);
if (config != null) {
registerSink(name, description, sink);
}
@@ -301,6 +300,7 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
? newSink(name, desc, sink, conf)
: newSink(name, desc, sink, config.subset(SINK_KEY));
sinks.put(name, sa);
+ allSinks.put(name, sink);
sa.start();
LOG.info("Registered sink "+ name);
}
@@ -508,6 +508,7 @@ public class MetricsSystemImpl extends MetricsSystem implements MetricsSource {
conf.getString(DESC_KEY, sinkName), conf);
sa.start();
sinks.put(sinkName, sa);
+ allSinks.put(sinkName, sa.sink());
} catch (Exception e) {
LOG.warn("Error creating sink '"+ sinkName +"'", e);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
index 6a52bf382df..126601fcbb6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableGaugeFloat.java
@@ -69,7 +69,7 @@ public class MutableGaugeFloat extends MutableGauge {
private void incr(float delta) {
while (true) {
- float current = value.get();
+ float current = Float.intBitsToFloat(value.get());
float next = current + delta;
if (compareAndSet(current, next)) {
setChanged();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
index f2e072545ad..b130aa6ada3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableStat.java
@@ -140,14 +140,14 @@ public class MutableStat extends MutableMetric {
if (all || changed()) {
numSamples += intervalStat.numSamples();
builder.addCounter(numInfo, numSamples)
- .addGauge(avgInfo, lastStat().mean());
+ .addGauge(avgInfo, intervalStat.mean());
if (extended) {
- builder.addGauge(stdevInfo, lastStat().stddev())
- .addGauge(iMinInfo, lastStat().min())
- .addGauge(iMaxInfo, lastStat().max())
+ builder.addGauge(stdevInfo, intervalStat.stddev())
+ .addGauge(iMinInfo, intervalStat.min())
+ .addGauge(iMaxInfo, intervalStat.max())
.addGauge(minInfo, minMax.min())
.addGauge(maxInfo, minMax.max())
- .addGauge(iNumInfo, lastStat().numSamples());
+ .addGauge(iNumInfo, intervalStat.numSamples());
}
if (changed()) {
if (numSamples > 0) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index c28471a3bda..49fd9194e5a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.HashBiMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.util.Shell.bashQuote;
+
/**
* A simple shell-based implementation of {@link IdMappingServiceProvider}
* Map id to user name or group name. It does update every 15 minutes. Only a
@@ -472,26 +474,27 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
boolean updated = false;
updateStaticMapping();
+ String name2 = bashQuote(name);
if (OS.startsWith("Linux") || OS.equals("SunOS") || OS.contains("BSD")) {
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
- getName2IdCmdNIX(name, true), ":",
+ getName2IdCmdNIX(name2, true), ":",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
- getName2IdCmdNIX(name, false), ":",
+ getName2IdCmdNIX(name2, false), ":",
staticMapping.uidMapping);
}
} else {
// Mac
if (isGrp) {
updated = updateMapInternal(gidNameMap, "group",
- getName2IdCmdMac(name, true), "\\s+",
+ getName2IdCmdMac(name2, true), "\\s+",
staticMapping.gidMapping);
} else {
updated = updateMapInternal(uidNameMap, "user",
- getName2IdCmdMac(name, false), "\\s+",
+ getName2IdCmdMac(name2, false), "\\s+",
staticMapping.uidMapping);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 9671d8da38f..8a5a0ee234f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -529,6 +529,18 @@ public class UserGroupInformation {
user.setLogin(login);
}
+ /** This method checks for a successful Kerberos login
+ * and returns true by default if it is not using Kerberos.
+ *
+ * @return true on successful login
+ */
+ public boolean isLoginSuccess() {
+ LoginContext login = user.getLogin();
+ return (login instanceof HadoopLoginContext)
+ ? ((HadoopLoginContext) login).isLoginSuccess()
+ : true;
+ }
+
/**
* Set the last login time for logged in user
* @param loginTime the number of milliseconds since the beginning of time
@@ -1276,6 +1288,23 @@ public class UserGroupInformation {
relogin(login, ignoreLastLoginTime);
}
+ /**
+ * Force re-Login a user in from the ticket cache irrespective of the last
+ * login time. This method assumes that login had happened already. The
+ * Subject field of this UserGroupInformation object is updated to have the
+ * new credentials.
+ *
+ * @throws IOException
+ * raised on errors performing I/O.
+ * @throws KerberosAuthException
+ * on a failure
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ public void forceReloginFromTicketCache() throws IOException {
+ reloginFromTicketCache(true);
+ }
+
/**
* Re-Login a user in from the ticket cache. This
* method assumes that login had happened already.
@@ -1287,6 +1316,11 @@ public class UserGroupInformation {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public void reloginFromTicketCache() throws IOException {
+ reloginFromTicketCache(false);
+ }
+
+ private void reloginFromTicketCache(boolean ignoreLastLoginTime)
+ throws IOException {
if (!shouldRelogin() || !isFromTicket()) {
return;
}
@@ -1294,7 +1328,7 @@ public class UserGroupInformation {
if (login == null) {
throw new KerberosAuthException(MUST_FIRST_LOGIN);
}
- relogin(login, false);
+ relogin(login, ignoreLastLoginTime);
}
private void relogin(HadoopLoginContext login, boolean ignoreLastLoginTime)
@@ -2083,6 +2117,11 @@ public class UserGroupInformation {
this.conf = conf;
}
+ /** Get the login status. */
+ public boolean isLoginSuccess() {
+ return isLoggedIn.get();
+ }
+
String getAppName() {
return appName;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/package-info.java
index e1060e2196d..3c75a2427d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Support for service-level authorization.
+ */
@InterfaceAudience.Public
@InterfaceStability.Evolving
package org.apache.hadoop.security.authorize;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/package-info.java
index 8e9398eb679..a58b3cdcfb9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Filters for HTTP service security.
+ */
@InterfaceAudience.Public
@InterfaceStability.Evolving
package org.apache.hadoop.security.http;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
index fe3233d848d..5ab38aa7420 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.hadoop.util.PlatformName.JAVA_VENDOR_NAME;
+import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
@@ -102,11 +102,11 @@ public class SSLFactory implements ConnectionConfigurator {
"ssl.server.exclude.cipher.list";
public static final String KEY_MANAGER_SSLCERTIFICATE =
- JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+ IBM_JAVA ? "ibmX509" :
KeyManagerFactory.getDefaultAlgorithm();
public static final String TRUST_MANAGER_SSLCERTIFICATE =
- JAVA_VENDOR_NAME.contains("IBM") ? "ibmX509" :
+ IBM_JAVA ? "ibmX509" :
TrustManagerFactory.getDefaultAlgorithm();
public static final String KEYSTORES_FACTORY_CLASS_KEY =
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/package-info.java
index c85f967ab67..0b3b8c46944 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * ZooKeeper secret manager for TokenIdentifiers and DelegationKeys.
+ */
@InterfaceAudience.LimitedPrivate({"HBase", "HDFS", "MapReduce"})
@InterfaceStability.Evolving
package org.apache.hadoop.security.token.delegation;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/package-info.java
index e015056b43e..cdf4e61050d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Support for delegation tokens.
+ */
@InterfaceAudience.Public
@InterfaceStability.Evolving
package org.apache.hadoop.security.token;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/package-info.java
index 37164855499..81409382648 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/package-info.java
@@ -15,6 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Support for services.
+ */
@InterfaceAudience.Public
package org.apache.hadoop.service;
import org.apache.hadoop.classification.InterfaceAudience;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index 972bbff4cfd..4e8a9c9b275 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -108,7 +108,7 @@ public class ApplicationClassLoader extends URLClassLoader {
throws MalformedURLException {
List urls = new ArrayList();
for (String element : classpath.split(File.pathSeparator)) {
- if (element.endsWith("/*")) {
+ if (element.endsWith(File.separator + "*")) {
List jars = FileUtil.getJarsInDirectory(element);
if (!jars.isEmpty()) {
for (Path jar: jars) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index d94668356e2..300f8145c31 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -147,8 +147,8 @@ public class HostsFileReader {
String filename, InputStream fileInputStream, Map map)
throws IOException {
Document dom;
- DocumentBuilderFactory builder = DocumentBuilderFactory.newInstance();
try {
+ DocumentBuilderFactory builder = XMLUtils.newSecureDocumentBuilderFactory();
DocumentBuilder db = builder.newDocumentBuilder();
dom = db.parse(fileInputStream);
// Examples:
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
index 18f6ccfdb17..c99290bc3d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
@@ -44,12 +44,7 @@ public class InstrumentedReadLock extends InstrumentedLock {
* there can be multiple threads that hold the read lock concurrently.
*/
private final ThreadLocal readLockHeldTimeStamp =
- new ThreadLocal() {
- @Override
- protected Long initialValue() {
- return Long.MAX_VALUE;
- };
- };
+ ThreadLocal.withInitial(() -> Long.MAX_VALUE);
public InstrumentedReadLock(String name, Logger logger,
ReentrantReadWriteLock readWriteLock,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
index 667b1ca6a4b..4637b5efe53 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedWriteLock.java
@@ -37,6 +37,9 @@ import org.slf4j.Logger;
@InterfaceStability.Unstable
public class InstrumentedWriteLock extends InstrumentedLock {
+ private final ReentrantReadWriteLock readWriteLock;
+ private volatile long writeLockHeldTimeStamp = 0;
+
public InstrumentedWriteLock(String name, Logger logger,
ReentrantReadWriteLock readWriteLock,
long minLoggingGapMs, long lockWarningThresholdMs) {
@@ -50,5 +53,28 @@ public class InstrumentedWriteLock extends InstrumentedLock {
long minLoggingGapMs, long lockWarningThresholdMs, Timer clock) {
super(name, logger, readWriteLock.writeLock(), minLoggingGapMs,
lockWarningThresholdMs, clock);
+ this.readWriteLock = readWriteLock;
+ }
+
+ @Override
+ public void unlock() {
+ boolean needReport = readWriteLock.getWriteHoldCount() == 1;
+ long localWriteReleaseTime = getTimer().monotonicNow();
+ long localWriteAcquireTime = writeLockHeldTimeStamp;
+ getLock().unlock();
+ if (needReport) {
+ writeLockHeldTimeStamp = 0;
+ check(localWriteAcquireTime, localWriteReleaseTime, true);
+ }
+ }
+
+ /**
+ * Starts timing for the instrumented write lock.
+ */
+ @Override
+ protected void startLockTiming() {
+ if (readWriteLock.getWriteHoldCount() == 1) {
+ writeLockHeldTimeStamp = getTimer().monotonicNow();
+ }
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
index 051e2680bc3..1383a7fafe7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LightWeightResizableGSet.java
@@ -33,7 +33,7 @@ import java.util.function.Consumer;
*
* This class does not support null element.
*
- * This class is not thread safe.
+ * This class is thread safe.
*
* @param Key type for looking up the elements
* @param Element type, which must be
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 65978f3c5f5..91868365b13 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -146,7 +146,8 @@ public abstract class Shell {
* @param arg the argument to quote
* @return the quoted string
*/
- static String bashQuote(String arg) {
+ @InterfaceAudience.Private
+ public static String bashQuote(String arg) {
StringBuilder buffer = new StringBuilder(arg.length() + 2);
buffer.append('\'')
.append(arg.replace("'", "'\\''"))
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
index ea835023e86..31fe3c6377b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
@@ -93,6 +93,10 @@ public class VersionInfo {
return info.getProperty("protocVersion", "Unknown");
}
+ protected String _getCompilePlatform() {
+ return info.getProperty("compilePlatform", "Unknown");
+ }
+
private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
/**
* Get the Hadoop version.
@@ -167,12 +171,21 @@ public class VersionInfo {
return COMMON_VERSION_INFO._getProtocVersion();
}
+ /**
+ * Returns the OS platform used for the build.
+ * @return the OS platform
+ */
+ public static String getCompilePlatform() {
+ return COMMON_VERSION_INFO._getCompilePlatform();
+ }
+
public static void main(String[] args) {
LOG.debug("version: "+ getVersion());
System.out.println("Hadoop " + getVersion());
System.out.println("Source code repository " + getUrl() + " -r " +
getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
+ System.out.println("Compiled on platform " + getCompilePlatform());
System.out.println("Compiled with protoc " + getProtocVersion());
System.out.println("From source with checksum " + getSrcChecksum());
System.out.println("This command was run using " +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
index e2b9e414ad3..8a5d2f36615 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/XMLUtils.java
@@ -18,13 +18,23 @@
package org.apache.hadoop.util;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParserFactory;
import javax.xml.transform.*;
+import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.stream.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.SAXException;
+
import java.io.*;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* General xml utilities.
@@ -33,6 +43,28 @@ import java.io.*;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class XMLUtils {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(XMLUtils.class);
+
+ public static final String DISALLOW_DOCTYPE_DECL =
+ "http://apache.org/xml/features/disallow-doctype-decl";
+ public static final String LOAD_EXTERNAL_DECL =
+ "http://apache.org/xml/features/nonvalidating/load-external-dtd";
+ public static final String EXTERNAL_GENERAL_ENTITIES =
+ "http://xml.org/sax/features/external-general-entities";
+ public static final String EXTERNAL_PARAMETER_ENTITIES =
+ "http://xml.org/sax/features/external-parameter-entities";
+ public static final String CREATE_ENTITY_REF_NODES =
+ "http://apache.org/xml/features/dom/create-entity-ref-nodes";
+ public static final String VALIDATION =
+ "http://xml.org/sax/features/validation";
+
+ private static final AtomicBoolean CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_DTD =
+ new AtomicBoolean(true);
+ private static final AtomicBoolean CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_STYLESHEET =
+ new AtomicBoolean(true);
+
/**
* Transform input xml given a stylesheet.
*
@@ -49,7 +81,7 @@ public class XMLUtils {
)
throws TransformerConfigurationException, TransformerException {
// Instantiate a TransformerFactory
- TransformerFactory tFactory = TransformerFactory.newInstance();
+ TransformerFactory tFactory = newSecureTransformerFactory();
// Use the TransformerFactory to process the
// stylesheet and generate a Transformer
@@ -61,4 +93,118 @@ public class XMLUtils {
// and send the output to a Result object.
transformer.transform(new StreamSource(xml), new StreamResult(out));
}
+
+ /**
+ * This method should be used if you need a {@link DocumentBuilderFactory}. Use this method
+ * instead of {@link DocumentBuilderFactory#newInstance()}. The factory that is returned has
+ * secure configuration enabled.
+ *
+ * @return a {@link DocumentBuilderFactory} with secure configuration enabled
+ * @throws ParserConfigurationException if the {@code JAXP} parser does not support the
+ * secure configuration
+ */
+ public static DocumentBuilderFactory newSecureDocumentBuilderFactory()
+ throws ParserConfigurationException {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ dbf.setFeature(DISALLOW_DOCTYPE_DECL, true);
+ dbf.setFeature(LOAD_EXTERNAL_DECL, false);
+ dbf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+ dbf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+ dbf.setFeature(CREATE_ENTITY_REF_NODES, false);
+ return dbf;
+ }
+
+ /**
+ * This method should be used if you need a {@link SAXParserFactory}. Use this method
+ * instead of {@link SAXParserFactory#newInstance()}. The factory that is returned has
+ * secure configuration enabled.
+ *
+ * @return a {@link SAXParserFactory} with secure configuration enabled
+ * @throws ParserConfigurationException if the {@code JAXP} parser does not support the
+ * secure configuration
+ * @throws SAXException if there are another issues when creating the factory
+ */
+ public static SAXParserFactory newSecureSAXParserFactory()
+ throws SAXException, ParserConfigurationException {
+ SAXParserFactory spf = SAXParserFactory.newInstance();
+ spf.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ spf.setFeature(DISALLOW_DOCTYPE_DECL, true);
+ spf.setFeature(LOAD_EXTERNAL_DECL, false);
+ spf.setFeature(EXTERNAL_GENERAL_ENTITIES, false);
+ spf.setFeature(EXTERNAL_PARAMETER_ENTITIES, false);
+ return spf;
+ }
+
+ /**
+ * This method should be used if you need a {@link TransformerFactory}. Use this method
+ * instead of {@link TransformerFactory#newInstance()}. The factory that is returned has
+ * secure configuration enabled.
+ *
+ * @return a {@link TransformerFactory} with secure configuration enabled
+ * @throws TransformerConfigurationException if the {@code JAXP} transformer does not
+ * support the secure configuration
+ */
+ public static TransformerFactory newSecureTransformerFactory()
+ throws TransformerConfigurationException {
+ TransformerFactory trfactory = TransformerFactory.newInstance();
+ trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ setOptionalSecureTransformerAttributes(trfactory);
+ return trfactory;
+ }
+
+ /**
+ * This method should be used if you need a {@link SAXTransformerFactory}. Use this method
+ * instead of {@link SAXTransformerFactory#newInstance()}. The factory that is returned has
+ * secure configuration enabled.
+ *
+ * @return a {@link SAXTransformerFactory} with secure configuration enabled
+ * @throws TransformerConfigurationException if the {@code JAXP} transformer does not
+ * support the secure configuration
+ */
+ public static SAXTransformerFactory newSecureSAXTransformerFactory()
+ throws TransformerConfigurationException {
+ SAXTransformerFactory trfactory = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
+ trfactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true);
+ setOptionalSecureTransformerAttributes(trfactory);
+ return trfactory;
+ }
+
+ /**
+ * These attributes are recommended for maximum security but some JAXP transformers do
+ * not support them. If at any stage, we fail to set these attributes, then we won't try again
+ * for subsequent transformers.
+ *
+ * @param transformerFactory to update
+ */
+ private static void setOptionalSecureTransformerAttributes(
+ TransformerFactory transformerFactory) {
+ bestEffortSetAttribute(transformerFactory, CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_DTD,
+ XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ bestEffortSetAttribute(transformerFactory, CAN_SET_TRANSFORMER_ACCESS_EXTERNAL_STYLESHEET,
+ XMLConstants.ACCESS_EXTERNAL_STYLESHEET, "");
+ }
+
+ /**
+ * Set an attribute value on a {@link TransformerFactory}. If the TransformerFactory
+ * does not support the attribute, the method just returns false and
+ * logs the issue at debug level.
+ *
+ * @param transformerFactory to update
+ * @param flag that indicates whether to do the update and the flag can be set to
+ * false if an update fails
+ * @param name of the attribute to set
+ * @param value to set on the attribute
+ */
+ static void bestEffortSetAttribute(TransformerFactory transformerFactory, AtomicBoolean flag,
+ String name, Object value) {
+ if (flag.get()) {
+ try {
+ transformerFactory.setAttribute(name, value);
+ } catch (Throwable t) {
+ flag.set(false);
+ LOG.debug("Issue setting TransformerFactory attribute {}: {}", name, t.toString());
+ }
+ }
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/package-info.java
index 2effb65872e..871005adc0c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/package-info.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/package-info.java
@@ -1,5 +1,4 @@
/*
- * *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -15,9 +14,11 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- * /
*/
+/**
+ * Support for concurrent execution.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.util.concurrent;
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
index 6f8558b8d4f..0f075c8139a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
+++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
@@ -24,3 +24,4 @@ date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
protocVersion=${hadoop.protobuf.version}
+compilePlatform=${os.detected.classifier}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 17cd228dc1b..e18a50c72e8 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1094,14 +1094,6 @@
-
- fs.viewfs.overload.scheme.target.swift.impl
- org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
- The SwiftNativeFileSystem for view file system overload scheme
- when child file system and ViewFSOverloadScheme's schemes are swift.
-
-
-
fs.viewfs.overload.scheme.target.oss.impl
org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
@@ -1211,12 +1203,6 @@
File space usage statistics refresh interval in msec.
-
- fs.swift.impl
- org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem
- The implementation class of the OpenStack Swift Filesystem
-
-
fs.automatic.close
true
@@ -2180,6 +2166,12 @@ The switch to turn S3A auditing on or off.
The AbstractFileSystem for gs: uris.
+
+ fs.azure.enable.readahead
+ true
+ Enabled readahead/prefetching in AbfsInputStream.
+
+
io.seqfile.compress.blocksize
1000000
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index 4f76979ea6a..9095d6f9890 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -22,7 +22,17 @@ Purpose
This document describes how to install and configure Hadoop clusters ranging from a few nodes to extremely large clusters with thousands of nodes. To play with Hadoop, you may first want to install it on a single machine (see [Single Node Setup](./SingleCluster.html)).
-This document does not cover advanced topics such as [Security](./SecureMode.html) or High Availability.
+This document does not cover advanced topics such as High Availability.
+
+*Important*: all production Hadoop clusters use Kerberos to authenticate callers
+and secure access to HDFS data as well as restriction access to computation
+services (YARN etc.).
+
+These instructions do not cover integration with any Kerberos services,
+-everyone bringing up a production cluster should include connecting to their
+organisation's Kerberos infrastructure as a key part of the deployment.
+
+See [Security](./SecureMode.html) for details on how to secure a cluster.
Prerequisites
-------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index 281e42dad88..a00feb039d8 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -208,7 +208,8 @@ The following table lists the configuration property names that are deprecated i
| mapred.task.profile.params | mapreduce.task.profile.params |
| mapred.task.profile.reduces | mapreduce.task.profile.reduces |
| mapred.task.timeout | mapreduce.task.timeout |
-| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
+| mapred.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
+| mapreduce.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
| mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum |
| mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin |
| mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin |
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 9a690a8c5cc..451b33d74fa 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -59,7 +59,7 @@ Copies source paths to stdout.
Options
-* The `-ignoreCrc` option disables checkshum verification.
+* The `-ignoreCrc` option disables checksum verification.
Example:
@@ -73,18 +73,19 @@ Returns 0 on success and -1 on error.
checksum
--------
-Usage: `hadoop fs -checksum [-v] URI`
+Usage: `hadoop fs -checksum [-v] URI [URI ...]`
-Returns the checksum information of a file.
+Returns the checksum information of the file(s).
Options
-* The `-v` option displays blocks size for the file.
+* The `-v` option displays blocks size for the file(s).
Example:
* `hadoop fs -checksum hdfs://nn1.example.com/file1`
* `hadoop fs -checksum file:///etc/hosts`
+* `hadoop fs -checksum file:///etc/hosts hdfs://nn1.example.com/file1`
chgrp
-----
@@ -177,7 +178,7 @@ Returns 0 on success and -1 on error.
cp
----
-Usage: `hadoop fs -cp [-f] [-p | -p[topax]] [-t ] [-q ] URI [URI ...] `
+Usage: `hadoop fs -cp [-f] [-p | -p[topax]] [-d] [-t ] [-q ] URI [URI ...] `
Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory.
@@ -187,13 +188,14 @@ Options:
* `-f` : Overwrite the destination if it already exists.
* `-d` : Skip creation of temporary file with the suffix `._COPYING_`.
-* `-p` : Preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag.
+* `-p` : Preserve file attributes [topax] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag.
* `-t ` : Number of threads to be used, default is 1. Useful when copying directories containing more than 1 file.
* `-q ` : Thread pool queue size to be used, default is 1024. It takes effect only when thread count greater than 1.
Example:
* `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2`
+* `hadoop fs -cp -f -d /user/hadoop/file1 /user/hadoop/file2`
* `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
* `hadoop fs -cp -t 5 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
* `hadoop fs -cp -t 10 -q 2048 /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
@@ -403,7 +405,7 @@ Returns 0 on success and non-zero on error.
getmerge
--------
-Usage: `hadoop fs -getmerge [-nl] `
+Usage: `hadoop fs -getmerge [-nl] [-skip-empty-file] `
Takes a source directory and a destination file as input and concatenates files in src into the destination local file. Optionally -nl can be set to enable adding a newline character (LF) at the end of each file.
-skip-empty-file can be used to avoid unwanted newline characters in case of empty files.
@@ -412,6 +414,7 @@ Examples:
* `hadoop fs -getmerge -nl /src /opt/output.txt`
* `hadoop fs -getmerge -nl /src/file1.txt /src/file2.txt /output.txt`
+* `hadoop fs -getmerge -nl -skip-empty-file /src/file1.txt /src/file2.txt /output.txt`
Exit Code:
@@ -852,7 +855,7 @@ Return the help for an individual command.
====================================================
The Hadoop FileSystem shell works with Object Stores such as Amazon S3,
-Azure WASB and OpenStack Swift.
+Azure ABFS and Google GCS.
@@ -972,7 +975,7 @@ this will be in the bucket; the `rm` operation will then take time proportional
to the size of the data. Furthermore, the deleted files will continue to incur
storage costs.
-To avoid this, use the the `-skipTrash` option.
+To avoid this, use the `-skipTrash` option.
```bash
hadoop fs -rm -skipTrash s3a://bucket/dataset
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 04cbd9fedf8..e7d387b1131 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -220,7 +220,7 @@ Each metrics record contains tags such as ProcessName, SessionId, and Hostname a
| `WarmUpEDEKTimeNumOps` | Total number of warming up EDEK |
| `WarmUpEDEKTimeAvgTime` | Average time of warming up EDEK in milliseconds |
| `WarmUpEDEKTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in warming up EDEK in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
-| `ResourceCheckTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of of NameNode resource check latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
+| `ResourceCheckTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of NameNode resource check latency in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
| `EditLogTailTimeNumOps` | Total number of times the standby NameNode tailed the edit log |
| `EditLogTailTimeAvgTime` | Average time (in milliseconds) spent by standby NameNode in tailing edit log |
| `EditLogTailTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in tailing edit logs by standby NameNode in milliseconds (*num* seconds granularity). Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. |
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index ebfc16c1a52..98c3dd2bbb9 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -595,7 +595,7 @@ hadoop kdiag \
--keytab zk.service.keytab --principal zookeeper/devix.example.org@REALM
```
-This attempts to to perform all diagnostics without failing early, load in
+This attempts to perform all diagnostics without failing early, load in
the HDFS and YARN XML resources, require a minimum key length of 1024 bytes,
and log in as the principal `zookeeper/devix.example.org@REALM`, whose key must be in
the keytab `zk.service.keytab`
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
index 8d0a7d195a8..3c8af8fd6e9 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
@@ -26,6 +26,15 @@ Purpose
This document describes how to set up and configure a single-node Hadoop installation so that you can quickly perform simple operations using Hadoop MapReduce and the Hadoop Distributed File System (HDFS).
+
+*Important*: all production Hadoop clusters use Kerberos to authenticate callers
+and secure access to HDFS data as well as restriction access to computation
+services (YARN etc.).
+
+These instructions do not cover integration with any Kerberos services,
+-everyone bringing up a production cluster should include connecting to their
+organisation's Kerberos infrastructure as a key part of the deployment.
+
Prerequisites
-------------
@@ -33,8 +42,6 @@ $H3 Supported Platforms
* GNU/Linux is supported as a development and production platform. Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
-* Windows is also a supported platform but the followings steps are for Linux only. To set up Hadoop on Windows, see [wiki page](http://wiki.apache.org/hadoop/Hadoop2OnWindows).
-
$H3 Required Software
Required software for Linux include:
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 004220c4bed..fafe2819cf6 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -501,7 +501,7 @@ Where
def blocks(FS, p, s, s + l) = a list of the blocks containing data(FS, path)[s:s+l]
-Note that that as `length(FS, f) ` is defined as `0` if `isDir(FS, f)`, the result
+Note that as `length(FS, f) ` is defined as `0` if `isDir(FS, f)`, the result
of `getFileBlockLocations()` on a directory is `[]`
@@ -701,13 +701,13 @@ The behavior of the returned stream is covered in [Output](outputstream.html).
clients creating files with `overwrite==true` to fail if the file is created
by another client between the two tests.
-* S3A, Swift and potentially other Object Stores do not currently change the `FS` state
+* The S3A and potentially other Object Stores connectors not currently change the `FS` state
until the output stream `close()` operation is completed.
This is a significant difference between the behavior of object stores
and that of filesystems, as it allows >1 client to create a file with `overwrite=false`,
and potentially confuse file/directory logic. In particular, using `create()` to acquire
an exclusive lock on a file (whoever creates the file without an error is considered
-the holder of the lock) may not not a safe algorithm to use when working with object stores.
+the holder of the lock) may not be a safe algorithm to use when working with object stores.
* Object stores may create an empty file as a marker when a file is created.
However, object stores with `overwrite=true` semantics may not implement this atomically,
@@ -1225,7 +1225,7 @@ the parent directories of the destination then exist:
There is a check for and rejection if the `parent(dest)` is a file, but
no checks for any other ancestors.
-*Other Filesystems (including Swift) *
+*Other Filesystems*
Other filesystems strictly reject the operation, raising a `FileNotFoundException`
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
index 16a14150ef9..084c0eaff33 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdatainputstreambuilder.md
@@ -167,7 +167,7 @@ rather than just any FS-specific subclass implemented by the implementation
custom subclasses.
This is critical to ensure safe use of the feature: directory listing/
-status serialization/deserialization can result result in the `withFileStatus()`
+status serialization/deserialization can result in the `withFileStatus()`
argument not being the custom subclass returned by the Filesystem instance's
own `getFileStatus()`, `listFiles()`, `listLocatedStatus()` calls, etc.
@@ -686,4 +686,4 @@ public T load(FileSystem fs,
*Note:* : in Hadoop 3.3.2 and earlier, the `withFileStatus(status)` call
required a non-null parameter; this has since been relaxed.
For maximum compatibility across versions, only invoke the method
-when the file status is known to be non-null.
\ No newline at end of file
+when the file status is known to be non-null.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
index 59a93c5887a..ad6d107d06c 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
@@ -228,7 +228,7 @@ Accordingly: *Use if and only if you are confident that the conditions are met.*
### `fs.s3a.create.header` User-supplied header support
-Options with the prefix `fs.s3a.create.header.` will be added to to the
+Options with the prefix `fs.s3a.create.header.` will be added to the
S3 object metadata as "user defined metadata".
This metadata is visible to all applications. It can also be retrieved through the
FileSystem/FileContext `listXAttrs()` and `getXAttrs()` API calls with the prefix `header.`
@@ -236,4 +236,4 @@ FileSystem/FileContext `listXAttrs()` and `getXAttrs()` API calls with the prefi
When an object is renamed, the metadata is propagated the copy created.
It is possible to probe an S3A Filesystem instance for this capability through
-the `hasPathCapability(path, "fs.s3a.create.header")` check.
\ No newline at end of file
+the `hasPathCapability(path, "fs.s3a.create.header")` check.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
index 903d2bb90ff..76782b45409 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/introduction.md
@@ -30,8 +30,8 @@ are places where HDFS diverges from the expected behaviour of a POSIX
filesystem.
The bundled S3A FileSystem clients make Amazon's S3 Object Store ("blobstore")
-accessible through the FileSystem API. The Swift FileSystem driver provides similar
-functionality for the OpenStack Swift blobstore. The Azure WASB and ADL object
+accessible through the FileSystem API.
+The Azure ABFS, WASB and ADL object
storage FileSystems talks to Microsoft's Azure storage. All of these
bind to object stores, which do have different behaviors, especially regarding
consistency guarantees, and atomicity of operations.
@@ -314,10 +314,10 @@ child entries
This specification refers to *Object Stores* in places, often using the
term *Blobstore*. Hadoop does provide FileSystem client classes for some of these
-even though they violate many of the requirements. This is why, although
-Hadoop can read and write data in an object store, the two which Hadoop ships
-with direct support for — Amazon S3 and OpenStack Swift — cannot
-be used as direct replacements for HDFS.
+even though they violate many of the requirements.
+
+Consult the documentation for a specific store to determine its compatibility
+with specific applications and services.
*What is an Object Store?*
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
index 1498d8db2e2..3b486ea3d4e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/outputstream.md
@@ -980,7 +980,7 @@ throw `UnsupportedOperationException`.
### `StreamCapabilities`
Implementors of filesystem clients SHOULD implement the `StreamCapabilities`
-interface and its `hasCapabilities()` method to to declare whether or not
+interface and its `hasCapabilities()` method to declare whether or not
an output streams offer the visibility and durability guarantees of `Syncable`.
Implementors of `StreamCapabilities.hasCapabilities()` MUST NOT declare that
@@ -1013,4 +1013,4 @@ all data to the datanodes.
1. `close()` SHALL return once the guarantees of `hflush()` are met: the data is
visible to others.
-1. For durability guarantees, `hsync()` MUST be called first.
\ No newline at end of file
+1. For durability guarantees, `hsync()` MUST be called first.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
index 4c6fa3ff0f6..53eb9870bc1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/testing.md
@@ -66,55 +66,6 @@ Example:
-
-### swift://
-
-The OpenStack Swift login details must be defined in the file
-`/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml`.
-The standard hadoop-common `contract-test-options.xml` resource file cannot be
-used, as that file does not get included in `hadoop-common-test.jar`.
-
-
-In `/hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml`
-the Swift bucket name must be defined in the property `fs.contract.test.fs.swift`,
-along with the login details for the specific Swift service provider in which the
-bucket is posted.
-
-
-
- fs.contract.test.fs.swift
- swift://swiftbucket.rackspace/
-
-
-
- fs.swift.service.rackspace.auth.url
- https://auth.api.rackspacecloud.com/v2.0/tokens
- Rackspace US (multiregion)
-
-
-
- fs.swift.service.rackspace.username
- this-is-your-username
-
-
-
- fs.swift.service.rackspace.region
- DFW
-
-
-
- fs.swift.service.rackspace.apikey
- ab0bceyoursecretapikeyffef
-
-
-
-
-1. Often the different public cloud Swift infrastructures exhibit different behaviors
-(authentication and throttling in particular). We recommand that testers create
-accounts on as many of these providers as possible and test against each of them.
-1. They can be slow, especially remotely. Remote links are also the most likely
-to make eventual-consistency behaviors visible, which is a mixed benefit.
-
## Testing a new filesystem
The core of adding a new FileSystem to the contract tests is adding a
@@ -228,8 +179,6 @@ Passing all the FileSystem contract tests does not mean that a filesystem can be
* Scalability: does it support files as large as HDFS, or as many in a single directory?
* Durability: do files actually last -and how long for?
-Proof that this is is true is the fact that the Amazon S3 and OpenStack Swift object stores are eventually consistent object stores with non-atomic rename and delete operations. Single threaded test cases are unlikely to see some of the concurrency issues, while consistency is very often only visible in tests that span a datacenter.
-
There are also some specific aspects of the use of the FileSystem API:
* Compatibility with the `hadoop -fs` CLI.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
index e2646e05624..817863027f7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-configuration.md
@@ -143,7 +143,7 @@ too must have this context defined.
### Identifying the system accounts `hadoop.registry.system.acls`
-These are the the accounts which are given full access to the base of the
+These are the accounts which are given full access to the base of the
registry. The Resource Manager needs this option to create the root paths.
Client applications writing to the registry access to the nodes it creates.
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-security.md b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-security.md
index 6317681a716..71c868b557a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-security.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/registry/registry-security.md
@@ -29,7 +29,7 @@ a secure registry:
1. Allow the RM to create per-user regions of the registration space
1. Allow applications belonging to a user to write registry entries
into their part of the space. These may be short-lived or long-lived
-YARN applications, or they may be be static applications.
+YARN applications, or they may be static applications.
1. Prevent other users from writing into another user's part of the registry.
1. Allow system services to register to a `/services` section of the registry.
1. Provide read access to clients of a registry.
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
index ada4cd80e48..f80c62535a1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.XMLUtils;
+
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -34,7 +36,6 @@ import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
import java.io.File;
import java.util.ArrayList;
@@ -76,7 +77,7 @@ public class CLITestHelper {
boolean success = false;
testConfigFile = TEST_CACHE_DATA_DIR + File.separator + testConfigFile;
try {
- SAXParser p = (SAXParserFactory.newInstance()).newSAXParser();
+ SAXParser p = XMLUtils.newSecureSAXParserFactory().newSAXParser();
p.parse(testConfigFile, getConfigParser());
success = true;
} catch (Exception e) {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index c31229ba9fc..74b2f55065d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -135,7 +135,6 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.s3a.impl");
xmlPropsToSkipCompare.
add("fs.viewfs.overload.scheme.target.swebhdfs.impl");
- xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.swift.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.webhdfs.impl");
xmlPropsToSkipCompare.add("fs.viewfs.overload.scheme.target.wasb.impl");
@@ -223,8 +222,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("hadoop.common.configuration.version");
// - org.apache.hadoop.fs.FileSystem
xmlPropsToSkipCompare.add("fs.har.impl.disable.cache");
- // - org.apache.hadoop.fs.FileSystem#getFileSystemClass()
- xmlPropsToSkipCompare.add("fs.swift.impl");
+
// - package org.apache.hadoop.tracing.TraceUtils ?
xmlPropsToSkipCompare.add("hadoop.htrace.span.receiver.classes");
// Private keys
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
index 9d7f4255978..6db47d6d22f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfServlet.java
@@ -41,9 +41,12 @@ import org.xml.sax.InputSource;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.util.XMLUtils;
+
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Mockito;
+
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.mock;
import static org.junit.Assert.*;
@@ -223,8 +226,7 @@ public class TestConfServlet {
ConfServlet.writeResponse(getTestConf(), sw, "xml");
String xml = sw.toString();
- DocumentBuilderFactory docBuilderFactory
- = DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory docBuilderFactory = XMLUtils.newSecureDocumentBuilderFactory();
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc = builder.parse(new InputSource(new StringReader(xml)));
NodeList nameNodes = doc.getElementsByTagName("name");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 152159b3f3e..879f1781d74 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -194,7 +194,7 @@ public abstract class TestConfigurationFieldsBase {
HashMap retVal = new HashMap<>();
// Setup regexp for valid properties
- String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
+ String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z%s0-9_-]+)+$";
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
index cb6a1fb31e6..b0c3b090022 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
@@ -36,6 +36,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -62,6 +63,8 @@ public class TestKeyProvider {
} catch (IOException e) {
assertTrue(true);
}
+ intercept(NullPointerException.class, () ->
+ KeyProvider.getBaseName(null));
}
@Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index c884e223365..94d90b2eb97 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -1321,16 +1321,16 @@ public class TestFileUtil {
if (wildcardPath.equals(classPath)) {
// add wildcard matches
for (File wildcardMatch: wildcardMatches) {
- expectedClassPaths.add(wildcardMatch.toURI().toURL()
+ expectedClassPaths.add(wildcardMatch.getCanonicalFile().toURI().toURL()
.toExternalForm());
}
} else {
File fileCp = null;
if(!new Path(classPath).isAbsolute()) {
- fileCp = new File(tmp, classPath);
+ fileCp = new File(tmp, classPath).getCanonicalFile();
}
else {
- fileCp = new File(classPath);
+ fileCp = new File(classPath).getCanonicalFile();
}
if (nonExistentSubdir.equals(classPath)) {
// expect to maintain trailing path separator if present in input, even
@@ -1385,7 +1385,8 @@ public class TestFileUtil {
for (Path jar: jars) {
URL url = jar.toUri().toURL();
assertTrue("the jar should match either of the jars",
- url.equals(jar1.toURI().toURL()) || url.equals(jar2.toURI().toURL()));
+ url.equals(jar1.getCanonicalFile().toURI().toURL()) ||
+ url.equals(jar2.getCanonicalFile().toURI().toURL()));
}
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
index 5ed4d9bc9a7..3d8ea0e826c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java
@@ -143,6 +143,11 @@ public class TestFilterFileSystem {
of the filter such as checksums.
*/
MultipartUploaderBuilder createMultipartUploader(Path basePath);
+
+ FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException;
+
+ FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress, boolean appendToNewBlock) throws IOException;
}
@Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index 711ab94fdf1..b227e169088 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -250,6 +250,11 @@ public class TestHarFileSystem {
MultipartUploaderBuilder createMultipartUploader(Path basePath)
throws IOException;
+
+ FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException;
+
+ FSDataOutputStream append(Path f, int bufferSize,
+ Progressable progress, boolean appendToNewBlock) throws IOException;
}
@Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
index ebf0e14053b..e964d23f4b7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
@@ -61,6 +61,9 @@ public class TestVectoredReadUtils extends HadoopTestBase {
.describedAs("Slicing on the same offset shouldn't " +
"create a new buffer")
.isEqualTo(slice);
+ Assertions.assertThat(slice.position())
+ .describedAs("Slicing should return buffers starting from position 0")
+ .isEqualTo(0);
// try slicing a range
final int offset = 100;
@@ -77,6 +80,9 @@ public class TestVectoredReadUtils extends HadoopTestBase {
.describedAs("Slicing should use the same underlying " +
"data")
.isEqualTo(slice.array());
+ Assertions.assertThat(slice.position())
+ .describedAs("Slicing should return buffers starting from position 0")
+ .isEqualTo(0);
// test the contents of the slice
intBuffer = slice.asIntBuffer();
for(int i=0; i < sliceLength / Integer.BYTES; ++i) {
@@ -96,7 +102,10 @@ public class TestVectoredReadUtils extends HadoopTestBase {
@Test
public void testMerge() {
- FileRange base = FileRange.createFileRange(2000, 1000);
+ // a reference to use for tracking
+ Object tracker1 = "one";
+ Object tracker2 = "two";
+ FileRange base = FileRange.createFileRange(2000, 1000, tracker1);
CombinedFileRange mergeBase = new CombinedFileRange(2000, 3000, base);
// test when the gap between is too big
@@ -104,44 +113,48 @@ public class TestVectoredReadUtils extends HadoopTestBase {
FileRange.createFileRange(5000, 1000), 2000, 4000));
assertEquals("Number of ranges in merged range shouldn't increase",
1, mergeBase.getUnderlying().size());
- assertEquals("post merge offset", 2000, mergeBase.getOffset());
- assertEquals("post merge length", 1000, mergeBase.getLength());
+ assertFileRange(mergeBase, 2000, 1000);
// test when the total size gets exceeded
assertFalse("Large size ranges shouldn't get merged", mergeBase.merge(5000, 6000,
FileRange.createFileRange(5000, 1000), 2001, 3999));
assertEquals("Number of ranges in merged range shouldn't increase",
1, mergeBase.getUnderlying().size());
- assertEquals("post merge offset", 2000, mergeBase.getOffset());
- assertEquals("post merge length", 1000, mergeBase.getLength());
+ assertFileRange(mergeBase, 2000, 1000);
// test when the merge works
assertTrue("ranges should get merged ", mergeBase.merge(5000, 6000,
- FileRange.createFileRange(5000, 1000), 2001, 4000));
+ FileRange.createFileRange(5000, 1000, tracker2),
+ 2001, 4000));
assertEquals("post merge size", 2, mergeBase.getUnderlying().size());
- assertEquals("post merge offset", 2000, mergeBase.getOffset());
- assertEquals("post merge length", 4000, mergeBase.getLength());
+ assertFileRange(mergeBase, 2000, 4000);
+
+ Assertions.assertThat(mergeBase.getUnderlying().get(0).getReference())
+ .describedAs("reference of range %s", mergeBase.getUnderlying().get(0))
+ .isSameAs(tracker1);
+ Assertions.assertThat(mergeBase.getUnderlying().get(1).getReference())
+ .describedAs("reference of range %s", mergeBase.getUnderlying().get(1))
+ .isSameAs(tracker2);
// reset the mergeBase and test with a 10:1 reduction
mergeBase = new CombinedFileRange(200, 300, base);
- assertEquals(200, mergeBase.getOffset());
- assertEquals(100, mergeBase.getLength());
+ assertFileRange(mergeBase, 200, 100);
+
assertTrue("ranges should get merged ", mergeBase.merge(500, 600,
FileRange.createFileRange(5000, 1000), 201, 400));
assertEquals("post merge size", 2, mergeBase.getUnderlying().size());
- assertEquals("post merge offset", 200, mergeBase.getOffset());
- assertEquals("post merge length", 400, mergeBase.getLength());
+ assertFileRange(mergeBase, 200, 400);
}
@Test
public void testSortAndMerge() {
List input = Arrays.asList(
- FileRange.createFileRange(3000, 100),
- FileRange.createFileRange(2100, 100),
- FileRange.createFileRange(1000, 100)
+ FileRange.createFileRange(3000, 100, "1"),
+ FileRange.createFileRange(2100, 100, null),
+ FileRange.createFileRange(1000, 100, "3")
);
assertFalse("Ranges are non disjoint", VectoredReadUtils.isOrderedDisjoint(input, 100, 800));
- List outputList = VectoredReadUtils.mergeSortedRanges(
+ final List outputList = VectoredReadUtils.mergeSortedRanges(
Arrays.asList(sortRanges(input)), 100, 1001, 2500);
Assertions.assertThat(outputList)
.describedAs("merged range size")
@@ -150,51 +163,105 @@ public class TestVectoredReadUtils extends HadoopTestBase {
Assertions.assertThat(output.getUnderlying())
.describedAs("merged range underlying size")
.hasSize(3);
- assertEquals("range[1000,3100)", output.toString());
+ // range[1000,3100)
+ assertFileRange(output, 1000, 2100);
assertTrue("merged output ranges are disjoint",
VectoredReadUtils.isOrderedDisjoint(outputList, 100, 800));
// the minSeek doesn't allow the first two to merge
assertFalse("Ranges are non disjoint",
VectoredReadUtils.isOrderedDisjoint(input, 100, 1000));
- outputList = VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(input)),
+ final List list2 = VectoredReadUtils.mergeSortedRanges(
+ Arrays.asList(sortRanges(input)),
100, 1000, 2100);
- Assertions.assertThat(outputList)
+ Assertions.assertThat(list2)
.describedAs("merged range size")
.hasSize(2);
- assertEquals("range[1000,1100)", outputList.get(0).toString());
- assertEquals("range[2100,3100)", outputList.get(1).toString());
+ assertFileRange(list2.get(0), 1000, 100);
+
+ // range[2100,3100)
+ assertFileRange(list2.get(1), 2100, 1000);
+
assertTrue("merged output ranges are disjoint",
- VectoredReadUtils.isOrderedDisjoint(outputList, 100, 1000));
+ VectoredReadUtils.isOrderedDisjoint(list2, 100, 1000));
// the maxSize doesn't allow the third range to merge
assertFalse("Ranges are non disjoint",
VectoredReadUtils.isOrderedDisjoint(input, 100, 800));
- outputList = VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(input)),
+ final List list3 = VectoredReadUtils.mergeSortedRanges(
+ Arrays.asList(sortRanges(input)),
100, 1001, 2099);
- Assertions.assertThat(outputList)
+ Assertions.assertThat(list3)
.describedAs("merged range size")
.hasSize(2);
- assertEquals("range[1000,2200)", outputList.get(0).toString());
- assertEquals("range[3000,3100)", outputList.get(1).toString());
+ // range[1000,2200)
+ CombinedFileRange range0 = list3.get(0);
+ assertFileRange(range0, 1000, 1200);
+ assertFileRange(range0.getUnderlying().get(0),
+ 1000, 100, "3");
+ assertFileRange(range0.getUnderlying().get(1),
+ 2100, 100, null);
+ CombinedFileRange range1 = list3.get(1);
+ // range[3000,3100)
+ assertFileRange(range1, 3000, 100);
+ assertFileRange(range1.getUnderlying().get(0),
+ 3000, 100, "1");
+
assertTrue("merged output ranges are disjoint",
- VectoredReadUtils.isOrderedDisjoint(outputList, 100, 800));
+ VectoredReadUtils.isOrderedDisjoint(list3, 100, 800));
// test the round up and round down (the maxSize doesn't allow any merges)
assertFalse("Ranges are non disjoint",
VectoredReadUtils.isOrderedDisjoint(input, 16, 700));
- outputList = VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(input)),
+ final List list4 = VectoredReadUtils.mergeSortedRanges(
+ Arrays.asList(sortRanges(input)),
16, 1001, 100);
- Assertions.assertThat(outputList)
+ Assertions.assertThat(list4)
.describedAs("merged range size")
.hasSize(3);
- assertEquals("range[992,1104)", outputList.get(0).toString());
- assertEquals("range[2096,2208)", outputList.get(1).toString());
- assertEquals("range[2992,3104)", outputList.get(2).toString());
+ // range[992,1104)
+ assertFileRange(list4.get(0), 992, 112);
+ // range[2096,2208)
+ assertFileRange(list4.get(1), 2096, 112);
+ // range[2992,3104)
+ assertFileRange(list4.get(2), 2992, 112);
assertTrue("merged output ranges are disjoint",
- VectoredReadUtils.isOrderedDisjoint(outputList, 16, 700));
+ VectoredReadUtils.isOrderedDisjoint(list4, 16, 700));
}
+ /**
+ * Assert that a file range satisfies the conditions.
+ * @param range range to validate
+ * @param offset offset of range
+ * @param length range length
+ */
+ private void assertFileRange(FileRange range, long offset, int length) {
+ Assertions.assertThat(range)
+ .describedAs("file range %s", range)
+ .isNotNull();
+ Assertions.assertThat(range.getOffset())
+ .describedAs("offset of %s", range)
+ .isEqualTo(offset);
+ Assertions.assertThat(range.getLength())
+ .describedAs("length of %s", range)
+ .isEqualTo(length);
+ }
+
+ /**
+ * Assert that a file range satisfies the conditions.
+ * @param range range to validate
+ * @param offset offset of range
+ * @param length range length
+ * @param reference reference; may be null.
+ */
+ private void assertFileRange(FileRange range, long offset, int length, Object reference) {
+ assertFileRange(range, offset, length);
+ Assertions.assertThat(range.getReference())
+ .describedAs("reference field of file range %s", range)
+ .isEqualTo(reference);
+ }
+
+
@Test
public void testSortAndMergeMoreCases() throws Exception {
List input = Arrays.asList(
@@ -214,7 +281,9 @@ public class TestVectoredReadUtils extends HadoopTestBase {
Assertions.assertThat(output.getUnderlying())
.describedAs("merged range underlying size")
.hasSize(4);
- assertEquals("range[1000,3110)", output.toString());
+
+ assertFileRange(output, 1000, 2110);
+
assertTrue("merged output ranges are disjoint",
VectoredReadUtils.isOrderedDisjoint(outputList, 1, 800));
@@ -227,7 +296,8 @@ public class TestVectoredReadUtils extends HadoopTestBase {
Assertions.assertThat(output.getUnderlying())
.describedAs("merged range underlying size")
.hasSize(4);
- assertEquals("range[1000,3200)", output.toString());
+ assertFileRange(output, 1000, 2200);
+
assertTrue("merged output ranges are disjoint",
VectoredReadUtils.isOrderedDisjoint(outputList, 1, 800));
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
index e86c4be97b9..12ab62556a1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/impl/prefetch/TestFilePosition.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import org.apache.hadoop.test.AbstractHadoopTestBase;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -43,6 +44,7 @@ public class TestFilePosition extends AbstractHadoopTestBase {
new FilePosition(10, 5);
new FilePosition(5, 10);
new FilePosition(10, 5).setData(data, 3, 4);
+ new FilePosition(10, 10).setData(data, 3, 13);
// Verify it throws correctly.
@@ -94,11 +96,11 @@ public class TestFilePosition extends AbstractHadoopTestBase {
"'readOffset' must not be negative", () -> pos.setData(data, 4, -4));
intercept(IllegalArgumentException.class,
- "'readOffset' (15) must be within the range [4, 13]",
+ "'readOffset' (15) must be within the range [4, 14]",
() -> pos.setData(data, 4, 15));
intercept(IllegalArgumentException.class,
- "'readOffset' (3) must be within the range [4, 13]",
+ "'readOffset' (3) must be within the range [4, 14]",
() -> pos.setData(data, 4, 3));
}
@@ -192,4 +194,31 @@ public class TestFilePosition extends AbstractHadoopTestBase {
}
assertTrue(pos.bufferFullyRead());
}
+
+ @Test
+ public void testBounds() {
+ int bufferSize = 8;
+ long fileSize = bufferSize;
+
+ ByteBuffer buffer = ByteBuffer.allocate(bufferSize);
+ BufferData data = new BufferData(0, buffer);
+ FilePosition pos = new FilePosition(fileSize, bufferSize);
+
+ long eofOffset = fileSize;
+ pos.setData(data, 0, eofOffset);
+
+ assertThat(pos.isWithinCurrentBuffer(eofOffset))
+ .describedAs("EOF offset %d should be within the current buffer", eofOffset)
+ .isTrue();
+ assertThat(pos.absolute())
+ .describedAs("absolute() should return the EOF offset")
+ .isEqualTo(eofOffset);
+
+ assertThat(pos.setAbsolute(eofOffset))
+ .describedAs("setAbsolute() should return true on the EOF offset %d", eofOffset)
+ .isTrue();
+ assertThat(pos.absolute())
+ .describedAs("absolute() should return the EOF offset")
+ .isEqualTo(eofOffset);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
index 4eb1d433bee..e806dde11b0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
@@ -31,6 +31,8 @@ import java.nio.file.Files;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Test;
@@ -46,22 +48,19 @@ public class TestTextCommand {
private static final String TEXT_FILENAME =
new File(TEST_ROOT_DIR, "testtextfile.txt").toURI().getPath();
+ private static final String SEPARATOR = System.getProperty("line.separator");
+
/**
* Tests whether binary Avro data files are displayed correctly.
*/
@Test (timeout = 30000)
public void testDisplayForAvroFiles() throws Exception {
String expectedOutput =
- "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" +
- System.getProperty("line.separator") +
- "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" +
- System.getProperty("line.separator") +
- "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" +
- System.getProperty("line.separator") +
- "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" +
- System.getProperty("line.separator") +
- "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" +
- System.getProperty("line.separator");
+ "{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + SEPARATOR
+ + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}" + SEPARATOR
+ + "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}" + SEPARATOR
+ + "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}" + SEPARATOR
+ + "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" + SEPARATOR;
String output = readUsingTextCommand(AVRO_FILENAME,
generateWeatherAvroBinaryData());
@@ -104,11 +103,16 @@ public class TestTextCommand {
throws Exception {
createFile(fileName, fileContents);
- // Prepare and call the Text command's protected getInputStream method
- // using reflection.
Configuration conf = new Configuration();
URI localPath = new URI(fileName);
- PathData pathData = new PathData(localPath, conf);
+ return readUsingTextCommand(localPath, conf);
+ }
+ // Read a file using Display.Text class.
+ private String readUsingTextCommand(URI uri, Configuration conf)
+ throws Exception {
+ // Prepare and call the Text command's protected getInputStream method
+ // using reflection.
+ PathData pathData = new PathData(uri, conf);
Display.Text text = new Display.Text() {
@Override
public InputStream getInputStream(PathData item) throws IOException {
@@ -116,7 +120,7 @@ public class TestTextCommand {
}
};
text.setConf(conf);
- InputStream stream = (InputStream) text.getInputStream(pathData);
+ InputStream stream = text.getInputStream(pathData);
return inputStreamToString(stream);
}
@@ -232,5 +236,21 @@ public class TestTextCommand {
return contents;
}
+
+ @Test
+ public void testDisplayForNonWritableSequenceFile() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
+ Path path = new Path(String.valueOf(TEST_ROOT_DIR), "NonWritableSequenceFile");
+ SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(path),
+ SequenceFile.Writer.keyClass(String.class), SequenceFile.Writer.valueClass(String.class));
+ writer.append("Key1", "Value1");
+ writer.append("Key2", "Value2");
+ writer.close();
+ String expected = "Key1\tValue1" + SEPARATOR + "Key2\tValue2" + SEPARATOR;
+ URI uri = path.toUri();
+ System.out.println(expected);
+ assertEquals(expected, readUsingTextCommand(uri, conf));
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
index b70e011f6aa..c15ec8caa4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestDefaultStringifier.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.junit.Assert.assertEquals;
public class TestDefaultStringifier {
@@ -98,7 +99,7 @@ public class TestDefaultStringifier {
}
@Test
- public void testStoreLoadArray() throws IOException {
+ public void testStoreLoadArray() throws Exception {
LOG.info("Testing DefaultStringifier#storeArray() and #loadArray()");
conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization");
@@ -107,6 +108,8 @@ public class TestDefaultStringifier {
Integer[] array = new Integer[] {1,2,3,4,5};
+ intercept(IndexOutOfBoundsException.class, () ->
+ DefaultStringifier.storeArray(conf, new Integer[] {}, keyName));
DefaultStringifier.storeArray(conf, array, keyName);
Integer[] claimedArray = DefaultStringifier.loadArray(conf, keyName, Integer.class);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index ffa17224b03..25c69765494 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -1216,11 +1216,6 @@ public class TestIPC {
@Test(timeout=30000)
public void testInterrupted() {
Client client = new Client(LongWritable.class, conf);
- Client.getClientExecutor().submit(new Runnable() {
- public void run() {
- while(true);
- }
- });
Thread.currentThread().interrupt();
client.stop();
try {
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index 101750d72c8..084a3dbd4ae 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.test.MockitoUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.event.Level;
@@ -62,13 +63,16 @@ import org.slf4j.event.Level;
import javax.net.SocketFactory;
import java.io.Closeable;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InterruptedIOException;
+import java.io.OutputStream;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.Socket;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
@@ -89,6 +93,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantLock;
import static org.assertj.core.api.Assertions.assertThat;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
@@ -993,6 +998,196 @@ public class TestRPC extends TestRpcBase {
}
}
+ /**
+ * This tests the case where the server isn't receiving new data and
+ * multiple threads queue up to send rpc requests. Only one of the requests
+ * should be written and all of the calling threads should be interrupted.
+ *
+ * We use a mock SocketFactory so that we can control when the input and
+ * output streams are frozen.
+ */
+ @Test(timeout=30000)
+ public void testSlowConnection() throws Exception {
+ SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
+ Socket mockSocket = Mockito.mock(Socket.class);
+ Mockito.when(mockFactory.createSocket()).thenReturn(mockSocket);
+ Mockito.when(mockSocket.getPort()).thenReturn(1234);
+ Mockito.when(mockSocket.getLocalPort()).thenReturn(2345);
+ MockOutputStream mockOutputStream = new MockOutputStream();
+ Mockito.when(mockSocket.getOutputStream()).thenReturn(mockOutputStream);
+ // Use an input stream that always blocks
+ Mockito.when(mockSocket.getInputStream()).thenReturn(new InputStream() {
+ @Override
+ public int read() throws IOException {
+ // wait forever
+ while (true) {
+ try {
+ Thread.sleep(TimeUnit.DAYS.toMillis(1));
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException("test");
+ }
+ }
+ }
+ });
+ Configuration clientConf = new Configuration();
+ // disable ping & timeout to minimize traffic
+ clientConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, false);
+ clientConf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 0);
+ RPC.setProtocolEngine(clientConf, TestRpcService.class, ProtobufRpcEngine.class);
+ // set async mode so that we don't need to implement the input stream
+ final boolean wasAsync = Client.isAsynchronousMode();
+ TestRpcService client = null;
+ try {
+ Client.setAsynchronousMode(true);
+ client = RPC.getProtocolProxy(
+ TestRpcService.class,
+ 0,
+ new InetSocketAddress("localhost", 1234),
+ UserGroupInformation.getCurrentUser(),
+ clientConf,
+ mockFactory).getProxy();
+ // The connection isn't actually made until the first call.
+ client.ping(null, newEmptyRequest());
+ mockOutputStream.waitForFlush(1);
+ final long headerAndFirst = mockOutputStream.getBytesWritten();
+ client.ping(null, newEmptyRequest());
+ mockOutputStream.waitForFlush(2);
+ final long second = mockOutputStream.getBytesWritten() - headerAndFirst;
+ // pause the writer thread
+ mockOutputStream.pause();
+ // create a set of threads to create calls that will back up
+ ExecutorService pool = Executors.newCachedThreadPool();
+ Future[] futures = new Future[numThreads];
+ final AtomicInteger doneThreads = new AtomicInteger(0);
+ for(int thread = 0; thread < numThreads; ++thread) {
+ final TestRpcService finalClient = client;
+ futures[thread] = pool.submit(new Callable() {
+ @Override
+ public Void call() throws Exception {
+ finalClient.ping(null, newEmptyRequest());
+ doneThreads.incrementAndGet();
+ return null;
+ }
+ });
+ }
+ // wait until the threads have started writing
+ mockOutputStream.waitForWriters();
+ // interrupt all the threads
+ for(int thread=0; thread < numThreads; ++thread) {
+ assertTrue("cancel thread " + thread,
+ futures[thread].cancel(true));
+ }
+ // wait until all the writers are cancelled
+ pool.shutdown();
+ pool.awaitTermination(10, TimeUnit.SECONDS);
+ mockOutputStream.resume();
+ // wait for the in flight rpc request to be flushed
+ mockOutputStream.waitForFlush(3);
+ // All the threads should have been interrupted
+ assertEquals(0, doneThreads.get());
+ // make sure that only one additional rpc request was sent
+ assertEquals(headerAndFirst + second * 2,
+ mockOutputStream.getBytesWritten());
+ } finally {
+ Client.setAsynchronousMode(wasAsync);
+ if (client != null) {
+ RPC.stopProxy(client);
+ }
+ }
+ }
+
+ private static final class MockOutputStream extends OutputStream {
+ private long bytesWritten = 0;
+ private AtomicInteger flushCount = new AtomicInteger(0);
+ private ReentrantLock lock = new ReentrantLock(true);
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ lock.lock();
+ bytesWritten += 1;
+ lock.unlock();
+ }
+
+ @Override
+ public void flush() {
+ flushCount.incrementAndGet();
+ }
+
+ public synchronized long getBytesWritten() {
+ return bytesWritten;
+ }
+
+ public void pause() {
+ lock.lock();
+ }
+
+ public void resume() {
+ lock.unlock();
+ }
+
+ private static final int DELAY_MS = 250;
+
+ /**
+ * Wait for the Nth flush, which we assume will happen exactly when the
+ * Nth RPC request is sent.
+ * @param flush the total flush count to wait for
+ * @throws InterruptedException
+ */
+ public void waitForFlush(int flush) throws InterruptedException {
+ while (flushCount.get() < flush) {
+ Thread.sleep(DELAY_MS);
+ }
+ }
+
+ public void waitForWriters() throws InterruptedException {
+ while (!lock.hasQueuedThreads()) {
+ Thread.sleep(DELAY_MS);
+ }
+ }
+ }
+
+ /**
+ * This test causes an exception in the RPC connection setup to make
+ * sure that threads aren't leaked.
+ */
+ @Test(timeout=30000)
+ public void testBadSetup() throws Exception {
+ SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
+ Mockito.when(mockFactory.createSocket())
+ .thenThrow(new IOException("can't connect"));
+ Configuration clientConf = new Configuration();
+ // Set an illegal value to cause an exception in the constructor
+ clientConf.set(CommonConfigurationKeys.IPC_MAXIMUM_RESPONSE_LENGTH,
+ "xxx");
+ RPC.setProtocolEngine(clientConf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ TestRpcService client = null;
+ int threadCount = Thread.getAllStackTraces().size();
+ try {
+ try {
+ client = RPC.getProtocolProxy(
+ TestRpcService.class,
+ 0,
+ new InetSocketAddress("localhost", 1234),
+ UserGroupInformation.getCurrentUser(),
+ clientConf,
+ mockFactory).getProxy();
+ client.ping(null, newEmptyRequest());
+ assertTrue("Didn't throw exception!", false);
+ } catch (ServiceException nfe) {
+ // ensure no extra threads are running.
+ assertEquals(threadCount, Thread.getAllStackTraces().size());
+ } catch (Throwable t) {
+ assertTrue("wrong exception: " + t, false);
+ }
+ } finally {
+ if (client != null) {
+ RPC.stopProxy(client);
+ }
+ }
+ }
+
@Test
public void testConnectionPing() throws Exception {
Server server;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
index d0eeea3e513..6c627116f8c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
@@ -142,6 +142,18 @@ public class TestLogThrottlingHelper {
assertTrue(helper.record("bar", 0).shouldLog());
}
+ @Test
+ public void testInfrequentPrimaryAndDependentLoggers() {
+ helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
+
+ assertTrue(helper.record("foo", 0).shouldLog());
+ assertTrue(helper.record("bar", 0).shouldLog());
+
+ // Both should log once the period has elapsed
+ assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
+ assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
+ }
+
@Test
public void testMultipleLoggersWithValues() {
helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
index 5a1f1d1376d..1e841a68654 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestMetricsSystemImpl.java
@@ -438,6 +438,8 @@ public class TestMetricsSystemImpl {
r = recs.get(1);
assertTrue("NumActiveSinks should be 3", Iterables.contains(r.metrics(),
new MetricGaugeInt(MsInfo.NumActiveSinks, 3)));
+ assertTrue("NumAllSinks should be 3",
+ Iterables.contains(r.metrics(), new MetricGaugeInt(MsInfo.NumAllSinks, 3)));
}
@Test
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
index 10c8057c69e..9984c9b95fb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/lib/TestMutableMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.metrics2.lib;
+import static org.apache.hadoop.metrics2.impl.MsInfo.Context;
import static org.apache.hadoop.metrics2.lib.Interns.info;
import static org.apache.hadoop.test.MetricsAsserts.*;
import static org.mockito.AdditionalMatchers.eq;
@@ -290,6 +291,27 @@ public class TestMutableMetrics {
}
}
+ /**
+ * MutableStat should output 0 instead of the previous state when there is no change.
+ */
+ @Test public void testMutableWithoutChanged() {
+ MetricsRecordBuilder builderWithChange = mockMetricsRecordBuilder();
+ MetricsRecordBuilder builderWithoutChange = mockMetricsRecordBuilder();
+ MetricsRegistry registry = new MetricsRegistry("test");
+ MutableStat stat = registry.newStat("Test", "Test", "Ops", "Val", true);
+ stat.add(1000, 1000);
+ stat.add(1000, 2000);
+ registry.snapshot(builderWithChange, true);
+
+ assertCounter("TestNumOps", 2000L, builderWithChange);
+ assertGauge("TestINumOps", 2000L, builderWithChange);
+ assertGauge("TestAvgVal", 1.5, builderWithChange);
+
+ registry.snapshot(builderWithoutChange, true);
+ assertGauge("TestINumOps", 0L, builderWithoutChange);
+ assertGauge("TestAvgVal", 0.0, builderWithoutChange);
+ }
+
@Test
public void testDuplicateMetrics() {
MutableRatesWithAggregation rates = new MutableRatesWithAggregation();
@@ -479,4 +501,15 @@ public class TestMutableMetrics {
verify(mb, times(2)).addGauge(
info("FooNumOps", "Number of ops for stat with 5s interval"), (long) 0);
}
+
+ /**
+ * Test {@link MutableGaugeFloat#incr()}.
+ */
+ @Test(timeout = 30000)
+ public void testMutableGaugeFloat() {
+ MutableGaugeFloat mgf = new MutableGaugeFloat(Context, 3.2f);
+ assertEquals(3.2f, mgf.value(), 0.0);
+ mgf.incr();
+ assertEquals(4.2f, mgf.value(), 0.0);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
index 1ea3ef18608..4d0f8d2e04f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestInstrumentedReadWriteLock.java
@@ -233,4 +233,111 @@ public class TestInstrumentedReadWriteLock {
assertEquals(2, wlogged.get());
assertEquals(1, wsuppresed.get());
}
+
+
+ /**
+ * Tests the warning when the write lock is held longer than threshold.
+ */
+ @Test(timeout=10000)
+ public void testWriteLockLongHoldingReportWithReentrant() {
+ String testname = name.getMethodName();
+ final AtomicLong time = new AtomicLong(0);
+ Timer mclock = new Timer() {
+ @Override
+ public long monotonicNow() {
+ return time.get();
+ }
+ };
+
+ final AtomicLong wlogged = new AtomicLong(0);
+ final AtomicLong wsuppresed = new AtomicLong(0);
+ final AtomicLong totalHeldTime = new AtomicLong(0);
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
+ InstrumentedWriteLock writeLock = new InstrumentedWriteLock(testname, LOG,
+ readWriteLock, 2000, 300, mclock) {
+ @Override
+ protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+ totalHeldTime.addAndGet(lockHeldTime);
+ wlogged.incrementAndGet();
+ wsuppresed.set(stats.getSuppressedCount());
+ }
+ };
+
+ InstrumentedReadLock readLock = new InstrumentedReadLock(testname, LOG,
+ readWriteLock, 2000, 300, mclock) {
+ @Override
+ protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+ totalHeldTime.addAndGet(lockHeldTime);
+ wlogged.incrementAndGet();
+ wsuppresed.set(stats.getSuppressedCount());
+ }
+ };
+
+ writeLock.lock(); // t = 0
+ time.set(100);
+
+ writeLock.lock(); // t = 100
+ time.set(500);
+
+ writeLock.lock(); // t = 500
+ time.set(2900);
+ writeLock.unlock(); // t = 2900
+
+ readLock.lock(); // t = 2900
+ time.set(3000);
+ readLock.unlock(); // t = 3000
+
+ writeLock.unlock(); // t = 3000
+
+ writeLock.unlock(); // t = 3000
+ assertEquals(1, wlogged.get());
+ assertEquals(0, wsuppresed.get());
+ assertEquals(3000, totalHeldTime.get());
+ }
+
+ /**
+ * Tests the warning when the read lock is held longer than threshold.
+ */
+ @Test(timeout=10000)
+ public void testReadLockLongHoldingReportWithReentrant() {
+ String testname = name.getMethodName();
+ final AtomicLong time = new AtomicLong(0);
+ Timer mclock = new Timer() {
+ @Override
+ public long monotonicNow() {
+ return time.get();
+ }
+ };
+
+ final AtomicLong wlogged = new AtomicLong(0);
+ final AtomicLong wsuppresed = new AtomicLong(0);
+ final AtomicLong totalHelpTime = new AtomicLong(0);
+ ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(true);
+ InstrumentedReadLock readLock = new InstrumentedReadLock(testname, LOG,
+ readWriteLock, 2000, 300, mclock) {
+ @Override
+ protected void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
+ totalHelpTime.addAndGet(lockHeldTime);
+ wlogged.incrementAndGet();
+ wsuppresed.set(stats.getSuppressedCount());
+ }
+ };
+
+ readLock.lock(); // t = 0
+ time.set(100);
+
+ readLock.lock(); // t = 100
+ time.set(500);
+
+ readLock.lock(); // t = 500
+ time.set(3000);
+ readLock.unlock(); // t = 3000
+
+ readLock.unlock(); // t = 3000
+
+ readLock.unlock(); // t = 3000
+ assertEquals(1, wlogged.get());
+ assertEquals(0, wsuppresed.get());
+ assertEquals(3000, totalHelpTime.get());
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java
new file mode 100644
index 00000000000..6db16b6c0c5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestXMLUtils.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.InputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.xml.XMLConstants;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.SAXParser;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.apache.hadoop.test.AbstractHadoopTestBase;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Assert;
+import org.junit.Test;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+public class TestXMLUtils extends AbstractHadoopTestBase {
+
+ @Test
+ public void testSecureDocumentBuilderFactory() throws Exception {
+ DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+ Document doc = db.parse(new InputSource(new StringReader("")));
+ Assertions.assertThat(doc).describedAs("parsed document").isNotNull();
+ }
+
+ @Test(expected = SAXException.class)
+ public void testExternalDtdWithSecureDocumentBuilderFactory() throws Exception {
+ DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+ try (InputStream stream = getResourceStream("/xml/external-dtd.xml")) {
+ Document doc = db.parse(stream);
+ }
+ }
+
+ @Test(expected = SAXException.class)
+ public void testEntityDtdWithSecureDocumentBuilderFactory() throws Exception {
+ DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+ try (InputStream stream = getResourceStream("/xml/entity-dtd.xml")) {
+ Document doc = db.parse(stream);
+ }
+ }
+
+ @Test
+ public void testSecureSAXParserFactory() throws Exception {
+ SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+ parser.parse(new InputSource(new StringReader("")), new DefaultHandler());
+ }
+
+ @Test(expected = SAXException.class)
+ public void testExternalDtdWithSecureSAXParserFactory() throws Exception {
+ SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+ try (InputStream stream = getResourceStream("/xml/external-dtd.xml")) {
+ parser.parse(stream, new DefaultHandler());
+ }
+ }
+
+ @Test(expected = SAXException.class)
+ public void testEntityDtdWithSecureSAXParserFactory() throws Exception {
+ SAXParser parser = XMLUtils.newSecureSAXParserFactory().newSAXParser();
+ try (InputStream stream = getResourceStream("/xml/entity-dtd.xml")) {
+ parser.parse(stream, new DefaultHandler());
+ }
+ }
+
+ @Test
+ public void testSecureTransformerFactory() throws Exception {
+ Transformer transformer = XMLUtils.newSecureTransformerFactory().newTransformer();
+ DocumentBuilder db = XMLUtils.newSecureDocumentBuilderFactory().newDocumentBuilder();
+ Document doc = db.parse(new InputSource(new StringReader("")));
+ try (StringWriter stringWriter = new StringWriter()) {
+ transformer.transform(new DOMSource(doc), new StreamResult(stringWriter));
+ Assertions.assertThat(stringWriter.toString()).contains("")));
+ try (StringWriter stringWriter = new StringWriter()) {
+ transformer.transform(new DOMSource(doc), new StreamResult(stringWriter));
+ Assertions.assertThat(stringWriter.toString()).contains("
+
+
+
+ ]>
+&lol;
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml b/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml
new file mode 100644
index 00000000000..08a13938f5f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/resources/xml/external-dtd.xml
@@ -0,0 +1,23 @@
+
+
+
+
+ First Last
+ Acme
+ (555) 123-4567
+
diff --git a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
index 74130cff19b..45684053a03 100644
--- a/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
+++ b/hadoop-common-project/hadoop-minikdc/src/test/java/org/apache/hadoop/minikdc/TestMiniKdc.java
@@ -38,8 +38,35 @@ import java.util.HashMap;
import java.util.Arrays;
public class TestMiniKdc extends KerberosSecurityTestcase {
- private static final boolean IBM_JAVA = System.getProperty("java.vendor")
- .contains("IBM");
+ private static final boolean IBM_JAVA = shouldUseIbmPackages();
+ // duplicated to avoid cycles in the build
+ private static boolean shouldUseIbmPackages() {
+ final List ibmTechnologyEditionSecurityModules = Arrays.asList(
+ "com.ibm.security.auth.module.JAASLoginModule",
+ "com.ibm.security.auth.module.Win64LoginModule",
+ "com.ibm.security.auth.module.NTLoginModule",
+ "com.ibm.security.auth.module.AIX64LoginModule",
+ "com.ibm.security.auth.module.LinuxLoginModule",
+ "com.ibm.security.auth.module.Krb5LoginModule"
+ );
+
+ if (System.getProperty("java.vendor").contains("IBM")) {
+ return ibmTechnologyEditionSecurityModules
+ .stream().anyMatch((module) -> isSystemClassAvailable(module));
+ }
+
+ return false;
+ }
+
+ private static boolean isSystemClassAvailable(String className) {
+ try {
+ Class.forName(className);
+ return true;
+ } catch (Exception ignored) {
+ return false;
+ }
+ }
+
@Test
public void testMiniKdcStart() {
MiniKdc kdc = getKdc();
@@ -117,9 +144,9 @@ public class TestMiniKdc extends KerberosSecurityTestcase {
options.put("debug", "true");
return new AppConfigurationEntry[]{
- new AppConfigurationEntry(getKrb5LoginModuleName(),
- AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
- options)};
+ new AppConfigurationEntry(getKrb5LoginModuleName(),
+ AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+ options)};
}
}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
index 252eae64b53..53647126463 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
@@ -26,6 +26,7 @@ import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.util.ReferenceCountUtil;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.oncrpc.RpcAcceptedReply.AcceptState;
import org.apache.hadoop.oncrpc.security.VerifierNone;
@@ -163,8 +164,16 @@ public abstract class RpcProgram extends ChannelInboundHandlerAdapter {
public void channelRead(ChannelHandlerContext ctx, Object msg)
throws Exception {
RpcInfo info = (RpcInfo) msg;
+ try {
+ channelRead(ctx, info);
+ } finally {
+ ReferenceCountUtil.release(info.data());
+ }
+ }
+
+ private void channelRead(ChannelHandlerContext ctx, RpcInfo info)
+ throws Exception {
RpcCall call = (RpcCall) info.header();
-
SocketAddress remoteAddress = info.remoteAddress();
if (LOG.isTraceEnabled()) {
LOG.trace(program + " procedure #" + call.getProcedure());
@@ -256,4 +265,4 @@ public abstract class RpcProgram extends ChannelInboundHandlerAdapter {
public int getPortmapUdpTimeoutMillis() {
return portmapUdpTimeoutMillis;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java
index caba13105cc..d814052e43d 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.oncrpc;
+import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.util.List;
@@ -26,6 +27,7 @@ import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.socket.DatagramPacket;
import io.netty.handler.codec.ByteToMessageDecoder;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -129,15 +131,17 @@ public final class RpcUtil {
RpcInfo info = null;
try {
RpcCall callHeader = RpcCall.read(in);
- ByteBuf dataBuffer = Unpooled.wrappedBuffer(in.buffer()
- .slice());
+ ByteBuf dataBuffer = buf.slice(b.position(), b.remaining());
info = new RpcInfo(callHeader, dataBuffer, ctx, ctx.channel(),
remoteAddress);
} catch (Exception exc) {
LOG.info("Malformed RPC request from " + remoteAddress);
} finally {
- buf.release();
+ // only release buffer if it is not passed to downstream handler
+ if (info == null) {
+ buf.release();
+ }
}
if (info != null) {
@@ -170,15 +174,18 @@ public final class RpcUtil {
*/
@ChannelHandler.Sharable
private static final class RpcUdpResponseStage extends
- ChannelInboundHandlerAdapter {
+ SimpleChannelInboundHandler {
+ public RpcUdpResponseStage() {
+ // do not auto release the RpcResponse message.
+ super(false);
+ }
@Override
- public void channelRead(ChannelHandlerContext ctx, Object msg)
- throws Exception {
- RpcResponse r = (RpcResponse) msg;
- // TODO: check out https://github.com/netty/netty/issues/1282 for
- // correct usage
- ctx.channel().writeAndFlush(r.data());
+ protected void channelRead0(ChannelHandlerContext ctx,
+ RpcResponse response) throws Exception {
+ ByteBuf buf = Unpooled.wrappedBuffer(response.data());
+ ctx.writeAndFlush(new DatagramPacket(
+ buf, (InetSocketAddress) response.recipient()));
}
}
}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
index 7d1130b40ff..953d74648db 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/portmap/Portmap.java
@@ -117,15 +117,13 @@ final class Portmap {
.childOption(ChannelOption.SO_REUSEADDR, true)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer() {
- private final IdleStateHandler idleStateHandler = new IdleStateHandler(
- 0, 0, idleTimeMilliSeconds, TimeUnit.MILLISECONDS);
-
@Override
protected void initChannel(SocketChannel ch) throws Exception {
ChannelPipeline p = ch.pipeline();
p.addLast(RpcUtil.constructRpcFrameDecoder(),
- RpcUtil.STAGE_RPC_MESSAGE_PARSER, idleStateHandler, handler,
+ RpcUtil.STAGE_RPC_MESSAGE_PARSER, new IdleStateHandler(0, 0,
+ idleTimeMilliSeconds, TimeUnit.MILLISECONDS), handler,
RpcUtil.STAGE_RPC_TCP_RESPONSE);
}});
diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
index 84fa71a269d..35ab5cdc3da 100644
--- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
+++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/portmap/TestPortmap.java
@@ -23,8 +23,10 @@ import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.util.Arrays;
import java.util.Map;
+import org.apache.hadoop.oncrpc.RpcReply;
import org.junit.Assert;
import org.apache.hadoop.oncrpc.RpcCall;
@@ -35,6 +37,8 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
public class TestPortmap {
private static Portmap pm = new Portmap();
private static final int SHORT_TIMEOUT_MILLISECONDS = 10;
@@ -92,6 +96,19 @@ public class TestPortmap {
pm.getUdpServerLoAddress());
try {
s.send(p);
+
+ // verify that portmap server responds a UDF packet back to the client
+ byte[] receiveData = new byte[65535];
+ DatagramPacket receivePacket = new DatagramPacket(receiveData,
+ receiveData.length);
+ s.setSoTimeout(2000);
+ s.receive(receivePacket);
+
+ // verify that the registration is accepted.
+ XDR xdr = new XDR(Arrays.copyOfRange(receiveData, 0,
+ receivePacket.getLength()));
+ RpcReply reply = RpcReply.read(xdr);
+ assertEquals(reply.getState(), RpcReply.ReplyState.MSG_ACCEPTED);
} finally {
s.close();
}
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 0a5db2565b8..0b1c6012673 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -41,11 +41,21 @@
hadoop-hdfs-client
provided