diff --git a/LICENSE.txt b/LICENSE.txt index 75c55620643..f8de86a1053 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -617,7 +617,7 @@ OTHER DEALINGS IN THE SOFTWARE. The binary distribution of this product bundles these dependencies under the following license: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.3.7 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css @@ -761,7 +761,7 @@ THE SOFTWARE. For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js +hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.3.1.min.js hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 diff --git a/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml b/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml index ccc89c8bf0f..084384d689c 100644 --- a/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml +++ b/hadoop-build-tools/src/main/resources/checkstyle/suppressions.xml @@ -18,4 +18,5 @@ + diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index da519a3998f..b9363de5691 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -83,13 +83,6 @@ hadoop-minicluster true - - - org.apache.hadoop - hadoop-yarn-server-timelineservice - @@ -149,14 +142,6 @@ org.apache.hadoop hadoop-yarn-common - - org.apache.hadoop - hadoop-yarn-server-common - - - org.apache.zookeeper - zookeeper - org.fusesource.leveldbjni leveldbjni-all @@ -225,6 +210,110 @@ javax.servlet javax.servlet-api + + + + org.apache.hadoop + hadoop-auth + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + com.nimbusds + nimbus-jose-jwt + + + net.minidev + json-smart + + + net.minidev + accessors-smart + + + org.apache.kerby + kerb-simplekdc + + + org.apache.kerby + kerb-util + + + org.apache.kerby + token-provider + + + org.apache.kerby + kerb-common + + + org.apache.kerby + kerb-crypto + + + org.apache.kerby + kerby-util + + + org.apache.kerby + kerb-common + + + org.apache.kerby + kerby-pkix + + + org.apache.kerby + kerby-asn1 + + + org.apache.kerby + kerb-core + + + org.apache.kerby + kerby-config + + + org.apache.kerby + kerby-xdr + + + org.apache.kerby + kerb-identity + + + org.apache.kerby + kerb-server + + + org.apache.kerby + kerb-identity + + + org.apache.kerby + kerb-admin + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + + commons-net + commons-net + - org.apache.hadoop:hadoop-yarn-server-timelineservice log4j:log4j @@ -670,10 +757,43 @@ capacity-scheduler.xml krb5.conf + .keep + + + + org.ehcache + + ehcache-107ext.xsd + ehcache-core.xsd + + + + + + dnsjava:dnsjava + + dig* + jnamed* + lookup* + update* + + + + + + + microsoft/ + ${shaded.dependency.prefix}.microsoft. + + **/pom.xml + + + org/ ${shaded.dependency.prefix}.org. diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index bee14305230..cbedd972188 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -2040,6 +2040,35 @@ function hadoop_start_secure_daemon_wrapper return 0 } +## @description Wait till process dies or till timeout +## @audience private +## @stability evolving +## @param pid +## @param timeout +function wait_process_to_die_or_timeout +{ + local pid=$1 + local timeout=$2 + + # Normalize timeout + # Round up or down + timeout=$(printf "%.0f\n" "${timeout}") + if [[ ${timeout} -lt 1 ]]; then + # minimum 1 second + timeout=1 + fi + + # Wait to see if it's still alive + for (( i=0; i < "${timeout}"; i++ )) + do + if kill -0 "${pid}" > /dev/null 2>&1; then + sleep 1 + else + break + fi + done +} + ## @description Stop the non-privileged `command` daemon with that ## @description that is running at `pidfile`. ## @audience public @@ -2060,11 +2089,14 @@ function hadoop_stop_daemon pid=$(cat "$pidfile") kill "${pid}" >/dev/null 2>&1 - sleep "${HADOOP_STOP_TIMEOUT}" + + wait_process_to_die_or_timeout "${pid}" "${HADOOP_STOP_TIMEOUT}" + if kill -0 "${pid}" > /dev/null 2>&1; then hadoop_error "WARNING: ${cmd} did not stop gracefully after ${HADOOP_STOP_TIMEOUT} seconds: Trying to kill with kill -9" kill -9 "${pid}" >/dev/null 2>&1 fi + wait_process_to_die_or_timeout "${pid}" "${HADOOP_STOP_TIMEOUT}" if ps -p "${pid}" > /dev/null 2>&1; then hadoop_error "ERROR: Unable to kill ${pid}" else diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 707b921bf9f..c309941e757 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -3605,6 +3605,7 @@ public abstract class FileSystem extends Configured implements Closeable { private volatile long bytesReadDistanceOfOneOrTwo; private volatile long bytesReadDistanceOfThreeOrFour; private volatile long bytesReadDistanceOfFiveOrLarger; + private volatile long bytesReadErasureCoded; /** * Add another StatisticsData object to this one. @@ -3621,6 +3622,7 @@ public abstract class FileSystem extends Configured implements Closeable { other.bytesReadDistanceOfThreeOrFour; this.bytesReadDistanceOfFiveOrLarger += other.bytesReadDistanceOfFiveOrLarger; + this.bytesReadErasureCoded += other.bytesReadErasureCoded; } /** @@ -3638,6 +3640,7 @@ public abstract class FileSystem extends Configured implements Closeable { -this.bytesReadDistanceOfThreeOrFour; this.bytesReadDistanceOfFiveOrLarger = -this.bytesReadDistanceOfFiveOrLarger; + this.bytesReadErasureCoded = -this.bytesReadErasureCoded; } @Override @@ -3682,6 +3685,10 @@ public abstract class FileSystem extends Configured implements Closeable { public long getBytesReadDistanceOfFiveOrLarger() { return bytesReadDistanceOfFiveOrLarger; } + + public long getBytesReadErasureCoded() { + return bytesReadErasureCoded; + } } private interface StatisticsAggregator { @@ -3873,6 +3880,14 @@ public abstract class FileSystem extends Configured implements Closeable { getThreadStatistics().writeOps += count; } + /** + * Increment the bytes read on erasure-coded files in the statistics. + * @param newBytes the additional bytes read + */ + public void incrementBytesReadErasureCoded(long newBytes) { + getThreadStatistics().bytesReadErasureCoded += newBytes; + } + /** * Increment the bytes read by the network distance in the statistics * In the common network topology setup, distance value should be an even @@ -4067,6 +4082,25 @@ public abstract class FileSystem extends Configured implements Closeable { }); } + /** + * Get the total number of bytes read on erasure-coded files. + * @return the number of bytes + */ + public long getBytesReadErasureCoded() { + return visitAll(new StatisticsAggregator() { + private long bytesReadErasureCoded = 0; + + @Override + public void accept(StatisticsData data) { + bytesReadErasureCoded += data.bytesReadErasureCoded; + } + + public Long aggregate() { + return bytesReadErasureCoded; + } + }); + } + @Override public String toString() { return visitAll(new StatisticsAggregator() { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java index 8c633f6f359..43c23abadea 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java @@ -46,7 +46,8 @@ public class FileSystemStorageStatistics extends StorageStatistics { "bytesReadLocalHost", "bytesReadDistanceOfOneOrTwo", "bytesReadDistanceOfThreeOrFour", - "bytesReadDistanceOfFiveOrLarger" + "bytesReadDistanceOfFiveOrLarger", + "bytesReadErasureCoded" }; private static class LongStatisticIterator @@ -104,6 +105,8 @@ public class FileSystemStorageStatistics extends StorageStatistics { return data.getBytesReadDistanceOfThreeOrFour(); case "bytesReadDistanceOfFiveOrLarger": return data.getBytesReadDistanceOfFiveOrLarger(); + case "bytesReadErasureCoded": + return data.getBytesReadErasureCoded(); default: return null; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 5ef78f2f6a0..df89598e3c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -198,6 +198,12 @@ public class FileUtil { * use getCanonicalPath in File to get the target of the symlink but that * does not indicate if the given path refers to a symlink. */ + + if (f == null) { + LOG.warn("Can not read a null symLink"); + return ""; + } + try { return Shell.execCommand( Shell.getReadlinkCommand(f.toString())).trim(); @@ -1033,6 +1039,13 @@ public class FileUtil { * @return 0 on success */ public static int symLink(String target, String linkname) throws IOException{ + + if (target == null || linkname == null) { + LOG.warn("Can not create a symLink with a target = " + target + + " and link =" + linkname); + return 1; + } + // Run the input paths through Java's File so that they are converted to the // native OS form File targetFile = new File( diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java index 03c7aeddd9c..e62c86ff214 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsUrlConnection.java @@ -57,7 +57,7 @@ class FsUrlConnection extends URLConnection { try { LOG.debug("Connecting to {}", url); FileSystem fs = FileSystem.get(url.toURI(), conf); - is = fs.open(new Path(url.getPath())); + is = fs.open(new Path(url.toURI())); } catch (URISyntaxException e) { throw new IOException(e.toString()); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index c273c7852b4..2435671a31a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -1420,8 +1420,11 @@ public final class HttpServer2 implements FilterContainer { if (servletContext.getAttribute(ADMINS_ACL) != null && !userHasAdministratorAccess(servletContext, remoteUser)) { - response.sendError(HttpServletResponse.SC_FORBIDDEN, "User " - + remoteUser + " is unauthorized to access this page."); + response.sendError(HttpServletResponse.SC_FORBIDDEN, + "Unauthenticated users are not " + + "authorized to access this page."); + LOG.warn("User " + remoteUser + " is unauthorized to access the page " + + request.getRequestURI() + "."); return false; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java index e84574709f7..cb71a80d5f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawDecoder.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Abstract native raw decoder for all native coders to extend with. @@ -34,36 +35,46 @@ abstract class AbstractNativeRawDecoder extends RawErasureDecoder { public static Logger LOG = LoggerFactory.getLogger(AbstractNativeRawDecoder.class); + // Protect ISA-L coder data structure in native layer from being accessed and + // updated concurrently by the init, release and decode functions. + protected final ReentrantReadWriteLock decoderLock = + new ReentrantReadWriteLock(); + public AbstractNativeRawDecoder(ErasureCoderOptions coderOptions) { super(coderOptions); } @Override - protected synchronized void doDecode(ByteBufferDecodingState decodingState) + protected void doDecode(ByteBufferDecodingState decodingState) throws IOException { - if (nativeCoder == 0) { - throw new IOException(String.format("%s closed", - getClass().getSimpleName())); - } - int[] inputOffsets = new int[decodingState.inputs.length]; - int[] outputOffsets = new int[decodingState.outputs.length]; - - ByteBuffer buffer; - for (int i = 0; i < decodingState.inputs.length; ++i) { - buffer = decodingState.inputs[i]; - if (buffer != null) { - inputOffsets[i] = buffer.position(); + decoderLock.readLock().lock(); + try { + if (nativeCoder == 0) { + throw new IOException(String.format("%s closed", + getClass().getSimpleName())); } - } + int[] inputOffsets = new int[decodingState.inputs.length]; + int[] outputOffsets = new int[decodingState.outputs.length]; - for (int i = 0; i < decodingState.outputs.length; ++i) { - buffer = decodingState.outputs[i]; - outputOffsets[i] = buffer.position(); - } + ByteBuffer buffer; + for (int i = 0; i < decodingState.inputs.length; ++i) { + buffer = decodingState.inputs[i]; + if (buffer != null) { + inputOffsets[i] = buffer.position(); + } + } - performDecodeImpl(decodingState.inputs, inputOffsets, - decodingState.decodeLength, decodingState.erasedIndexes, - decodingState.outputs, outputOffsets); + for (int i = 0; i < decodingState.outputs.length; ++i) { + buffer = decodingState.outputs[i]; + outputOffsets[i] = buffer.position(); + } + + performDecodeImpl(decodingState.inputs, inputOffsets, + decodingState.decodeLength, decodingState.erasedIndexes, + decodingState.outputs, outputOffsets); + } finally { + decoderLock.readLock().unlock(); + } } protected abstract void performDecodeImpl(ByteBuffer[] inputs, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java index cab53839b34..44d89c2a1c0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractNativeRawEncoder.java @@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * Abstract native raw encoder for all native coders to extend with. @@ -34,34 +35,44 @@ abstract class AbstractNativeRawEncoder extends RawErasureEncoder { public static Logger LOG = LoggerFactory.getLogger(AbstractNativeRawEncoder.class); + // Protect ISA-L coder data structure in native layer from being accessed and + // updated concurrently by the init, release and encode functions. + protected final ReentrantReadWriteLock encoderLock = + new ReentrantReadWriteLock(); + public AbstractNativeRawEncoder(ErasureCoderOptions coderOptions) { super(coderOptions); } @Override - protected synchronized void doEncode(ByteBufferEncodingState encodingState) + protected void doEncode(ByteBufferEncodingState encodingState) throws IOException { - if (nativeCoder == 0) { - throw new IOException(String.format("%s closed", - getClass().getSimpleName())); - } - int[] inputOffsets = new int[encodingState.inputs.length]; - int[] outputOffsets = new int[encodingState.outputs.length]; - int dataLen = encodingState.inputs[0].remaining(); + encoderLock.readLock().lock(); + try { + if (nativeCoder == 0) { + throw new IOException(String.format("%s closed", + getClass().getSimpleName())); + } + int[] inputOffsets = new int[encodingState.inputs.length]; + int[] outputOffsets = new int[encodingState.outputs.length]; + int dataLen = encodingState.inputs[0].remaining(); - ByteBuffer buffer; - for (int i = 0; i < encodingState.inputs.length; ++i) { - buffer = encodingState.inputs[i]; - inputOffsets[i] = buffer.position(); - } + ByteBuffer buffer; + for (int i = 0; i < encodingState.inputs.length; ++i) { + buffer = encodingState.inputs[i]; + inputOffsets[i] = buffer.position(); + } - for (int i = 0; i < encodingState.outputs.length; ++i) { - buffer = encodingState.outputs[i]; - outputOffsets[i] = buffer.position(); - } + for (int i = 0; i < encodingState.outputs.length; ++i) { + buffer = encodingState.outputs[i]; + outputOffsets[i] = buffer.position(); + } - performEncodeImpl(encodingState.inputs, inputOffsets, dataLen, - encodingState.outputs, outputOffsets); + performEncodeImpl(encodingState.inputs, inputOffsets, dataLen, + encodingState.outputs, outputOffsets); + } finally { + encoderLock.readLock().unlock(); + } } protected abstract void performEncodeImpl( diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java index 85722223039..dc2c33a6bf6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawDecoder.java @@ -36,19 +36,30 @@ public class NativeRSRawDecoder extends AbstractNativeRawDecoder { public NativeRSRawDecoder(ErasureCoderOptions coderOptions) { super(coderOptions); - initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + decoderLock.writeLock().lock(); + try { + initImpl(coderOptions.getNumDataUnits(), + coderOptions.getNumParityUnits()); + } finally { + decoderLock.writeLock().unlock(); + } } @Override - protected synchronized void performDecodeImpl( + protected void performDecodeImpl( ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased, ByteBuffer[] outputs, int[] outputOffsets) throws IOException { decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets); } @Override - public synchronized void release() { - destroyImpl(); + public void release() { + decoderLock.writeLock().lock(); + try { + destroyImpl(); + } finally { + decoderLock.writeLock().unlock(); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java index 754ec884102..ad06927ffe3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawEncoder.java @@ -36,19 +36,30 @@ public class NativeRSRawEncoder extends AbstractNativeRawEncoder { public NativeRSRawEncoder(ErasureCoderOptions coderOptions) { super(coderOptions); - initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + encoderLock.writeLock().lock(); + try { + initImpl(coderOptions.getNumDataUnits(), + coderOptions.getNumParityUnits()); + } finally { + encoderLock.writeLock().unlock(); + } } @Override - protected synchronized void performEncodeImpl( + protected void performEncodeImpl( ByteBuffer[] inputs, int[] inputOffsets, int dataLen, ByteBuffer[] outputs, int[] outputOffsets) throws IOException { encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets); } @Override - public synchronized void release() { - destroyImpl(); + public void release() { + encoderLock.writeLock().lock(); + try { + destroyImpl(); + } finally { + encoderLock.writeLock().unlock(); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java index 17630424985..dd708eb53e1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawDecoder.java @@ -36,19 +36,30 @@ public class NativeXORRawDecoder extends AbstractNativeRawDecoder { public NativeXORRawDecoder(ErasureCoderOptions coderOptions) { super(coderOptions); - initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + decoderLock.writeLock().lock(); + try { + initImpl(coderOptions.getNumDataUnits(), + coderOptions.getNumParityUnits()); + } finally { + decoderLock.writeLock().unlock(); + } } @Override - protected synchronized void performDecodeImpl( + protected void performDecodeImpl( ByteBuffer[] inputs, int[] inputOffsets, int dataLen, int[] erased, ByteBuffer[] outputs, int[] outputOffsets) throws IOException { decodeImpl(inputs, inputOffsets, dataLen, erased, outputs, outputOffsets); } @Override - public synchronized void release() { - destroyImpl(); + public void release() { + decoderLock.writeLock().lock(); + try { + destroyImpl(); + } finally { + decoderLock.writeLock().unlock(); + } } private native void initImpl(int numDataUnits, int numParityUnits); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java index 7f4265b2fa5..66b0a1bff7d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawEncoder.java @@ -36,19 +36,30 @@ public class NativeXORRawEncoder extends AbstractNativeRawEncoder { public NativeXORRawEncoder(ErasureCoderOptions coderOptions) { super(coderOptions); - initImpl(coderOptions.getNumDataUnits(), coderOptions.getNumParityUnits()); + encoderLock.writeLock().lock(); + try { + initImpl(coderOptions.getNumDataUnits(), + coderOptions.getNumParityUnits()); + } finally { + encoderLock.writeLock().unlock(); + } } @Override - protected synchronized void performEncodeImpl( + protected void performEncodeImpl( ByteBuffer[] inputs, int[] inputOffsets, int dataLen, ByteBuffer[] outputs, int[] outputOffsets) throws IOException { encodeImpl(inputs, inputOffsets, dataLen, outputs, outputOffsets); } @Override - public synchronized void release() { - destroyImpl(); + public void release() { + encoderLock.writeLock().lock(); + try { + destroyImpl(); + } finally { + encoderLock.writeLock().unlock(); + } } private native void initImpl(int numDataUnits, int numParityUnits); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java new file mode 100644 index 00000000000..a30e4a84dd8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/IngressPortBasedResolver.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import com.google.common.annotations.VisibleForTesting; +import java.net.InetAddress; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An implementation of SaslPropertiesResolver. Used on server side, + * returns SASL properties based on the port the client is connecting + * to. This should be used along with server side enabling multiple ports + * TODO: when NN multiple listener is enabled, automatically use this + * resolver without having to set in config. + * + * For configuration, for example if server runs on two ports 9000 and 9001, + * and we want to specify 9000 to use auth-conf and 9001 to use auth. + * + * We need to set the following configuration properties: + * ingress.port.sasl.configured.ports=9000,9001 + * ingress.port.sasl.prop.9000=privacy + * ingress.port.sasl.prop.9001=authentication + * + * One note is that, if there is misconfiguration that a port, say, 9002 is + * given in ingress.port.sasl.configured.ports, but it's sasl prop is not + * set, a default of QOP of privacy (auth-conf) will be used. In addition, + * if a port is not given even in ingress.port.sasl.configured.ports, but + * is being checked in getServerProperties(), the default SASL prop will + * be returned. Both of these two cases are considered misconfiguration. + */ +public class IngressPortBasedResolver extends SaslPropertiesResolver { + + public static final Logger LOG = + LoggerFactory.getLogger(IngressPortBasedResolver.class.getName()); + + static final String INGRESS_PORT_SASL_PROP_PREFIX = "ingress.port.sasl.prop"; + + static final String INGRESS_PORT_SASL_CONFIGURED_PORTS = + "ingress.port.sasl.configured.ports"; + + // no need to concurrent map, because after setConf() it never change, + // only for read. + private HashMap> portPropMapping; + + @Override + public void setConf(Configuration conf) { + super.setConf(conf); + portPropMapping = new HashMap<>(); + Collection portStrings = + conf.getTrimmedStringCollection(INGRESS_PORT_SASL_CONFIGURED_PORTS); + for (String portString : portStrings) { + int port = Integer.parseInt(portString); + String configKey = INGRESS_PORT_SASL_PROP_PREFIX + "." + portString; + Map props = getSaslProperties(conf, configKey, + SaslRpcServer.QualityOfProtection.PRIVACY); + portPropMapping.put(port, props); + } + LOG.debug("Configured with port to QOP mapping as:" + portPropMapping); + } + + /** + * Identify the Sasl Properties to be used for a connection with a client. + * @param clientAddress client's address + * @param ingressPort the port that the client is connecting + * @return the sasl properties to be used for the connection. + */ + @Override + @VisibleForTesting + public Map getServerProperties(InetAddress clientAddress, + int ingressPort) { + LOG.debug("Resolving SASL properties for " + clientAddress + " " + + ingressPort); + if (!portPropMapping.containsKey(ingressPort)) { + LOG.warn("An un-configured port is being requested " + ingressPort + + " using default"); + return getDefaultProperties(); + } + return portPropMapping.get(ingressPort); + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java index 305443cea88..64b86e3f274 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslPropertiesResolver.java @@ -18,7 +18,6 @@ package org.apache.hadoop.security; import java.net.InetAddress; -import java.util.Locale; import java.util.Map; import java.util.TreeMap; @@ -95,6 +94,17 @@ public class SaslPropertiesResolver implements Configurable{ return properties; } + /** + * Identify the Sasl Properties to be used for a connection with a client. + * @param clientAddress client's address + * @param ingressPort the port that the client is connecting + * @return the sasl properties to be used for the connection. + */ + public Map getServerProperties(InetAddress clientAddress, + int ingressPort){ + return properties; + } + /** * Identify the Sasl Properties to be used for a connection with a server. * @param serverAddress server's address @@ -103,4 +113,39 @@ public class SaslPropertiesResolver implements Configurable{ public Map getClientProperties(InetAddress serverAddress){ return properties; } + + /** + * Identify the Sasl Properties to be used for a connection with a server. + * @param serverAddress server's address + * @param ingressPort the port that is used to connect to server + * @return the sasl properties to be used for the connection. + */ + public Map getClientProperties(InetAddress serverAddress, + int ingressPort) { + return properties; + } + + /** + * A util function to retrieve specific additional sasl property from config. + * Used by subclasses to read sasl properties used by themselves. + * @param conf the configuration + * @param configKey the config key to look for + * @param defaultQOP the default QOP if the key is missing + * @return sasl property associated with the given key + */ + static Map getSaslProperties(Configuration conf, + String configKey, QualityOfProtection defaultQOP) { + Map saslProps = new TreeMap<>(); + String[] qop = conf.getStrings(configKey, defaultQOP.toString()); + + for (int i=0; i < qop.length; i++) { + qop[i] = QualityOfProtection.valueOf( + StringUtils.toUpperCase(qop[i])).getSaslQop(); + } + + saslProps.put(Sasl.QOP, StringUtils.join(",", qop)); + saslProps.put(Sasl.SERVER_AUTH, "true"); + + return saslProps; + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java index a64c4de7b66..5964886b9a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/WhitelistBasedResolver.java @@ -20,15 +20,10 @@ package org.apache.hadoop.security; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Map; -import java.util.TreeMap; - -import javax.security.sasl.Sasl; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.SaslPropertiesResolver; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.apache.hadoop.util.CombinedIPWhiteList; -import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,18 +129,7 @@ public class WhitelistBasedResolver extends SaslPropertiesResolver { } static Map getSaslProperties(Configuration conf) { - Map saslProps =new TreeMap(); - String[] qop = conf.getStrings(HADOOP_RPC_PROTECTION_NON_WHITELIST, - QualityOfProtection.PRIVACY.toString()); - - for (int i=0; i < qop.length; i++) { - qop[i] = QualityOfProtection.valueOf( - StringUtils.toUpperCase(qop[i])).getSaslQop(); - } - - saslProps.put(Sasl.QOP, StringUtils.join(",", qop)); - saslProps.put(Sasl.SERVER_AUTH, "true"); - - return saslProps; + return getSaslProperties(conf, HADOOP_RPC_PROTECTION_NON_WHITELIST, + QualityOfProtection.PRIVACY); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 04b4b4fe394..0b76f0df2a0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -22,7 +22,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; -import java.io.InputStream; import java.io.InterruptedIOException; import java.nio.charset.Charset; import java.util.Arrays; @@ -1018,17 +1017,7 @@ public abstract class Shell { } // close the input stream try { - // JDK 7 tries to automatically drain the input streams for us - // when the process exits, but since close is not synchronized, - // it creates a race if we close the stream first and the same - // fd is recycled. the stream draining thread will attempt to - // drain that fd!! it may block, OOM, or cause bizarre behavior - // see: https://bugs.openjdk.java.net/browse/JDK-8024521 - // issue is fixed in build 7u60 - InputStream stdout = process.getInputStream(); - synchronized (stdout) { - inReader.close(); - } + inReader.close(); } catch (IOException ioe) { LOG.warn("Error while closing the input stream", ioe); } @@ -1037,10 +1026,7 @@ public abstract class Shell { joinThread(errThread); } try { - InputStream stderr = process.getErrorStream(); - synchronized (stderr) { - errReader.close(); - } + errReader.close(); } catch (IOException ioe) { LOG.warn("Error while closing the error stream", ioe); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java index 11d38c2298f..8276b6e29c6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java @@ -290,6 +290,18 @@ public final class ZKCuratorManager { * @throws Exception If it cannot create the file. */ public void createRootDirRecursively(String path) throws Exception { + createRootDirRecursively(path, null); + } + + /** + * Utility function to ensure that the configured base znode exists. + * This recursively creates the znode as well as all of its parents. + * @param path Path of the znode to create. + * @param zkAcl ACLs for ZooKeeper. + * @throws Exception If it cannot create the file. + */ + public void createRootDirRecursively(String path, List zkAcl) + throws Exception { String[] pathParts = path.split("/"); Preconditions.checkArgument( pathParts.length >= 1 && pathParts[0].isEmpty(), @@ -298,7 +310,7 @@ public final class ZKCuratorManager { for (int i = 1; i < pathParts.length; i++) { sb.append("/").append(pathParts[i]); - create(sb.toString()); + create(sb.toString(), zkAcl); } } diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 25384914e49..676ab0b508e 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -181,6 +181,18 @@ Each metrics record contains tags such as ProcessName, SessionId, and Hostname a | `WarmUpEDEKTimeAvgTime` | Average time of warming up EDEK in milliseconds | | `ResourceCheckTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of NameNode resource check latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | | `StorageBlockReport`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of storage block report latency in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | +| `EditLogTailTimeNumOps` | Total number of times the standby NameNode tailed the edit log | +| `EditLogTailTimeAvgTime` | Average time (in milliseconds) spent by standby NameNode in tailing edit log | +| `EditLogTailTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in tailing edit logs by standby NameNode, in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | +| `EditLogFetchTimeNumOps` | Total number of times the standby NameNode fetched remote edit streams from journal nodes | +| `EditLogFetchTimeAvgTime` | Average time (in milliseconds) spent by standby NameNode in fetching remote edit streams from journal nodes | +| `EditLogFetchTime`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time spent in fetching edit streams from journal nodes by standby NameNode, in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | +| `NumEditLogLoadedNumOps` | Total number of times edits were loaded by standby NameNode | +| `NumEditLogLoadedAvgCount` | Average number of edits loaded by standby NameNode in each edit log tailing | +| `NumEditLogLoaded`*num*`s(50/75/90/95/99)thPercentileCount` | The 50/75/90/95/99th percentile of number of edits loaded by standby NameNode in each edit log tailing. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | +| `EditLogTailIntervalNumOps` | Total number of intervals between edit log tailings by standby NameNode | +| `EditLogTailIntervalAvgTime` | Average time of intervals between edit log tailings by standby NameNode in milliseconds | +| `EditLogTailInterval`*num*`s(50/75/90/95/99)thPercentileLatency` | The 50/75/90/95/99th percentile of time between edit log tailings by standby NameNode, in milliseconds. Percentile measurement is off by default, by watching no intervals. The intervals are specified by `dfs.metrics.percentiles.intervals`. | FSNamesystem ------------ diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/CHANGES.3.0.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/CHANGES.3.0.3.md new file mode 100644 index 00000000000..48065430f0b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/CHANGES.3.0.3.md @@ -0,0 +1,309 @@ + + +# Apache Hadoop Changelog + +## Release 3.0.3 - 2018-05-31 + +### INCOMPATIBLE CHANGES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | RBF: Use the ZooKeeper as the default State Store | Minor | documentation | Yiqun Lin | Yiqun Lin | + + +### NEW FEATURES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13283](https://issues.apache.org/jira/browse/HDFS-13283) | Percentage based Reserved Space Calculation for DataNode | Major | datanode, hdfs | Lukas Majercak | Lukas Majercak | + + +### IMPROVEMENTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-12455](https://issues.apache.org/jira/browse/HDFS-12455) | WebHDFS - Adding "snapshot enabled" status to ListStatus query result. | Major | snapshots, webhdfs | Ajay Kumar | Ajay Kumar | +| [HDFS-13062](https://issues.apache.org/jira/browse/HDFS-13062) | Provide support for JN to use separate journal disk per namespace | Major | federation, journal-node | Bharat Viswanadham | Bharat Viswanadham | +| [HDFS-12933](https://issues.apache.org/jira/browse/HDFS-12933) | Improve logging when DFSStripedOutputStream failed to write some blocks | Minor | erasure-coding | Xiao Chen | chencan | +| [HADOOP-13972](https://issues.apache.org/jira/browse/HADOOP-13972) | ADLS to support per-store configuration | Major | fs/adl | John Zhuge | Sharad Sonker | +| [YARN-7813](https://issues.apache.org/jira/browse/YARN-7813) | Capacity Scheduler Intra-queue Preemption should be configurable for each queue | Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | +| [HDFS-13175](https://issues.apache.org/jira/browse/HDFS-13175) | Add more information for checking argument in DiskBalancerVolume | Minor | diskbalancer | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [HDFS-11187](https://issues.apache.org/jira/browse/HDFS-11187) | Optimize disk access for last partial chunk checksum of Finalized replica | Major | datanode | Wei-Chiu Chuang | Gabor Bota | +| [MAPREDUCE-7061](https://issues.apache.org/jira/browse/MAPREDUCE-7061) | SingleCluster setup document needs to be updated | Major | . | Bharat Viswanadham | Bharat Viswanadham | +| [HADOOP-15263](https://issues.apache.org/jira/browse/HADOOP-15263) | hadoop cloud-storage module to mark hadoop-common as provided; add azure-datalake | Minor | build | Steve Loughran | Steve Loughran | +| [MAPREDUCE-7060](https://issues.apache.org/jira/browse/MAPREDUCE-7060) | Cherry Pick PathOutputCommitter class/factory to branch-3.0 | Minor | . | Steve Loughran | Steve Loughran | +| [HADOOP-15279](https://issues.apache.org/jira/browse/HADOOP-15279) | increase maven heap size recommendations | Minor | build, documentation, test | Allen Wittenauer | Allen Wittenauer | +| [HDFS-13170](https://issues.apache.org/jira/browse/HDFS-13170) | Port webhdfs unmaskedpermission parameter to HTTPFS | Major | . | Stephen O'Donnell | Stephen O'Donnell | +| [HDFS-13225](https://issues.apache.org/jira/browse/HDFS-13225) | StripeReader#checkMissingBlocks() 's IOException info is incomplete | Major | erasure-coding, hdfs-client | lufei | lufei | +| [HDFS-11394](https://issues.apache.org/jira/browse/HDFS-11394) | Support for getting erasure coding policy through WebHDFS#FileStatus | Major | erasure-coding, namenode | Kai Sasaki | Kai Sasaki | +| [HADOOP-15311](https://issues.apache.org/jira/browse/HADOOP-15311) | HttpServer2 needs a way to configure the acceptor/selector count | Major | common | Erik Krogen | Erik Krogen | +| [HDFS-11600](https://issues.apache.org/jira/browse/HDFS-11600) | Refactor TestDFSStripedOutputStreamWithFailure test classes | Minor | erasure-coding, test | Andrew Wang | SammiChen | +| [HDFS-12884](https://issues.apache.org/jira/browse/HDFS-12884) | BlockUnderConstructionFeature.truncateBlock should be of type BlockInfo | Major | namenode | Konstantin Shvachko | chencan | +| [HADOOP-15334](https://issues.apache.org/jira/browse/HADOOP-15334) | Upgrade Maven surefire plugin | Major | build | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-15312](https://issues.apache.org/jira/browse/HADOOP-15312) | Undocumented KeyProvider configuration keys | Major | . | Wei-Chiu Chuang | LiXin Ge | +| [YARN-7623](https://issues.apache.org/jira/browse/YARN-7623) | Fix the CapacityScheduler Queue configuration documentation | Major | . | Arun Suresh | Jonathan Hung | +| [HDFS-13314](https://issues.apache.org/jira/browse/HDFS-13314) | NameNode should optionally exit if it detects FsImage corruption | Major | namenode | Arpit Agarwal | Arpit Agarwal | +| [HADOOP-15342](https://issues.apache.org/jira/browse/HADOOP-15342) | Update ADLS connector to use the current SDK version (2.2.7) | Major | fs/adl | Atul Sikaria | Atul Sikaria | +| [HDFS-13462](https://issues.apache.org/jira/browse/HDFS-13462) | Add BIND\_HOST configuration for JournalNode's HTTP and RPC Servers | Major | hdfs, journal-node | Lukas Majercak | Lukas Majercak | +| [HADOOP-14841](https://issues.apache.org/jira/browse/HADOOP-14841) | Kms client should disconnect if unable to get output stream from connection. | Major | kms | Xiao Chen | Rushabh S Shah | +| [HDFS-12981](https://issues.apache.org/jira/browse/HDFS-12981) | renameSnapshot a Non-Existent snapshot to itself should throw error | Minor | hdfs | Sailesh Patel | Kitti Nanasi | +| [YARN-8201](https://issues.apache.org/jira/browse/YARN-8201) | Skip stacktrace of few exception from ClientRMService | Minor | . | Bibin A Chundatt | Bilwa S T | +| [HADOOP-15441](https://issues.apache.org/jira/browse/HADOOP-15441) | Log kms url and token service at debug level. | Minor | . | Wei-Chiu Chuang | Gabor Bota | +| [HDFS-13544](https://issues.apache.org/jira/browse/HDFS-13544) | Improve logging for JournalNode in federated cluster | Major | federation, hdfs | Hanisha Koneru | Hanisha Koneru | +| [HADOOP-15486](https://issues.apache.org/jira/browse/HADOOP-15486) | Make NetworkTopology#netLock fair | Major | net | Nanda kumar | Nanda kumar | +| [HDFS-13493](https://issues.apache.org/jira/browse/HDFS-13493) | Reduce the HttpServer2 thread count on DataNodes | Major | datanode | Erik Krogen | Erik Krogen | +| [HADOOP-15449](https://issues.apache.org/jira/browse/HADOOP-15449) | Increase default timeout of ZK session to avoid frequent NameNode failover | Critical | common | Karthik Palanisamy | Karthik Palanisamy | + + +### BUG FIXES: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-11968](https://issues.apache.org/jira/browse/HDFS-11968) | ViewFS: StoragePolicies commands fail with HDFS federation | Major | hdfs | Mukul Kumar Singh | Mukul Kumar Singh | +| [HDFS-12813](https://issues.apache.org/jira/browse/HDFS-12813) | RequestHedgingProxyProvider can hide Exception thrown from the Namenode for proxy size of 1 | Major | ha | Mukul Kumar Singh | Mukul Kumar Singh | +| [HDFS-13048](https://issues.apache.org/jira/browse/HDFS-13048) | LowRedundancyReplicatedBlocks metric can be negative | Major | metrics | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13115](https://issues.apache.org/jira/browse/HDFS-13115) | In getNumUnderConstructionBlocks(), ignore the inodeIds for which the inodes have been deleted | Major | . | Yongjun Zhang | Yongjun Zhang | +| [HDFS-10453](https://issues.apache.org/jira/browse/HDFS-10453) | ReplicationMonitor thread could stuck for long time due to the race between replication and delete of same file in a large cluster. | Major | namenode | He Xiaoqiao | He Xiaoqiao | +| [MAPREDUCE-7053](https://issues.apache.org/jira/browse/MAPREDUCE-7053) | Timed out tasks can fail to produce thread dump | Major | . | Jason Lowe | Jason Lowe | +| [HADOOP-15206](https://issues.apache.org/jira/browse/HADOOP-15206) | BZip2 drops and duplicates records when input split size is small | Major | . | Aki Tanaka | Aki Tanaka | +| [YARN-7937](https://issues.apache.org/jira/browse/YARN-7937) | Fix http method name in Cluster Application Timeout Update API example request | Minor | docs, documentation | Charan Hebri | Charan Hebri | +| [YARN-7947](https://issues.apache.org/jira/browse/YARN-7947) | Capacity Scheduler intra-queue preemption can NPE for non-schedulable apps | Major | capacity scheduler, scheduler preemption | Eric Payne | Eric Payne | +| [HADOOP-10571](https://issues.apache.org/jira/browse/HADOOP-10571) | Use Log.\*(Object, Throwable) overload to log exceptions | Major | . | Arpit Agarwal | Andras Bokor | +| [HDFS-12781](https://issues.apache.org/jira/browse/HDFS-12781) | After Datanode down, In Namenode UI Datanode tab is throwing warning message. | Major | datanode | Harshakiran Reddy | Brahma Reddy Battula | +| [HDFS-12070](https://issues.apache.org/jira/browse/HDFS-12070) | Failed block recovery leaves files open indefinitely and at risk for data loss | Major | . | Daryn Sharp | Kihwal Lee | +| [HDFS-13145](https://issues.apache.org/jira/browse/HDFS-13145) | SBN crash when transition to ANN with in-progress edit tailing enabled | Major | ha, namenode | Chao Sun | Chao Sun | +| [HDFS-13114](https://issues.apache.org/jira/browse/HDFS-13114) | CryptoAdmin#ReencryptZoneCommand should resolve Namespace info from path | Major | encryption, hdfs | Hanisha Koneru | Hanisha Koneru | +| [HDFS-13081](https://issues.apache.org/jira/browse/HDFS-13081) | Datanode#checkSecureConfig should allow SASL and privileged HTTP | Major | datanode, security | Xiaoyu Yao | Ajay Kumar | +| [MAPREDUCE-7059](https://issues.apache.org/jira/browse/MAPREDUCE-7059) | Downward Compatibility issue: MR job fails because of unknown setErasureCodingPolicy method from 3.x client to HDFS 2.x cluster | Critical | job submission | Jiandan Yang | Jiandan Yang | +| [HADOOP-15275](https://issues.apache.org/jira/browse/HADOOP-15275) | Incorrect javadoc for return type of RetryPolicy#shouldRetry | Minor | documentation | Nanda kumar | Nanda kumar | +| [YARN-7511](https://issues.apache.org/jira/browse/YARN-7511) | NPE in ContainerLocalizer when localization failed for running container | Major | nodemanager | Tao Yang | Tao Yang | +| [MAPREDUCE-7023](https://issues.apache.org/jira/browse/MAPREDUCE-7023) | TestHadoopArchiveLogs.testCheckFilesAndSeedApps fails on rerun | Minor | test | Gergely Novák | Gergely Novák | +| [HDFS-13040](https://issues.apache.org/jira/browse/HDFS-13040) | Kerberized inotify client fails despite kinit properly | Major | namenode | Wei-Chiu Chuang | Xiao Chen | +| [YARN-7736](https://issues.apache.org/jira/browse/YARN-7736) | Fix itemization in YARN federation document | Minor | documentation | Akira Ajisaka | Sen Zhao | +| [HDFS-13164](https://issues.apache.org/jira/browse/HDFS-13164) | File not closed if streamer fail with DSQuotaExceededException | Major | hdfs-client | Xiao Chen | Xiao Chen | +| [HADOOP-15289](https://issues.apache.org/jira/browse/HADOOP-15289) | FileStatus.readFields() assertion incorrect | Critical | . | Steve Loughran | Steve Loughran | +| [HDFS-13109](https://issues.apache.org/jira/browse/HDFS-13109) | Support fully qualified hdfs path in EZ commands | Major | hdfs | Hanisha Koneru | Hanisha Koneru | +| [HADOOP-15296](https://issues.apache.org/jira/browse/HADOOP-15296) | Fix a wrong link for RBF in the top page | Minor | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-15273](https://issues.apache.org/jira/browse/HADOOP-15273) | distcp can't handle remote stores with different checksum algorithms | Critical | tools/distcp | Steve Loughran | Steve Loughran | +| [MAPREDUCE-6930](https://issues.apache.org/jira/browse/MAPREDUCE-6930) | mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores are both present twice in mapred-default.xml | Major | mrv2 | Daniel Templeton | Sen Zhao | +| [HDFS-13190](https://issues.apache.org/jira/browse/HDFS-13190) | Document WebHDFS support for snapshot diff | Major | documentation, webhdfs | Xiaoyu Yao | Lokesh Jain | +| [HDFS-13244](https://issues.apache.org/jira/browse/HDFS-13244) | Add stack, conf, metrics links to utilities dropdown in NN webUI | Major | . | Bharat Viswanadham | Bharat Viswanadham | +| [HDFS-12156](https://issues.apache.org/jira/browse/HDFS-12156) | TestFSImage fails without -Pnative | Major | test | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13239](https://issues.apache.org/jira/browse/HDFS-13239) | Fix non-empty dir warning message when setting default EC policy | Minor | . | Hanisha Koneru | Bharat Viswanadham | +| [YARN-8022](https://issues.apache.org/jira/browse/YARN-8022) | ResourceManager UI cluster/app/\ page fails to render | Blocker | webapp | Tarun Parimi | Tarun Parimi | +| [MAPREDUCE-7064](https://issues.apache.org/jira/browse/MAPREDUCE-7064) | Flaky test TestTaskAttempt#testReducerCustomResourceTypes | Major | client, test | Peter Bacsko | Peter Bacsko | +| [HDFS-12723](https://issues.apache.org/jira/browse/HDFS-12723) | TestReadStripedFileWithMissingBlocks#testReadFileWithMissingBlocks failing consistently. | Major | . | Rushabh S Shah | Ajay Kumar | +| [YARN-7636](https://issues.apache.org/jira/browse/YARN-7636) | Re-reservation count may overflow when cluster resource exhausted for a long time | Major | capacityscheduler | Tao Yang | Tao Yang | +| [HDFS-12886](https://issues.apache.org/jira/browse/HDFS-12886) | Ignore minReplication for block recovery | Major | hdfs, namenode | Lukas Majercak | Lukas Majercak | +| [HDFS-13296](https://issues.apache.org/jira/browse/HDFS-13296) | GenericTestUtils generates paths with drive letter in Windows and fail webhdfs related test cases | Major | . | Xiao Liang | Xiao Liang | +| [HDFS-13268](https://issues.apache.org/jira/browse/HDFS-13268) | TestWebHdfsFileContextMainOperations fails on Windows | Major | . | Íñigo Goiri | Xiao Liang | +| [YARN-8054](https://issues.apache.org/jira/browse/YARN-8054) | Improve robustness of the LocalDirsHandlerService MonitoringTimerTask thread | Major | . | Jonathan Eagles | Jonathan Eagles | +| [HDFS-13195](https://issues.apache.org/jira/browse/HDFS-13195) | DataNode conf page cannot display the current value after reconfig | Minor | datanode | maobaolong | maobaolong | +| [YARN-8063](https://issues.apache.org/jira/browse/YARN-8063) | DistributedShellTimelinePlugin wrongly check for entityId instead of entityType | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-8062](https://issues.apache.org/jira/browse/YARN-8062) | yarn rmadmin -getGroups returns group from which the user has been removed | Critical | . | Sumana Sathish | Sunil Govindan | +| [YARN-8068](https://issues.apache.org/jira/browse/YARN-8068) | Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ | Blocker | yarn | Sunil Govindan | Sunil Govindan | +| [YARN-7734](https://issues.apache.org/jira/browse/YARN-7734) | YARN-5418 breaks TestContainerLogsPage.testContainerLogPageAccess | Major | . | Miklos Szegedi | Tao Yang | +| [HDFS-13087](https://issues.apache.org/jira/browse/HDFS-13087) | Snapshotted encryption zone information should be immutable | Major | encryption | LiXin Ge | LiXin Ge | +| [HADOOP-12862](https://issues.apache.org/jira/browse/HADOOP-12862) | LDAP Group Mapping over SSL can not specify trust store | Major | . | Wei-Chiu Chuang | Wei-Chiu Chuang | +| [HDFS-13349](https://issues.apache.org/jira/browse/HDFS-13349) | Unresolved merge conflict in ViewFs.md | Blocker | documentation | Gera Shegalov | Yiqun Lin | +| [HADOOP-15317](https://issues.apache.org/jira/browse/HADOOP-15317) | Improve NetworkTopology chooseRandom's loop | Major | . | Xiao Chen | Xiao Chen | +| [HADOOP-15355](https://issues.apache.org/jira/browse/HADOOP-15355) | TestCommonConfigurationFields is broken by HADOOP-15312 | Major | test | Konstantin Shvachko | LiXin Ge | +| [HDFS-13350](https://issues.apache.org/jira/browse/HDFS-13350) | Negative legacy block ID will confuse Erasure Coding to be considered as striped block | Major | erasure-coding | Lei (Eddy) Xu | Lei (Eddy) Xu | +| [YARN-7905](https://issues.apache.org/jira/browse/YARN-7905) | Parent directory permission incorrect during public localization | Critical | . | Bibin A Chundatt | Bilwa S T | +| [HDFS-13420](https://issues.apache.org/jira/browse/HDFS-13420) | License header is displayed in ArchivalStorage/MemoryStorage html pages | Minor | documentation | Akira Ajisaka | Akira Ajisaka | +| [HDFS-13328](https://issues.apache.org/jira/browse/HDFS-13328) | Abstract ReencryptionHandler recursive logic in separate class. | Major | namenode | Surendra Singh Lilhore | Surendra Singh Lilhore | +| [HADOOP-15357](https://issues.apache.org/jira/browse/HADOOP-15357) | Configuration.getPropsWithPrefix no longer does variable substitution | Major | . | Jim Brennan | Jim Brennan | +| [MAPREDUCE-7062](https://issues.apache.org/jira/browse/MAPREDUCE-7062) | Update mapreduce.job.tags description for making use for ATSv2 purpose. | Major | . | Charan Hebri | Charan Hebri | +| [YARN-8073](https://issues.apache.org/jira/browse/YARN-8073) | TimelineClientImpl doesn't honor yarn.timeline-service.versions configuration | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-13427](https://issues.apache.org/jira/browse/HDFS-13427) | Fix the section titles of transparent encryption document | Minor | documentation | Akira Ajisaka | Akira Ajisaka | +| [YARN-7527](https://issues.apache.org/jira/browse/YARN-7527) | Over-allocate node resource in async-scheduling mode of CapacityScheduler | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-8120](https://issues.apache.org/jira/browse/YARN-8120) | JVM can crash with SIGSEGV when exiting due to custom leveldb logger | Major | nodemanager, resourcemanager | Jason Lowe | Jason Lowe | +| [YARN-8147](https://issues.apache.org/jira/browse/YARN-8147) | TestClientRMService#testGetApplications sporadically fails | Major | test | Jason Lowe | Jason Lowe | +| [HDFS-13436](https://issues.apache.org/jira/browse/HDFS-13436) | Fix javadoc of package-info.java | Major | documentation | Akira Ajisaka | Akira Ajisaka | +| [HADOOP-14970](https://issues.apache.org/jira/browse/HADOOP-14970) | MiniHadoopClusterManager doesn't respect lack of format option | Minor | . | Erik Krogen | Erik Krogen | +| [HDFS-13330](https://issues.apache.org/jira/browse/HDFS-13330) | ShortCircuitCache#fetchOrCreate never retries | Major | . | Wei-Chiu Chuang | Gabor Bota | +| [YARN-8156](https://issues.apache.org/jira/browse/YARN-8156) | Increase the default value of yarn.timeline-service.app-collector.linger-period.ms | Major | . | Rohith Sharma K S | Charan Hebri | +| [YARN-8165](https://issues.apache.org/jira/browse/YARN-8165) | Incorrect queue name logging in AbstractContainerAllocator | Trivial | capacityscheduler | Weiwei Yang | Weiwei Yang | +| [HDFS-12828](https://issues.apache.org/jira/browse/HDFS-12828) | OIV ReverseXML Processor fails with escaped characters | Critical | hdfs | Erik Krogen | Erik Krogen | +| [HADOOP-15396](https://issues.apache.org/jira/browse/HADOOP-15396) | Some java source files are executable | Minor | . | Akira Ajisaka | Shashikant Banerjee | +| [YARN-6827](https://issues.apache.org/jira/browse/YARN-6827) | [ATS1/1.5] NPE exception while publishing recovering applications into ATS during RM restart. | Major | resourcemanager | Rohith Sharma K S | Rohith Sharma K S | +| [YARN-7786](https://issues.apache.org/jira/browse/YARN-7786) | NullPointerException while launching ApplicationMaster | Major | . | lujie | lujie | +| [HDFS-10183](https://issues.apache.org/jira/browse/HDFS-10183) | Prevent race condition during class initialization | Minor | fs | Pavel Avgustinov | Pavel Avgustinov | +| [HDFS-13388](https://issues.apache.org/jira/browse/HDFS-13388) | RequestHedgingProxyProvider calls multiple configured NNs all the time | Major | hdfs-client | Jinglun | Jinglun | +| [HDFS-13433](https://issues.apache.org/jira/browse/HDFS-13433) | webhdfs requests can be routed incorrectly in federated cluster | Critical | webhdfs | Arpit Agarwal | Arpit Agarwal | +| [HDFS-13408](https://issues.apache.org/jira/browse/HDFS-13408) | MiniDFSCluster to support being built on randomized base directory | Major | test | Xiao Liang | Xiao Liang | +| [HADOOP-15390](https://issues.apache.org/jira/browse/HADOOP-15390) | Yarn RM logs flooded by DelegationTokenRenewer trying to renew KMS tokens | Critical | . | Xiao Chen | Xiao Chen | +| [HDFS-13336](https://issues.apache.org/jira/browse/HDFS-13336) | Test cases of TestWriteToReplica failed in windows | Major | . | Xiao Liang | Xiao Liang | +| [YARN-7598](https://issues.apache.org/jira/browse/YARN-7598) | Document how to use classpath isolation for aux-services in YARN | Major | . | Xuan Gong | Xuan Gong | +| [YARN-8183](https://issues.apache.org/jira/browse/YARN-8183) | Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap | Critical | yarn | Sumana Sathish | Suma Shivaprasad | +| [HADOOP-15411](https://issues.apache.org/jira/browse/HADOOP-15411) | AuthenticationFilter should use Configuration.getPropsWithPrefix instead of iterator | Critical | . | Suma Shivaprasad | Suma Shivaprasad | +| [MAPREDUCE-7042](https://issues.apache.org/jira/browse/MAPREDUCE-7042) | Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled | Major | . | Yesha Vora | Xuan Gong | +| [YARN-8205](https://issues.apache.org/jira/browse/YARN-8205) | Application State is not updated to ATS if AM launching is delayed. | Critical | . | Sumana Sathish | Rohith Sharma K S | +| [YARN-8004](https://issues.apache.org/jira/browse/YARN-8004) | Add unit tests for inter queue preemption for dominant resource calculator | Critical | yarn | Sumana Sathish | Zian Chen | +| [YARN-8221](https://issues.apache.org/jira/browse/YARN-8221) | RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps | Major | webapp | Sunil Govindan | Sunil Govindan | +| [YARN-8210](https://issues.apache.org/jira/browse/YARN-8210) | AMRMClient logging on every heartbeat to track updation of AM RM token causes too many log lines to be generated in AM logs | Major | yarn | Suma Shivaprasad | Suma Shivaprasad | +| [HDFS-13509](https://issues.apache.org/jira/browse/HDFS-13509) | Bug fix for breakHardlinks() of ReplicaInfo/LocalReplica, and fix TestFileAppend failures on Windows | Major | . | Xiao Liang | Xiao Liang | +| [MAPREDUCE-7073](https://issues.apache.org/jira/browse/MAPREDUCE-7073) | Optimize TokenCache#obtainTokensForNamenodesInternal | Major | . | Bibin A Chundatt | Bibin A Chundatt | +| [HADOOP-15406](https://issues.apache.org/jira/browse/HADOOP-15406) | hadoop-nfs dependencies for mockito and junit are not test scope | Major | nfs | Jason Lowe | Jason Lowe | +| [YARN-6385](https://issues.apache.org/jira/browse/YARN-6385) | Fix checkstyle warnings in TestFileSystemApplicationHistoryStore | Minor | . | Yiqun Lin | Yiqun Lin | +| [YARN-8222](https://issues.apache.org/jira/browse/YARN-8222) | Fix potential NPE when gets RMApp from RM context | Critical | . | Tao Yang | Tao Yang | +| [HDFS-13481](https://issues.apache.org/jira/browse/HDFS-13481) | TestRollingFileSystemSinkWithHdfs#testFlushThread: test failed intermittently | Major | hdfs | Gabor Bota | Gabor Bota | +| [YARN-8217](https://issues.apache.org/jira/browse/YARN-8217) | RmAuthenticationFilterInitializer /TimelineAuthenticationFilterInitializer should use Configuration.getPropsWithPrefix instead of iterator | Major | . | Suma Shivaprasad | Suma Shivaprasad | +| [YARN-8025](https://issues.apache.org/jira/browse/YARN-8025) | UsersManangers#getComputedResourceLimitForActiveUsers throws NPE due to preComputedActiveUserLimit is empty | Major | yarn | Jiandan Yang | Tao Yang | +| [YARN-8232](https://issues.apache.org/jira/browse/YARN-8232) | RMContainer lost queue name when RM HA happens | Major | resourcemanager | Hu Ziqian | Hu Ziqian | +| [HDFS-13136](https://issues.apache.org/jira/browse/HDFS-13136) | Avoid taking FSN lock while doing group member lookup for FSD permission check | Major | namenode | Xiaoyu Yao | Xiaoyu Yao | +| [HDFS-13537](https://issues.apache.org/jira/browse/HDFS-13537) | TestHdfsHelper does not generate jceks path properly for relative path in Windows | Major | . | Xiao Liang | Xiao Liang | +| [YARN-7003](https://issues.apache.org/jira/browse/YARN-7003) | DRAINING state of queues is not recovered after RM restart | Major | capacityscheduler | Tao Yang | Tao Yang | +| [YARN-8244](https://issues.apache.org/jira/browse/YARN-8244) | TestContainerSchedulerQueuing.testStartMultipleContainers failed | Major | . | Miklos Szegedi | Jim Brennan | +| [YARN-8288](https://issues.apache.org/jira/browse/YARN-8288) | Fix wrong number of table columns in Resource Model doc | Major | . | Weiwei Yang | Weiwei Yang | +| [HDFS-13539](https://issues.apache.org/jira/browse/HDFS-13539) | DFSStripedInputStream NPE when reportCheckSumFailure | Major | . | Xiao Chen | Xiao Chen | +| [YARN-8278](https://issues.apache.org/jira/browse/YARN-8278) | DistributedScheduling is not working in HA | Blocker | . | Bibin A Chundatt | Bibin A Chundatt | +| [HDFS-13581](https://issues.apache.org/jira/browse/HDFS-13581) | DN UI logs link is broken when https is enabled | Minor | datanode | Namit Maheshwari | Shashikant Banerjee | +| [HDFS-13586](https://issues.apache.org/jira/browse/HDFS-13586) | Fsync fails on directories on Windows | Critical | datanode, hdfs | Lukas Majercak | Lukas Majercak | +| [YARN-8179](https://issues.apache.org/jira/browse/YARN-8179) | Preemption does not happen due to natural\_termination\_factor when DRF is used | Major | . | kyungwan nam | kyungwan nam | +| [HADOOP-15450](https://issues.apache.org/jira/browse/HADOOP-15450) | Avoid fsync storm triggered by DiskChecker and handle disk full situation | Blocker | . | Kihwal Lee | Arpit Agarwal | +| [HDFS-13601](https://issues.apache.org/jira/browse/HDFS-13601) | Optimize ByteString conversions in PBHelper | Major | . | Andrew Wang | Andrew Wang | +| [HDFS-13540](https://issues.apache.org/jira/browse/HDFS-13540) | DFSStripedInputStream should only allocate new buffers when reading | Major | . | Xiao Chen | Xiao Chen | +| [HDFS-13588](https://issues.apache.org/jira/browse/HDFS-13588) | Fix TestFsDatasetImpl test failures on Windows | Major | . | Xiao Liang | Xiao Liang | +| [YARN-8310](https://issues.apache.org/jira/browse/YARN-8310) | Handle old NMTokenIdentifier, AMRMTokenIdentifier, and ContainerTokenIdentifier formats | Major | . | Robert Kanter | Robert Kanter | +| [YARN-8344](https://issues.apache.org/jira/browse/YARN-8344) | Missing nm.stop() in TestNodeManagerResync to fix testKillContainersOnResync | Major | . | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | +| [YARN-8327](https://issues.apache.org/jira/browse/YARN-8327) | Fix TestAggregatedLogFormat#testReadAcontainerLogs1 on Windows | Major | log-aggregation | Giovanni Matteo Fumarola | Giovanni Matteo Fumarola | +| [YARN-8346](https://issues.apache.org/jira/browse/YARN-8346) | Upgrading to 3.1 kills running containers with error "Opportunistic container queue is full" | Blocker | . | Rohith Sharma K S | Jason Lowe | +| [HDFS-13611](https://issues.apache.org/jira/browse/HDFS-13611) | Unsafe use of Text as a ConcurrentHashMap key in PBHelperClient | Major | . | Andrew Wang | Andrew Wang | +| [HDFS-13618](https://issues.apache.org/jira/browse/HDFS-13618) | Fix TestDataNodeFaultInjector test failures on Windows | Major | test | Xiao Liang | Xiao Liang | +| [HADOOP-15473](https://issues.apache.org/jira/browse/HADOOP-15473) | Configure serialFilter in KeyProvider to avoid UnrecoverableKeyException caused by JDK-8189997 | Critical | kms | Gabor Bota | Gabor Bota | +| [YARN-8338](https://issues.apache.org/jira/browse/YARN-8338) | TimelineService V1.5 doesn't come up after HADOOP-15406 | Critical | . | Vinod Kumar Vavilapalli | Vinod Kumar Vavilapalli | + + +### TESTS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HADOOP-15313](https://issues.apache.org/jira/browse/HADOOP-15313) | TestKMS should close providers | Major | kms, test | Xiao Chen | Xiao Chen | +| [HDFS-13503](https://issues.apache.org/jira/browse/HDFS-13503) | Fix TestFsck test failures on Windows | Major | hdfs | Xiao Liang | Xiao Liang | +| [HDFS-13542](https://issues.apache.org/jira/browse/HDFS-13542) | TestBlockManager#testNeededReplicationWhileAppending fails due to improper cluster shutdown in TestBlockManager#testBlockManagerMachinesArray on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13551](https://issues.apache.org/jira/browse/HDFS-13551) | TestMiniDFSCluster#testClusterSetStorageCapacity does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-11700](https://issues.apache.org/jira/browse/HDFS-11700) | TestHDFSServerPorts#testBackupNodePorts doesn't pass on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13548](https://issues.apache.org/jira/browse/HDFS-13548) | TestResolveHdfsSymlink#testFcResolveAfs fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13567](https://issues.apache.org/jira/browse/HDFS-13567) | TestNameNodeMetrics#testGenerateEDEKTime,TestNameNodeMetrics#testResourceCheck should use a different cluster basedir | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13557](https://issues.apache.org/jira/browse/HDFS-13557) | TestDFSAdmin#testListOpenFiles fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13550](https://issues.apache.org/jira/browse/HDFS-13550) | TestDebugAdmin#testComputeMetaCommand fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13559](https://issues.apache.org/jira/browse/HDFS-13559) | TestBlockScanner does not close TestContext properly | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13570](https://issues.apache.org/jira/browse/HDFS-13570) | TestQuotaByStorageType,TestQuota,TestDFSOutputStream fail on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13558](https://issues.apache.org/jira/browse/HDFS-13558) | TestDatanodeHttpXFrame does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13554](https://issues.apache.org/jira/browse/HDFS-13554) | TestDatanodeRegistration#testForcedRegistration does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13556](https://issues.apache.org/jira/browse/HDFS-13556) | TestNestedEncryptionZones does not shut down cluster | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13560](https://issues.apache.org/jira/browse/HDFS-13560) | Insufficient system resources exist to complete the requested service for some tests on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13592](https://issues.apache.org/jira/browse/HDFS-13592) | TestNameNodePrunesMissingStorages#testNameNodePrunesUnreportedStorages does not shut down cluster properly | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13593](https://issues.apache.org/jira/browse/HDFS-13593) | TestBlockReaderLocalLegacy#testBlockReaderLocalLegacyWithAppend fails on Windows | Minor | test | Anbang Hu | Anbang Hu | +| [HDFS-13587](https://issues.apache.org/jira/browse/HDFS-13587) | TestQuorumJournalManager fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13619](https://issues.apache.org/jira/browse/HDFS-13619) | TestAuditLoggerWithCommands fails on Windows | Minor | test | Anbang Hu | Anbang Hu | +| [HDFS-13620](https://issues.apache.org/jira/browse/HDFS-13620) | Randomize the test directory path for TestHDFSFileSystemContract | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13591](https://issues.apache.org/jira/browse/HDFS-13591) | TestDFSShell#testSetrepLow fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HDFS-13632](https://issues.apache.org/jira/browse/HDFS-13632) | Randomize baseDir for MiniJournalCluster in MiniQJMHACluster for TestDFSAdminWithHA | Minor | . | Anbang Hu | Anbang Hu | + + +### SUB-TASKS: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13068](https://issues.apache.org/jira/browse/HDFS-13068) | RBF: Add router admin option to manage safe mode | Major | . | Íñigo Goiri | Yiqun Lin | +| [HADOOP-15040](https://issues.apache.org/jira/browse/HADOOP-15040) | Upgrade AWS SDK to 1.11.271: NPE bug spams logs w/ Yarn Log Aggregation | Blocker | fs/s3 | Aaron Fabbri | Aaron Fabbri | +| [HDFS-13119](https://issues.apache.org/jira/browse/HDFS-13119) | RBF: Manage unavailable clusters | Major | . | Íñigo Goiri | Yiqun Lin | +| [HADOOP-15247](https://issues.apache.org/jira/browse/HADOOP-15247) | Move commons-net up to 3.6 | Minor | fs | Steve Loughran | Steve Loughran | +| [HDFS-13187](https://issues.apache.org/jira/browse/HDFS-13187) | RBF: Fix Routers information shown in the web UI | Minor | . | Wei Yan | Wei Yan | +| [HDFS-13184](https://issues.apache.org/jira/browse/HDFS-13184) | RBF: Improve the unit test TestRouterRPCClientRetries | Minor | test | Yiqun Lin | Yiqun Lin | +| [HDFS-13199](https://issues.apache.org/jira/browse/HDFS-13199) | RBF: Fix the hdfs router page missing label icon issue | Major | federation, hdfs | maobaolong | maobaolong | +| [HADOOP-15264](https://issues.apache.org/jira/browse/HADOOP-15264) | AWS "shaded" SDK 1.11.271 is pulling in netty 4.1.17 | Blocker | fs/s3 | Steve Loughran | Steve Loughran | +| [HADOOP-15090](https://issues.apache.org/jira/browse/HADOOP-15090) | Add ADL troubleshooting doc | Major | documentation, fs/adl | Steve Loughran | Steve Loughran | +| [HDFS-13214](https://issues.apache.org/jira/browse/HDFS-13214) | RBF: Complete document of Router configuration | Major | . | Tao Jie | Yiqun Lin | +| [HADOOP-15267](https://issues.apache.org/jira/browse/HADOOP-15267) | S3A multipart upload fails when SSE-C encryption is enabled | Critical | fs/s3 | Anis Elleuch | Anis Elleuch | +| [HDFS-13230](https://issues.apache.org/jira/browse/HDFS-13230) | RBF: ConnectionManager's cleanup task will compare each pool's own active conns with its total conns | Minor | . | Wei Yan | Chao Sun | +| [HDFS-13233](https://issues.apache.org/jira/browse/HDFS-13233) | RBF: MountTableResolver doesn't return the correct mount point of the given path | Major | hdfs | wangzhiyuan | wangzhiyuan | +| [HADOOP-15277](https://issues.apache.org/jira/browse/HADOOP-15277) | remove .FluentPropertyBeanIntrospector from CLI operation log output | Minor | conf | Steve Loughran | Steve Loughran | +| [HDFS-13212](https://issues.apache.org/jira/browse/HDFS-13212) | RBF: Fix router location cache issue | Major | federation, hdfs | Weiwei Wu | Weiwei Wu | +| [HDFS-13232](https://issues.apache.org/jira/browse/HDFS-13232) | RBF: ConnectionPool should return first usable connection | Minor | . | Wei Yan | Ekanth S | +| [HDFS-13240](https://issues.apache.org/jira/browse/HDFS-13240) | RBF: Update some inaccurate document descriptions | Minor | . | Yiqun Lin | Yiqun Lin | +| [HDFS-11399](https://issues.apache.org/jira/browse/HDFS-11399) | Many tests fails in Windows due to injecting disk failures | Major | . | Yiqun Lin | Yiqun Lin | +| [HDFS-13241](https://issues.apache.org/jira/browse/HDFS-13241) | RBF: TestRouterSafemode failed if the port 8888 is in use | Major | hdfs, test | maobaolong | maobaolong | +| [HDFS-13253](https://issues.apache.org/jira/browse/HDFS-13253) | RBF: Quota management incorrect parent-child relationship judgement | Major | . | Yiqun Lin | Yiqun Lin | +| [HDFS-13226](https://issues.apache.org/jira/browse/HDFS-13226) | RBF: Throw the exception if mount table entry validated failed | Major | hdfs | maobaolong | maobaolong | +| [HDFS-12505](https://issues.apache.org/jira/browse/HDFS-12505) | Extend TestFileStatusWithECPolicy with a random EC policy | Major | erasure-coding, test | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-12587](https://issues.apache.org/jira/browse/HDFS-12587) | Use Parameterized tests in TestBlockInfoStriped and TestLowRedundancyBlockQueues to test all EC policies | Major | erasure-coding, test | Takanobu Asanuma | Takanobu Asanuma | +| [HADOOP-15308](https://issues.apache.org/jira/browse/HADOOP-15308) | TestConfiguration fails on Windows because of paths | Major | test | Íñigo Goiri | Xiao Liang | +| [HDFS-12773](https://issues.apache.org/jira/browse/HDFS-12773) | RBF: Improve State Store FS implementation | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13198](https://issues.apache.org/jira/browse/HDFS-13198) | RBF: RouterHeartbeatService throws out CachedStateStore related exceptions when starting router | Minor | . | Wei Yan | Wei Yan | +| [HDFS-13224](https://issues.apache.org/jira/browse/HDFS-13224) | RBF: Resolvers to support mount points across multiple subclusters | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-15262](https://issues.apache.org/jira/browse/HADOOP-15262) | AliyunOSS: move files under a directory in parallel when rename a directory | Major | fs/oss | wujinhu | wujinhu | +| [HDFS-13215](https://issues.apache.org/jira/browse/HDFS-13215) | RBF: Move Router to its own module | Major | . | Íñigo Goiri | Wei Yan | +| [HDFS-13250](https://issues.apache.org/jira/browse/HDFS-13250) | RBF: Router to manage requests across multiple subclusters | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13318](https://issues.apache.org/jira/browse/HDFS-13318) | RBF: Fix FindBugs in hadoop-hdfs-rbf | Minor | . | Íñigo Goiri | Ekanth S | +| [HDFS-12792](https://issues.apache.org/jira/browse/HDFS-12792) | RBF: Test Router-based federation using HDFSContract | Major | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-7986](https://issues.apache.org/jira/browse/YARN-7986) | ATSv2 REST API queries do not return results for uppercase application tags | Critical | . | Charan Hebri | Charan Hebri | +| [HDFS-12512](https://issues.apache.org/jira/browse/HDFS-12512) | RBF: Add WebHDFS | Major | fs | Íñigo Goiri | Wei Yan | +| [HDFS-13291](https://issues.apache.org/jira/browse/HDFS-13291) | RBF: Implement available space based OrderResolver | Major | . | Yiqun Lin | Yiqun Lin | +| [HDFS-13204](https://issues.apache.org/jira/browse/HDFS-13204) | RBF: Optimize name service safe mode icon | Minor | . | liuhongtong | liuhongtong | +| [HDFS-13352](https://issues.apache.org/jira/browse/HDFS-13352) | RBF: Add xsl stylesheet for hdfs-rbf-default.xml | Major | documentation | Takanobu Asanuma | Takanobu Asanuma | +| [HDFS-13347](https://issues.apache.org/jira/browse/HDFS-13347) | RBF: Cache datanode reports | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13289](https://issues.apache.org/jira/browse/HDFS-13289) | RBF: TestConnectionManager#testCleanup() test case need correction | Minor | . | Dibyendu Karmakar | Dibyendu Karmakar | +| [HDFS-13364](https://issues.apache.org/jira/browse/HDFS-13364) | RBF: Support NamenodeProtocol in the Router | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HADOOP-14651](https://issues.apache.org/jira/browse/HADOOP-14651) | Update okhttp version to 2.7.5 | Major | fs/adl | Ray Chiang | Ray Chiang | +| [YARN-6936](https://issues.apache.org/jira/browse/YARN-6936) | [Atsv2] Retrospect storing entities into sub application table from client perspective | Major | . | Rohith Sharma K S | Rohith Sharma K S | +| [HDFS-13353](https://issues.apache.org/jira/browse/HDFS-13353) | RBF: TestRouterWebHDFSContractCreate failed | Major | test | Takanobu Asanuma | Takanobu Asanuma | +| [YARN-8107](https://issues.apache.org/jira/browse/YARN-8107) | Give an informative message when incorrect format is used in ATSv2 filter attributes | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | +| [HDFS-13402](https://issues.apache.org/jira/browse/HDFS-13402) | RBF: Fix java doc for StateStoreFileSystemImpl | Minor | hdfs | Yiran Wu | Yiran Wu | +| [HDFS-13410](https://issues.apache.org/jira/browse/HDFS-13410) | RBF: Support federation with no subclusters | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13384](https://issues.apache.org/jira/browse/HDFS-13384) | RBF: Improve timeout RPC call mechanism | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13045](https://issues.apache.org/jira/browse/HDFS-13045) | RBF: Improve error message returned from subcluster | Minor | . | Wei Yan | Íñigo Goiri | +| [HDFS-13428](https://issues.apache.org/jira/browse/HDFS-13428) | RBF: Remove LinkedList From StateStoreFileImpl.java | Trivial | federation | BELUGA BEHR | BELUGA BEHR | +| [HDFS-13386](https://issues.apache.org/jira/browse/HDFS-13386) | RBF: Wrong date information in list file(-ls) result | Minor | . | Dibyendu Karmakar | Dibyendu Karmakar | +| [YARN-8027](https://issues.apache.org/jira/browse/YARN-8027) | Setting hostname of docker container breaks for --net=host in docker 1.13 | Major | yarn | Jim Brennan | Jim Brennan | +| [HADOOP-14999](https://issues.apache.org/jira/browse/HADOOP-14999) | AliyunOSS: provide one asynchronous multi-part based uploading mechanism | Major | fs/oss | Genmao Yu | Genmao Yu | +| [YARN-7810](https://issues.apache.org/jira/browse/YARN-7810) | TestDockerContainerRuntime test failures due to UID lookup of a non-existent user | Major | . | Shane Kumpf | Shane Kumpf | +| [HDFS-13435](https://issues.apache.org/jira/browse/HDFS-13435) | RBF: Improve the error loggings for printing the stack trace | Major | . | Yiqun Lin | Yiqun Lin | +| [YARN-7189](https://issues.apache.org/jira/browse/YARN-7189) | Container-executor doesn't remove Docker containers that error out early | Major | yarn | Eric Badger | Eric Badger | +| [HDFS-13466](https://issues.apache.org/jira/browse/HDFS-13466) | RBF: Add more router-related information to the UI | Minor | . | Wei Yan | Wei Yan | +| [HDFS-13453](https://issues.apache.org/jira/browse/HDFS-13453) | RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table | Major | . | Dibyendu Karmakar | Dibyendu Karmakar | +| [HDFS-13478](https://issues.apache.org/jira/browse/HDFS-13478) | RBF: Disabled Nameservice store API | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13490](https://issues.apache.org/jira/browse/HDFS-13490) | RBF: Fix setSafeMode in the Router | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13484](https://issues.apache.org/jira/browse/HDFS-13484) | RBF: Disable Nameservices from the federation | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13326](https://issues.apache.org/jira/browse/HDFS-13326) | RBF: Improve the interfaces to modify and view mount tables | Minor | . | Wei Yan | Gang Li | +| [HDFS-13499](https://issues.apache.org/jira/browse/HDFS-13499) | RBF: Show disabled name services in the UI | Minor | . | Íñigo Goiri | Íñigo Goiri | +| [YARN-8215](https://issues.apache.org/jira/browse/YARN-8215) | ATS v2 returns invalid YARN\_CONTAINER\_ALLOCATED\_HOST\_HTTP\_ADDRESS from NM | Critical | ATSv2 | Yesha Vora | Rohith Sharma K S | +| [HDFS-13508](https://issues.apache.org/jira/browse/HDFS-13508) | RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries | Minor | . | Ekanth S | Ekanth S | +| [HDFS-13434](https://issues.apache.org/jira/browse/HDFS-13434) | RBF: Fix dead links in RBF document | Major | documentation | Akira Ajisaka | Chetna Chaudhari | +| [YARN-8212](https://issues.apache.org/jira/browse/YARN-8212) | Pending backlog for async allocation threads should be configurable | Major | . | Weiwei Yang | Tao Yang | +| [HDFS-13488](https://issues.apache.org/jira/browse/HDFS-13488) | RBF: Reject requests when a Router is overloaded | Major | . | Íñigo Goiri | Íñigo Goiri | +| [HDFS-13525](https://issues.apache.org/jira/browse/HDFS-13525) | RBF: Add unit test TestStateStoreDisabledNameservice | Major | . | Yiqun Lin | Yiqun Lin | +| [YARN-8253](https://issues.apache.org/jira/browse/YARN-8253) | HTTPS Ats v2 api call fails with "bad HTTP parsed" | Critical | ATSv2 | Yesha Vora | Charan Hebri | +| [HADOOP-15454](https://issues.apache.org/jira/browse/HADOOP-15454) | TestRollingFileSystemSinkWithLocal fails on Windows | Major | test | Xiao Liang | Xiao Liang | +| [YARN-8247](https://issues.apache.org/jira/browse/YARN-8247) | Incorrect HTTP status code returned by ATSv2 for non-whitelisted users | Critical | ATSv2 | Charan Hebri | Rohith Sharma K S | +| [YARN-8130](https://issues.apache.org/jira/browse/YARN-8130) | Race condition when container events are published for KILLED applications | Major | ATSv2 | Charan Hebri | Rohith Sharma K S | +| [HADOOP-15498](https://issues.apache.org/jira/browse/HADOOP-15498) | TestHadoopArchiveLogs (#testGenerateScript, #testPrepareWorkingDir) fails on Windows | Minor | . | Anbang Hu | Anbang Hu | +| [HADOOP-15497](https://issues.apache.org/jira/browse/HADOOP-15497) | TestTrash should use proper test path to avoid failing on Windows | Minor | . | Anbang Hu | Anbang Hu | + + +### OTHER: + +| JIRA | Summary | Priority | Component | Reporter | Contributor | +|:---- |:---- | :--- |:---- |:---- |:---- | +| [HDFS-13052](https://issues.apache.org/jira/browse/HDFS-13052) | WebHDFS: Add support for snasphot diff | Major | . | Lokesh Jain | Lokesh Jain | +| [HADOOP-14742](https://issues.apache.org/jira/browse/HADOOP-14742) | Document multi-URI replication Inode for ViewFS | Major | documentation, viewfs | Chris Douglas | Gera Shegalov | + + diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/RELEASENOTES.3.0.3.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/RELEASENOTES.3.0.3.md new file mode 100644 index 00000000000..9f35dbe15e4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.3/RELEASENOTES.3.0.3.md @@ -0,0 +1,31 @@ + + +# Apache Hadoop 3.0.3 Release Notes + +These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements. + + +--- + +* [HDFS-13099](https://issues.apache.org/jira/browse/HDFS-13099) | *Minor* | **RBF: Use the ZooKeeper as the default State Store** + +Change default State Store from local file to ZooKeeper. This will require additional zk address to be configured. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java index 8debb697171..597eb93b58e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java @@ -51,7 +51,8 @@ public class TestFileSystemStorageStatistics { "bytesReadLocalHost", "bytesReadDistanceOfOneOrTwo", "bytesReadDistanceOfThreeOrFour", - "bytesReadDistanceOfFiveOrLarger" + "bytesReadDistanceOfFiveOrLarger", + "bytesReadErasureCoded" }; private FileSystem.Statistics statistics = @@ -74,6 +75,7 @@ public class TestFileSystemStorageStatistics { statistics.incrementBytesReadByDistance(0, RandomUtils.nextInt(100)); statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(100)); statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(100)); + statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(100)); } @Test @@ -126,6 +128,8 @@ public class TestFileSystemStorageStatistics { return statistics.getBytesReadByDistance(3); case "bytesReadDistanceOfFiveOrLarger": return statistics.getBytesReadByDistance(5); + case "bytesReadErasureCoded": + return statistics.getBytesReadErasureCoded(); default: return 0; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 7218a1bd221..f5571038a6b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -968,6 +968,160 @@ public class TestFileUtil { Assert.assertFalse(link.exists()); } + /** + * This test validates the correctness of + * {@link FileUtil#symLink(String, String)} in case of null pointer inputs. + * + * @throws IOException + */ + @Test + public void testSymlinkWithNullInput() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + File link = new File(del, "_link"); + + // Create the same symbolic link + // The operation should fail and returns 1 + int result = FileUtil.symLink(null, null); + Assert.assertEquals(1, result); + + // Create the same symbolic link + // The operation should fail and returns 1 + result = FileUtil.symLink(file.getAbsolutePath(), null); + Assert.assertEquals(1, result); + + // Create the same symbolic link + // The operation should fail and returns 1 + result = FileUtil.symLink(null, link.getAbsolutePath()); + Assert.assertEquals(1, result); + + file.delete(); + link.delete(); + } + + /** + * This test validates the correctness of + * {@link FileUtil#symLink(String, String)} in case the file already exists. + * + * @throws IOException + */ + @Test + public void testSymlinkFileAlreadyExists() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + File link = new File(del, "_link"); + + // Create a symbolic link + // The operation should succeed + int result1 = + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertEquals(0, result1); + + // Create the same symbolic link + // The operation should fail and returns 1 + result1 = FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertEquals(1, result1); + + file.delete(); + link.delete(); + } + + /** + * This test validates the correctness of + * {@link FileUtil#symLink(String, String)} in case the file and the link are + * the same file. + * + * @throws IOException + */ + @Test + public void testSymlinkSameFile() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + + // Create a symbolic link + // The operation should succeed + int result = + FileUtil.symLink(file.getAbsolutePath(), file.getAbsolutePath()); + + Assert.assertEquals(0, result); + + file.delete(); + } + + /** + * This test validates the correctness of + * {@link FileUtil#symLink(String, String)} in case we want to use a link for + * 2 different files. + * + * @throws IOException + */ + @Test + public void testSymlink2DifferentFile() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + File file = new File(del, FILE); + File fileSecond = new File(del, FILE + "_1"); + File link = new File(del, "_link"); + + // Create a symbolic link + // The operation should succeed + int result = + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertEquals(0, result); + + // The operation should fail and returns 1 + result = + FileUtil.symLink(fileSecond.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertEquals(1, result); + + file.delete(); + fileSecond.delete(); + link.delete(); + } + + /** + * This test validates the correctness of + * {@link FileUtil#symLink(String, String)} in case we want to use a 2 + * different links for the same file. + * + * @throws IOException + */ + @Test + public void testSymlink2DifferentLinks() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + File file = new File(del, FILE); + File link = new File(del, "_link"); + File linkSecond = new File(del, "_link_1"); + + // Create a symbolic link + // The operation should succeed + int result = + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertEquals(0, result); + + // The operation should succeed + result = + FileUtil.symLink(file.getAbsolutePath(), linkSecond.getAbsolutePath()); + + Assert.assertEquals(0, result); + + file.delete(); + link.delete(); + linkSecond.delete(); + } + private void doUntarAndVerify(File tarFile, File untarDir) throws IOException { if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { @@ -1287,4 +1441,56 @@ public class TestFileUtil { } } + /** + * This test validates the correctness of {@link FileUtil#readLink(File)} in + * case of null pointer inputs. + */ + @Test + public void testReadSymlinkWithNullInput() { + String result = FileUtil.readLink(null); + Assert.assertEquals("", result); + } + + /** + * This test validates the correctness of {@link FileUtil#readLink(File)}. + * + * @throws IOException + */ + @Test + public void testReadSymlink() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + File link = new File(del, "_link"); + + // Create a symbolic link + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + String result = FileUtil.readLink(link); + Assert.assertEquals(file.getAbsolutePath(), result); + + file.delete(); + link.delete(); + } + + /** + * This test validates the correctness of {@link FileUtil#readLink(File)} when + * it gets a file in input. + * + * @throws IOException + */ + @Test + public void testReadSymlinkWithAFileAsInput() throws IOException { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + + String result = FileUtil.readLink(file); + Assert.assertEquals("", result); + + file.delete(); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java index c005e77cb3a..df8c54b9cdd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java @@ -230,6 +230,12 @@ public final class RawErasureCoderBenchmark { throw e; } finally { executor.shutdown(); + if (encoder != null) { + encoder.release(); + } + if (decoder != null) { + decoder.release(); + } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestIngressPortBasedResolver.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestIngressPortBasedResolver.java new file mode 100644 index 00000000000..96c80af15f3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestIngressPortBasedResolver.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import javax.security.sasl.Sasl; +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +import static org.junit.Assert.*; + + +/** + * Test class for IngressPortBasedResolver. + */ +public class TestIngressPortBasedResolver { + + /** + * A simple test to test that for the configured ports, the resolver + * can return the current SASL properties. + */ + @Test + public void testResolver() { + Configuration conf = new Configuration(); + conf.set("ingress.port.sasl.configured.ports", "444,555,666,777"); + conf.set("ingress.port.sasl.prop.444", "authentication"); + conf.set("ingress.port.sasl.prop.555", "authentication,privacy"); + conf.set("ingress.port.sasl.prop.666", "privacy"); + + IngressPortBasedResolver resolver = new IngressPortBasedResolver(); + resolver.setConf(conf); + + // the client address does not matter, give it a null + assertEquals("auth", + resolver.getServerProperties(null, 444).get(Sasl.QOP)); + assertEquals("auth,auth-conf", + resolver.getServerProperties(null, 555).get(Sasl.QOP)); + assertEquals("auth-conf", + resolver.getServerProperties(null, 666).get(Sasl.QOP)); + assertEquals("auth-conf", + resolver.getServerProperties(null, 777).get(Sasl.QOP)); + assertEquals("auth", + resolver.getServerProperties(null, 888).get(Sasl.QOP)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java index caa65c5da05..d3848345e08 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java @@ -367,17 +367,31 @@ public class MetricsAsserts { } /** - * Asserts that the NumOps and quantiles for a metric have been changed at - * some point to a non-zero value. + * Asserts that the NumOps and quantiles for a metric with value name + * "Latency" have been changed at some point to a non-zero value. * * @param prefix of the metric * @param rb MetricsRecordBuilder with the metric */ - public static void assertQuantileGauges(String prefix, + public static void assertQuantileGauges(String prefix, MetricsRecordBuilder rb) { + assertQuantileGauges(prefix, rb, "Latency"); + } + + /** + * Asserts that the NumOps and quantiles for a metric have been changed at + * some point to a non-zero value, for the specified value name of the + * metrics (e.g., "Latency", "Count"). + * + * @param prefix of the metric + * @param rb MetricsRecordBuilder with the metric + * @param valueName the value name for the metric + */ + public static void assertQuantileGauges(String prefix, + MetricsRecordBuilder rb, String valueName) { verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l)); for (Quantile q : MutableQuantiles.quantiles) { - String nameTemplate = prefix + "%dthPercentileLatency"; + String nameTemplate = prefix + "%dthPercentile" + valueName; int percentile = (int) (100 * q.quantile); verify(rb).addGauge( eqName(info(String.format(nameTemplate, percentile), "")), diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java new file mode 100644 index 00000000000..31e1899421f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.nio.channels.ClosedChannelException; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestCloseableReferenceCount extends HadoopTestBase { + @Test + public void testReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testUnreference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.reference(); + assertFalse("New reference count should not equal STATUS_CLOSED_MASK", + clr.unreference()); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testUnreferenceCheckClosed() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.reference(); + clr.unreferenceCheckClosed(); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testSetClosed() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + assertTrue("Reference count should be open", clr.isOpen()); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + } + + @Test(expected = ClosedChannelException.class) + public void testReferenceClosedReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.reference(); + } + + @Test(expected = ClosedChannelException.class) + public void testUnreferenceClosedReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.unreferenceCheckClosed(); + } + + @Test(expected = ClosedChannelException.class) + public void testDoubleClose() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + assertTrue("Reference count should be open", clr.isOpen()); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.setClosed(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index 6b6c6c843dd..e92c9edb4fd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -137,7 +137,8 @@ public class TestDiskChecker { * @throws java.io.IOException if any */ protected File createTempFile() throws java.io.IOException { - File testDir = new File(System.getProperty("test.build.data")); + File testDir = + new File(System.getProperty("test.build.data", "target/test-dir")); return Files.createTempFile(testDir.toPath(), "test", "tmp").toFile(); } @@ -147,7 +148,8 @@ public class TestDiskChecker { * @throws java.io.IOException if any */ protected File createTempDir() throws java.io.IOException { - File testDir = new File(System.getProperty("test.build.data")); + File testDir = + new File(System.getProperty("test.build.data", "target/test-dir")); return Files.createTempDirectory(testDir.toPath(), "test").toFile(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java new file mode 100644 index 00000000000..03bbf7b12fe --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * 
+ * Story 1
+ * As a software developer,
+ *  I want to use the IntrusiveCollection class;
+ * So that I can save on memory usage during execution.
+ *
+ */ +package org.apache.hadoop.util; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; +import org.apache.hadoop.util.IntrusiveCollection.Element; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestIntrusiveCollection extends HadoopTestBase { + static class SimpleElement implements IntrusiveCollection.Element { + private Map, Element> + prevMap, nextMap; + private Map, Boolean> isMemberMap; + + public SimpleElement() { + prevMap = new HashMap<>(); + nextMap = new HashMap<>(); + isMemberMap = new HashMap<>(); + } + + @Override + public void insertInternal(IntrusiveCollection list, + Element prev, Element next) { + isMemberMap.put(list, true); + prevMap.put(list, prev); + nextMap.put(list, next); + } + + @Override + public void setPrev(IntrusiveCollection list, + Element prev) { + prevMap.put(list, prev); + } + + @Override + public void setNext(IntrusiveCollection list, + Element next) { + nextMap.put(list, next); + } + + @Override + public void removeInternal(IntrusiveCollection list) { + prevMap.remove(list); + nextMap.remove(list); + isMemberMap.remove(list); + } + + @Override + public Element getPrev(IntrusiveCollection list) { + return prevMap.getOrDefault(list, null); + } + + @Override + public Element getNext(IntrusiveCollection list) { + return nextMap.getOrDefault(list, null); + } + + @Override + public boolean isInList(IntrusiveCollection list) { + return isMemberMap.getOrDefault(list, false); + } + } + + /** + * 
+   * Scenario S1.1: Adding an element
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection is empty
+   * When    I insert an element
+   * Then    the IntrusiveCollection contains the newly added element.
+   *
+ */ + @Test + public void testShouldAddElement() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + + SimpleElement element = new SimpleElement(); + intrusiveCollection.add(element); + + assertFalse("Collection should not be empty", + intrusiveCollection.isEmpty()); + assertTrue("Collection should contain added element", + intrusiveCollection.contains(element)); + } + + /** + * 
+   * Scenario S1.2: Removing an element
+   * Given  an IntrusiveCollection has been created
+   *  and    the InstrusiveCollection contains a single element
+   * When    I remove the element
+   * Then    the IntrusiveCollection is empty.
+   *
+ */ + @Test + public void testShouldRemoveElement() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + SimpleElement element = new SimpleElement(); + intrusiveCollection.add(element); + + intrusiveCollection.remove(element); + + assertTrue("Collection should be empty", intrusiveCollection.isEmpty()); + assertFalse("Collection should not contain removed element", + intrusiveCollection.contains(element)); + } + + /** + * 
+   * Scenario S1.3: Removing all elements
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection contains multiple elements
+   * When    I remove all elements
+   * Then    the IntrusiveCollection is empty.
+   *
+ */ + @Test + public void testShouldRemoveAllElements() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + intrusiveCollection.add(new SimpleElement()); + intrusiveCollection.add(new SimpleElement()); + intrusiveCollection.add(new SimpleElement()); + + intrusiveCollection.clear(); + + assertTrue("Collection should be empty", intrusiveCollection.isEmpty()); + } + + /** + * 
+   * Scenario S1.4: Iterating through elements
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection contains multiple elements
+   * When    I iterate through the IntrusiveCollection
+   * Then    I get each element in the collection, successively.
+   *
+ */ + @Test + public void testIterateShouldReturnAllElements() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + SimpleElement elem1 = new SimpleElement(); + SimpleElement elem2 = new SimpleElement(); + SimpleElement elem3 = new SimpleElement(); + intrusiveCollection.add(elem1); + intrusiveCollection.add(elem2); + intrusiveCollection.add(elem3); + + Iterator iterator = intrusiveCollection.iterator(); + + assertEquals("First element returned is incorrect", elem1, iterator.next()); + assertEquals("Second element returned is incorrect", elem2, + iterator.next()); + assertEquals("Third element returned is incorrect", elem3, iterator.next()); + assertFalse("Iterator should not have next element", iterator.hasNext()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java new file mode 100644 index 00000000000..368fa37b7bd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Random; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class TestLimitInputStream extends HadoopTestBase { + static class RandomInputStream extends InputStream { + private Random rn = new Random(0); + + @Override + public int read() { return rn.nextInt(); } + } + + @Test + public void testRead() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 0)) { + assertEquals("Reading byte after reaching limit should return -1", -1, + limitInputStream.read()); + } + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 4)) { + assertEquals("Incorrect byte returned", new Random(0).nextInt(), + limitInputStream.read()); + } + } + + @Test(expected = IOException.class) + public void testResetWithoutMark() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 128)) { + limitInputStream.reset(); + } + } + + @Test + public void testReadBytes() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 128)) { + Random r = new Random(0); + byte[] data = new byte[4]; + byte[] expected = { (byte) r.nextInt(), (byte) r.nextInt(), + (byte) r.nextInt(), (byte) r.nextInt() }; + limitInputStream.read(data, 0, 4); + assertArrayEquals("Incorrect bytes returned", expected, data); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index d0ebc2b83f7..578d2671141 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -38,6 +39,8 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.util.Shell.*; +import static org.junit.Assume.assumeTrue; + import org.junit.Assume; import org.junit.Before; import org.junit.Rule; @@ -528,4 +531,9 @@ public class TestShell extends Assert { public void testIsJavaVersionAtLeast() { assertTrue(Shell.isJavaVersionAtLeast(8)); } + + @Test + public void testIsBashSupported() throws InterruptedIOException { + assumeTrue("Bash is not supported", Shell.checkIsBashSupported()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 96a64823639..3fdc1bb8f8b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.util.ArrayList; @@ -476,6 +477,32 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { executorService.awaitTermination(50, TimeUnit.SECONDS); } + @Test + public void testFormatTimeSortable() { + long timeDiff = 523452311; + String timeDiffStr = "99hrs, 59mins, 59sec"; + + assertEquals("Incorrect time diff string returned", timeDiffStr, + StringUtils.formatTimeSortable(timeDiff)); + } + + @Test + public void testIsAlpha() { + assertTrue("Reported hello as non-alpha string", + StringUtils.isAlpha("hello")); + assertFalse("Reported hello1 as alpha string", + StringUtils.isAlpha("hello1")); + } + + @Test + public void testEscapeHTML() { + String htmlStr = "

Hello. How are you?

"; + String escapedStr = "<p>Hello. How are you?</p>"; + + assertEquals("Incorrect escaped HTML string returned", + escapedStr, StringUtils.escapeHTML(htmlStr)); + } + // Benchmark for StringUtils split public static void main(String []args) { final String TO_SPLIT = "foo,bar,baz,blah,blah"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java new file mode 100644 index 00000000000..3aa549a4ca4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertEquals; + +public class TestUTF8ByteArrayUtils extends HadoopTestBase { + @Test + public void testFindByte() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Character 'a' does not exist in string", -1, + UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'a')); + assertEquals("Did not find first occurrence of character 'o'", 4, + UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'o')); + } + + @Test + public void testFindBytes() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Did not find first occurrence of pattern 'ello'", 1, + UTF8ByteArrayUtils.findBytes(data, 0, data.length, "ello".getBytes())); + assertEquals( + "Substring starting at position 2 does not contain pattern 'ello'", -1, + UTF8ByteArrayUtils.findBytes(data, 2, data.length, "ello".getBytes())); + } + + @Test + public void testFindNthByte() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Did not find 2nd occurrence of character 'l'", 3, + UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 2)); + assertEquals("4th occurrence of character 'l' does not exist", -1, + UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 4)); + assertEquals("Did not find 3rd occurrence of character 'l'", 10, + UTF8ByteArrayUtils.findNthByte(data, (byte) 'l', 3)); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_daemon.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_daemon.bats index 023d01c02c7..148380706db 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_daemon.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_daemon.bats @@ -15,7 +15,7 @@ load hadoop-functions_test_helper -@test "hadoop_stop_daemon" { +@test "hadoop_stop_daemon_changing_pid" { old_pid=12345 new_pid=54321 HADOOP_STOP_TIMEOUT=3 @@ -29,3 +29,25 @@ load hadoop-functions_test_helper [ -f pidfile ] [ "$(cat pidfile)" = "${new_pid}" ] } + +@test "hadoop_stop_daemon_force_kill" { + + HADOOP_STOP_TIMEOUT=4 + + # Run the following in a sub-shell so that its termination doesn't affect the test + (sh ${TESTBINDIR}/process_with_sigterm_trap.sh ${TMP}/pidfile &) + + # Wait for the process to go into tight loop + sleep 1 + + [ -f ${TMP}/pidfile ] + pid=$(cat "${TMP}/pidfile") + + run hadoop_stop_daemon my_command ${TMP}/pidfile 2>&1 + + # The process should no longer be alive + ! kill -0 ${pid} > /dev/null 2>&1 + + # The PID file should be gone + [ ! -f ${TMP}/pidfile ] +} diff --git a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml index 7055f2df615..9f4171bf793 100644 --- a/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml +++ b/hadoop-common-project/hadoop-kms/src/main/resources/kms-default.xml @@ -259,4 +259,39 @@ + + hadoop.kms.key.authorization.enable + true + Boolean property to Enable/Disable per Key authorization + + + + hadoop.security.kms.encrypted.key.cache.size + 100 + The size of the cache. This is the maximum number of EEKs that + can be cached under each key name. + + + + hadoop.security.kms.encrypted.key.cache.low.watermark + 0.3 + A low watermark on the cache. For each key name, if after a get call, + the number of cached EEKs are less than (size * low watermark), + then the cache under this key name will be filled asynchronously. + For each key name, only 1 thread could be running for the asynchronous filling. + + + + hadoop.security.kms.encrypted.key.cache.num.fill.threads + 2 + The maximum number of asynchronous threads overall, across key names, + allowed to fill the queue in a cache. + + + + hadoop.security.kms.encrypted.key.cache.expiry + 43200000 + The cache expiry time, in milliseconds. Internally Guava cache is used as the cache implementation. + The expiry approach is expireAfterAccess + diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Verifier.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Verifier.java index f3202a1d33c..3c0e5fe36e2 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Verifier.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/security/Verifier.java @@ -41,12 +41,18 @@ public abstract class Verifier extends RpcAuthInfo { public static Verifier readFlavorAndVerifier(XDR xdr) { AuthFlavor flavor = AuthFlavor.fromValue(xdr.readInt()); final Verifier verifer; - if(flavor == AuthFlavor.AUTH_NONE) { + if (flavor == AuthFlavor.AUTH_NONE) { verifer = new VerifierNone(); - } else if(flavor == AuthFlavor.RPCSEC_GSS) { + } else if (flavor == AuthFlavor.AUTH_SYS) { + // Added in HADOOP-15307 based on HDFS-5085: + // When the auth flavor is AUTH_SYS, the corresponding verifier is + // AUTH_NONE. I.e., it is impossible to have a verifier with auth + // flavor AUTH_SYS. + verifer = new VerifierNone(); + } else if (flavor == AuthFlavor.RPCSEC_GSS) { verifer = new VerifierGSS(); } else { - throw new UnsupportedOperationException("Unsupported verifier flavor" + throw new UnsupportedOperationException("Unsupported verifier flavor: " + flavor); } verifer.read(xdr); diff --git a/hadoop-dist/src/main/compose/ozoneperf/.env b/hadoop-dist/src/main/compose/ozoneperf/.env new file mode 100644 index 00000000000..cac418ae59e --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/.env @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} diff --git a/hadoop-dist/src/main/compose/ozoneperf/README.md b/hadoop-dist/src/main/compose/ozoneperf/README.md new file mode 100644 index 00000000000..a78f2087328 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/README.md @@ -0,0 +1,73 @@ + + +# Compose files for local performance tests + +This directory contains docker-compose definition for an ozone cluster where +all the metrics are saved to a prometheus instance. + + Prometheus follows a pull based approach where the metrics are published + on a HTTP endpoint. + + Our current approach: + + 1. A Java agent activates a prometheus metrics endpoint in every JVM instance + (use `init.sh` to download the agent) + + 2. The Java agent publishes all the jmx parameters in prometheus format AND + register the endpoint address to the consul. + + 3. Prometheus polls all the endpoints which are registered to consul. + + + +## How to use + +First of all download the required Java agent with running `./init.sh` + +After that you can start the cluster with docker-compose: + +``` +docker-compose up -d +``` + +After a while the cluster will be started. You can check the ozone web ui-s: + +https://localhost:9874 +https://localhost:9876 + +You can also scale up the datanodes: + +``` +docker-compose scale datanode=3 +``` + +Freon (Ozone test generator tool) is not part of docker-compose by default, +you can activate it using `compose-all.sh` instead of `docker-compose`: + +``` +compose-all.sh up -d +``` + +Now Freon is running. Let's try to check the metrics from the local Prometheus: + +http://localhost:9090/graph + +Example queries: + +``` +Hadoop_KeySpaceManager_NumKeyCommits +rate(Hadoop_KeySpaceManager_NumKeyCommits[10m]) +rate(Hadoop_Ozone_BYTES_WRITTEN[10m]) +``` diff --git a/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh new file mode 100755 index 00000000000..82ab8b3101d --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/compose-all.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker-compose -f docker-compose.yaml -f docker-compose-freon.yaml "$@" diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml new file mode 100644 index 00000000000..60bdc4a503f --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose-freon.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + freon: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","freon"] diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml new file mode 100644 index 00000000000..fb7873bf880 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + namenode: + image: apache/hadoop-runner + hostname: namenode + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9870:9870 + environment: + ENSURE_NAMENODE_DIR: /data/namenode + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/hdfs","namenode"] + datanode: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9864 + command: ["/opt/hadoop/bin/ozone","datanode"] + env_file: + - ./docker-config + ksm: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9874:9874 + environment: + ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION + env_file: + - ./docker-config + command: ["/opt/hadoop/bin/ozone","ksm"] + scm: + image: apache/hadoop-runner + volumes: + - ../../ozone:/opt/hadoop + - ./jmxpromo.jar:/opt/jmxpromo.jar + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["/opt/hadoop/bin/ozone","scm"] + consul: + image: consul + command: ["agent", "-dev", "-ui", "-client", "0.0.0.0"] + ports: + - 8500:8500 + prometheus: + image: prom/prometheus + volumes: + - "./prometheus.yml:/etc/prometheus.yml" + command: ["--config.file","/etc/prometheus.yml"] + ports: + - 9090:9090 diff --git a/hadoop-dist/src/main/compose/ozoneperf/docker-config b/hadoop-dist/src/main/compose/ozoneperf/docker-config new file mode 100644 index 00000000000..e4f5485ac57 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/docker-config @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 +OZONE-SITE.XML_ozone.ksm.address=ksm +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=True +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.handler.type=distributed +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 +HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +HADOOP_OPTS=-javaagent:/opt/jmxpromo.jar=port=0:consulHost=consul:consulMode=node + diff --git a/hadoop-dist/src/main/compose/ozoneperf/init.sh b/hadoop-dist/src/main/compose/ozoneperf/init.sh new file mode 100755 index 00000000000..cf25398bc20 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/init.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +EXPORTER_FILE="$DIR/jmxpromo.jar" +if [ ! -f "$EXPORTER_FILE" ]; then + wget https://github.com/flokkr/jmxpromo/releases/download/0.11/jmx_prometheus_javaagent-0.11.jar -O $EXPORTER_FILE +fi diff --git a/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml new file mode 100644 index 00000000000..80aa5203a20 --- /dev/null +++ b/hadoop-dist/src/main/compose/ozoneperf/prometheus.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + +scrape_configs: + - job_name: jmxexporter + consul_sd_configs: + - server: consul:8500 + services: + - jmxexporter diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index c78702486fb..92df46e492f 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@ -55,6 +55,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { private XceiverClientMetrics metrics; private ManagedChannel channel; private final Semaphore semaphore; + private boolean closed = false; /** * Constructs a client that can communicate with the Container framework on @@ -105,6 +106,7 @@ public class XceiverClientGrpc extends XceiverClientSpi { @Override public void close() { + closed = true; channel.shutdownNow(); try { channel.awaitTermination(60, TimeUnit.MINUTES); @@ -153,6 +155,14 @@ public class XceiverClientGrpc extends XceiverClientSpi { public CompletableFuture sendCommandAsync(ContainerCommandRequestProto request) throws IOException, ExecutionException, InterruptedException { + if(closed){ + throw new IOException("This channel is not connected."); + } + + if(channel == null || !isConnected()) { + reconnect(); + } + final CompletableFuture replyFuture = new CompletableFuture<>(); semaphore.acquire(); @@ -192,6 +202,19 @@ public class XceiverClientGrpc extends XceiverClientSpi { return replyFuture; } + private void reconnect() throws IOException { + try { + connect(); + } catch (Exception e) { + LOG.error("Error while connecting: ", e); + throw new IOException(e); + } + + if (channel == null || !isConnected()) { + throw new IOException("This channel is not connected."); + } + } + /** * Create a pipeline. * diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java index 6a2286c523f..7c568f6b879 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientHandler.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.ratis.shaded.io.netty.channel.Channel; import org.apache.ratis.shaded.io.netty.channel.ChannelHandlerContext; import org.apache.ratis.shaded.io.netty.channel.SimpleChannelInboundHandler; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java index 7bf8f01ee34..62b12e3e041 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.hdds.client; -import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java index 521408b7016..677b752bd9c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java @@ -39,6 +39,8 @@ import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY; + /** * A servlet to print out the running configuration data. */ @@ -154,7 +156,8 @@ public class HddsConfServlet extends HttpServlet { switch (cmd) { case "getOzoneTags": - out.write(gson.toJson(config.get("ozone.tags.system").split(","))); + out.write(gson.toJson(config.get(OZONE_TAGS_SYSTEM_KEY) + .split(","))); break; case "getPropertyByTag": String tags = request.getParameter("tags"); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java index 10fd96c175a..2c38d457284 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerInfo.java @@ -32,6 +32,8 @@ import org.apache.hadoop.util.Time; import java.io.IOException; import java.util.Comparator; +import static java.lang.Math.max; + /** * Class wraps ozone container info. */ @@ -60,6 +62,7 @@ public class ContainerInfo private long stateEnterTime; private String owner; private long containerID; + private long deleteTransactionId; ContainerInfo( long containerID, HddsProtos.LifeCycleState state, @@ -68,7 +71,8 @@ public class ContainerInfo long usedBytes, long numberOfKeys, long stateEnterTime, - String owner) { + String owner, + long deleteTransactionId) { this.containerID = containerID; this.pipeline = pipeline; this.allocatedBytes = allocatedBytes; @@ -78,6 +82,7 @@ public class ContainerInfo this.state = state; this.stateEnterTime = stateEnterTime; this.owner = owner; + this.deleteTransactionId = deleteTransactionId; } /** @@ -96,6 +101,7 @@ public class ContainerInfo builder.setStateEnterTime(info.getStateEnterTime()); builder.setOwner(info.getOwner()); builder.setContainerID(info.getContainerID()); + builder.setDeleteTransactionId(info.getDeleteTransactionId()); return builder.build(); } @@ -141,6 +147,14 @@ public class ContainerInfo return numberOfKeys; } + public long getDeleteTransactionId() { + return deleteTransactionId; + } + + public void updateDeleteTransactionId(long transactionId) { + deleteTransactionId = max(transactionId, deleteTransactionId); + } + public ContainerID containerID() { return new ContainerID(getContainerID()); } @@ -174,6 +188,7 @@ public class ContainerInfo builder.setState(state); builder.setStateEnterTime(stateEnterTime); builder.setContainerID(getContainerID()); + builder.setDeleteTransactionId(deleteTransactionId); if (getOwner() != null) { builder.setOwner(getOwner()); @@ -292,6 +307,7 @@ public class ContainerInfo private long stateEnterTime; private String owner; private long containerID; + private long deleteTransactionId; public Builder setContainerID(long id) { Preconditions.checkState(id >= 0); @@ -334,10 +350,15 @@ public class ContainerInfo return this; } + public Builder setDeleteTransactionId(long deleteTransactionId) { + this.deleteTransactionId = deleteTransactionId; + return this; + } + public ContainerInfo build() { return new - ContainerInfo(containerID, state, pipeline, - allocated, used, keys, stateEnterTime, owner); + ContainerInfo(containerID, state, pipeline, allocated, + used, keys, stateEnterTime, owner, deleteTransactionId); } } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index d1377becf96..856d088c792 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -31,6 +31,8 @@ import org.apache.ratis.util.TimeDuration; @InterfaceAudience.Public @InterfaceStability.Unstable public final class OzoneConfigKeys { + public static final String OZONE_TAGS_SYSTEM_KEY = + "ozone.tags.system"; public static final String DFS_CONTAINER_IPC_PORT = "dfs.container.ipc"; public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 451a08f6089..c40dc8e4ee1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -114,6 +114,8 @@ public final class OzoneConsts { public static final String OZONE_HANDLER_LOCAL = "local"; public static final String DELETING_KEY_PREFIX = "#deleting#"; + public static final String DELETED_KEY_PREFIX = "#deleted#"; + public static final String DELETE_TRANSACTION_KEY_PREFIX = "#delTX#"; public static final String OPEN_KEY_PREFIX = "#open#"; public static final String OPEN_KEY_ID_DELIMINATOR = "#"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java index d3a29435de3..a3430f868d0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java @@ -19,19 +19,30 @@ package org.apache.hadoop.utils; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.ozone.OzoneConsts; +import java.util.ArrayList; +import java.util.List; + /** * An utility class to filter levelDB keys. */ public final class MetadataKeyFilters { private static KeyPrefixFilter deletingKeyFilter = - new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX); + new MetadataKeyFilters.KeyPrefixFilter() + .addFilter(OzoneConsts.DELETING_KEY_PREFIX); + + private static KeyPrefixFilter deletedKeyFilter = + new MetadataKeyFilters.KeyPrefixFilter() + .addFilter(OzoneConsts.DELETED_KEY_PREFIX); private static KeyPrefixFilter normalKeyFilter = - new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX, - true); + new MetadataKeyFilters.KeyPrefixFilter() + .addFilter(OzoneConsts.DELETING_KEY_PREFIX, true) + .addFilter(OzoneConsts.DELETED_KEY_PREFIX, true) + .addFilter(OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX, true); private MetadataKeyFilters() { } @@ -40,6 +51,10 @@ public final class MetadataKeyFilters { return deletingKeyFilter; } + public static KeyPrefixFilter getDeletedKeyFilter() { + return deletedKeyFilter; + } + public static KeyPrefixFilter getNormalKeyFilter() { return normalKeyFilter; } @@ -72,37 +87,95 @@ public final class MetadataKeyFilters { */ public static class KeyPrefixFilter implements MetadataKeyFilter { - private String keyPrefix = null; + private List positivePrefixList = new ArrayList<>(); + private List negativePrefixList = new ArrayList<>(); + private boolean atleastOnePositiveMatch; private int keysScanned = 0; private int keysHinted = 0; - private Boolean negative; - public KeyPrefixFilter(String keyPrefix) { - this(keyPrefix, false); + public KeyPrefixFilter() {} + + /** + * KeyPrefixFilter constructor. It is made of positive and negative prefix + * list. PositivePrefixList is the list of prefixes which are accepted + * whereas negativePrefixList contains the list of prefixes which are + * rejected. + * + * @param atleastOnePositiveMatch if positive it requires key to be accepted + * by atleast one positive filter. + */ + public KeyPrefixFilter(boolean atleastOnePositiveMatch) { + this.atleastOnePositiveMatch = atleastOnePositiveMatch; } - public KeyPrefixFilter(String keyPrefix, boolean negative) { - this.keyPrefix = keyPrefix; - this.negative = negative; + public KeyPrefixFilter addFilter(String keyPrefix) { + addFilter(keyPrefix, false); + return this; + } + + public KeyPrefixFilter addFilter(String keyPrefix, boolean negative) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(keyPrefix), + "KeyPrefix is null or empty: " + keyPrefix); + // keyPrefix which needs to be added should not be prefix of any opposing + // filter already present. If keyPrefix is a negative filter it should not + // be a prefix of any positive filter. Nor should any opposing filter be + // a prefix of keyPrefix. + // For example if b0 is accepted b can not be rejected and + // if b is accepted b0 can not be rejected. If these scenarios need to be + // handled we need to add priorities. + if (negative) { + Preconditions.checkArgument(positivePrefixList.stream().noneMatch( + prefix -> prefix.startsWith(keyPrefix) || keyPrefix + .startsWith(prefix)), + "KeyPrefix: " + keyPrefix + " already accepted."); + this.negativePrefixList.add(keyPrefix); + } else { + Preconditions.checkArgument(negativePrefixList.stream().noneMatch( + prefix -> prefix.startsWith(keyPrefix) || keyPrefix + .startsWith(prefix)), + "KeyPrefix: " + keyPrefix + " already rejected."); + this.positivePrefixList.add(keyPrefix); + } + return this; } @Override public boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey) { keysScanned++; - boolean accept = false; - if (Strings.isNullOrEmpty(keyPrefix)) { - accept = true; - } else { - byte [] prefixBytes = keyPrefix.getBytes(); - if (currentKey != null && prefixMatch(prefixBytes, currentKey)) { - keysHinted++; - accept = true; - } else { - accept = false; - } + if (currentKey == null) { + return false; } - return (negative) ? !accept : accept; + boolean accept; + + // There are no filters present + if (positivePrefixList.isEmpty() && negativePrefixList.isEmpty()) { + return true; + } + + accept = !positivePrefixList.isEmpty() && positivePrefixList.stream() + .anyMatch(prefix -> { + byte[] prefixBytes = DFSUtil.string2Bytes(prefix); + return prefixMatch(prefixBytes, currentKey); + }); + if (accept) { + keysHinted++; + return true; + } else if (atleastOnePositiveMatch) { + return false; + } + + accept = !negativePrefixList.isEmpty() && negativePrefixList.stream() + .allMatch(prefix -> { + byte[] prefixBytes = DFSUtil.string2Bytes(prefix); + return !prefixMatch(prefixBytes, currentKey); + }); + if (accept) { + keysHinted++; + return true; + } + + return false; } @Override @@ -115,7 +188,7 @@ public final class MetadataKeyFilters { return keysHinted; } - private boolean prefixMatch(byte[] prefix, byte[] key) { + private static boolean prefixMatch(byte[] prefix, byte[] key) { Preconditions.checkNotNull(prefix); Preconditions.checkNotNull(key); if (key.length < prefix.length) { diff --git a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java index 20356b39805..df831161ca4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java +++ b/hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java @@ -49,12 +49,12 @@ public interface RatisHelper { static String toRaftPeerIdString(DatanodeDetails id) { return id.getUuidString() + "_" + - id.getPort(DatanodeDetails.Port.Name.RATIS); + id.getPort(DatanodeDetails.Port.Name.RATIS).getValue(); } static String toRaftPeerAddressString(DatanodeDetails id) { return id.getIpAddress() + ":" + - id.getPort(DatanodeDetails.Port.Name.RATIS); + id.getPort(DatanodeDetails.Port.Name.RATIS).getValue(); } static RaftPeerId toRaftPeerId(DatanodeDetails id) { diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index f834c73e59b..a9a703eb000 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -146,6 +146,7 @@ message SCMContainerInfo { required uint64 numberOfKeys = 6; optional int64 stateEnterTime = 7; required string owner = 8; + optional int64 deleteTransactionId = 9; } message GetScmInfoRequestProto { diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java index 06db1e15709..d697bbfe770 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/TestMetadataStore.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.utils; +import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows; + import com.google.common.collect.Lists; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -81,6 +83,11 @@ public class TestMetadataStore { @Before public void init() throws IOException { + if (OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(storeImpl)) { + // The initialization of RocksDB fails on Windows + assumeNotWindows(); + } + testDir = GenericTestUtils.getTestDir(getClass().getSimpleName() + "-" + storeImpl.toLowerCase()); @@ -153,9 +160,13 @@ public class TestMetadataStore { @After public void cleanup() throws IOException { - store.close(); - store.destroy(); - FileUtils.deleteDirectory(testDir); + if (store != null) { + store.close(); + store.destroy(); + } + if (testDir != null) { + FileUtils.deleteDirectory(testDir); + } } private byte[] getBytes(String str) { @@ -313,7 +324,7 @@ public class TestMetadataStore { // Filter keys by prefix. // It should returns all "b*" entries. - MetadataKeyFilter filter1 = new KeyPrefixFilter("b"); + MetadataKeyFilter filter1 = new KeyPrefixFilter().addFilter("b"); result = store.getRangeKVs(null, 100, filter1); Assert.assertEquals(10, result.size()); Assert.assertTrue(result.stream().allMatch(entry -> @@ -460,4 +471,63 @@ public class TestMetadataStore { Assert.assertEquals(8, count.get()); } + + @Test + public void testKeyPrefixFilter() throws IOException { + List> result = null; + RuntimeException exception = null; + + try { + new KeyPrefixFilter().addFilter("b0", true).addFilter("b"); + } catch (IllegalArgumentException e) { + exception = e; + } + Assert.assertTrue( + exception.getMessage().contains("KeyPrefix: b already rejected")); + + try { + new KeyPrefixFilter().addFilter("b0").addFilter("b", true); + } catch (IllegalArgumentException e) { + exception = e; + } + Assert.assertTrue( + exception.getMessage().contains("KeyPrefix: b already accepted")); + + try { + new KeyPrefixFilter().addFilter("b", true).addFilter("b0"); + } catch (IllegalArgumentException e) { + exception = e; + } + Assert.assertTrue( + exception.getMessage().contains("KeyPrefix: b0 already rejected")); + + try { + new KeyPrefixFilter().addFilter("b").addFilter("b0", true); + } catch (IllegalArgumentException e) { + exception = e; + } + Assert.assertTrue( + exception.getMessage().contains("KeyPrefix: b0 already accepted")); + + MetadataKeyFilter filter1 = new KeyPrefixFilter(true) + .addFilter("a0") + .addFilter("a1") + .addFilter("b", true); + result = store.getRangeKVs(null, 100, filter1); + Assert.assertEquals(2, result.size()); + Assert.assertTrue(result.stream() + .anyMatch(entry -> new String(entry.getKey()).startsWith("a0")) + && result.stream() + .anyMatch(entry -> new String(entry.getKey()).startsWith("a1"))); + + filter1 = new KeyPrefixFilter(true).addFilter("b", true); + result = store.getRangeKVs(null, 100, filter1); + Assert.assertEquals(0, result.size()); + + filter1 = new KeyPrefixFilter().addFilter("b", true); + result = store.getRangeKVs(null, 100, filter1); + Assert.assertEquals(10, result.size()); + Assert.assertTrue(result.stream() + .allMatch(entry -> new String(entry.getKey()).startsWith("a"))); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java index d1746f2bde6..5767f76b3dc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java @@ -33,6 +33,8 @@ import java.util.Map; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicLong; +import static java.lang.Math.max; + /** * This class maintains the information about a container in the ozone world. *

@@ -57,6 +59,7 @@ public class ContainerData { * Number of pending deletion blocks in container. */ private int numPendingDeletionBlocks; + private long deleteTransactionId; private AtomicLong readBytes; private AtomicLong writeBytes; private AtomicLong readCount; @@ -78,6 +81,7 @@ public class ContainerData { this.containerID = containerID; this.state = ContainerLifeCycleState.OPEN; this.numPendingDeletionBlocks = 0; + this.deleteTransactionId = 0; this.readCount = new AtomicLong(0L); this.readBytes = new AtomicLong(0L); this.writeCount = new AtomicLong(0L); @@ -101,6 +105,7 @@ public class ContainerData { this.containerID = containerID; this.state = state; this.numPendingDeletionBlocks = 0; + this.deleteTransactionId = 0; this.readCount = new AtomicLong(0L); this.readBytes = new AtomicLong(0L); this.writeCount = new AtomicLong(0L); @@ -285,7 +290,8 @@ public class ContainerData { * * @return String Name. */ - // TODO: check the ContainerCache class to see if we are using the ContainerID instead. + // TODO: check the ContainerCache class to see if + // we are using the ContainerID instead. /* public String getName() { return getContainerID(); @@ -424,6 +430,22 @@ public class ContainerData { return this.numPendingDeletionBlocks; } + /** + * Sets deleteTransactionId to latest delete transactionId for the container. + * + * @param transactionId latest transactionId of the container. + */ + public void updateDeleteTransactionId(long transactionId) { + deleteTransactionId = max(transactionId, deleteTransactionId); + } + + /** + * Return the latest deleteTransactionId of the container. + */ + public long getDeleteTransactionId() { + return deleteTransactionId; + } + /** * Get the number of bytes read from the container. * @return the number of bytes read from the container. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java index 19634f48b81..b2427549cf6 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java @@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.container.common.helpers; import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo; +import static java.lang.Math.max; + /** * Container Report iterates the closed containers and sends a container report * to SCM. @@ -35,6 +37,7 @@ public class ContainerReport { private long readBytes; private long writeBytes; private long containerID; + private long deleteTransactionId; public long getContainerID() { return containerID; @@ -63,6 +66,7 @@ public class ContainerReport { this.readBytes = 0L; this.writeCount = 0L; this.writeBytes = 0L; + this.deleteTransactionId = 0; } /** @@ -96,6 +100,9 @@ public class ContainerReport { if (info.hasWriteBytes()) { report.setWriteBytes(info.getWriteBytes()); } + if (info.hasDeleteTransactionId()) { + report.updateDeleteTransactionId(info.getDeleteTransactionId()); + } report.setContainerID(info.getContainerID()); return report; @@ -186,6 +193,10 @@ public class ContainerReport { this.bytesUsed = bytesUsed; } + public void updateDeleteTransactionId(long transactionId) { + this.deleteTransactionId = max(transactionId, deleteTransactionId); + } + /** * Gets a containerInfo protobuf message from ContainerReports. * @@ -202,6 +213,7 @@ public class ContainerReport { .setWriteBytes(this.getWriteBytes()) .setFinalhash(this.getFinalhash()) .setContainerID(this.getContainerID()) + .setDeleteTransactionId(this.deleteTransactionId) .build(); } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index af470153244..02572a8fa2b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -20,11 +20,13 @@ package org.apache.hadoop.ozone.container.common.impl; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.primitives.Longs; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; @@ -53,8 +55,6 @@ import org.apache.hadoop.ozone.container.common.interfaces import org.apache.hadoop.ozone.container.common.interfaces .ContainerLocationManager; import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager; -import org.apache.hadoop.ozone.container.common.interfaces - .ContainerReportManager; import org.apache.hadoop.ozone.container.common.interfaces.KeyManager; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.utils.MetadataKeyFilters; @@ -127,10 +127,8 @@ public class ContainerManagerImpl implements ContainerManager { private ChunkManager chunkManager; private KeyManager keyManager; private Configuration conf; - private DatanodeDetails datanodeDetails; private ContainerDeletionChoosingPolicy containerDeletionChooser; - private ContainerReportManager containerReportManager; /** * Init call that sets up a container Manager. @@ -154,7 +152,6 @@ public class ContainerManagerImpl implements ContainerManager { " directories must be greater than zero."); this.conf = config; - this.datanodeDetails = dnDetails; readLock(); try { @@ -203,9 +200,6 @@ public class ContainerManagerImpl implements ContainerManager { } this.locationManager = new ContainerLocationManagerImpl(containerDirs, dataDirs, config); - - this.containerReportManager = - new ContainerReportManagerImpl(config); } finally { readUnlock(); } @@ -254,12 +248,18 @@ public class ContainerManagerImpl implements ContainerManager { } containerData = ContainerData.getFromProtBuf(containerDataProto, conf); - // Initialize pending deletion blocks count in in-memory - // container status. + // Initialize pending deletion blocks and deleted blocks count in + // in-memory containerData. MetadataStore metadata = KeyUtils.getDB(containerData, conf); List> underDeletionBlocks = metadata .getSequentialRangeKVs(null, Integer.MAX_VALUE, MetadataKeyFilters.getDeletingKeyFilter()); + byte[] transactionID = metadata.get(DFSUtil.string2Bytes( + OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + containerID)); + if (transactionID != null) { + containerData + .updateDeleteTransactionId(Longs.fromByteArray(transactionID)); + } containerData.incrPendingDeletionBlocks(underDeletionBlocks.size()); List> liveKeys = metadata @@ -314,7 +314,8 @@ public class ContainerManagerImpl implements ContainerManager { writeLock(); try { if (containerMap.containsKey(containerData.getContainerID())) { - LOG.debug("container already exists. {}", containerData.getContainerID()); + LOG.debug("container already exists. {}", + containerData.getContainerID()); throw new StorageContainerException("container already exists.", CONTAINER_EXISTS); } @@ -595,7 +596,8 @@ public class ContainerManagerImpl implements ContainerManager { @Override public void updateContainer(long containerID, ContainerData data, boolean forceUpdate) throws StorageContainerException { - Preconditions.checkState(containerID >= 0, "Container ID cannot be negative."); + Preconditions.checkState(containerID >= 0, + "Container ID cannot be negative."); Preconditions.checkNotNull(data, "Container data cannot be null"); FileOutputStream containerStream = null; DigestOutputStream dos = null; @@ -711,7 +713,7 @@ public class ContainerManagerImpl implements ContainerManager { } /** - * Returns LifeCycle State of the container + * Returns LifeCycle State of the container. * @param containerID - Id of the container * @return LifeCycle State of the container * @throws StorageContainerException @@ -914,7 +916,8 @@ public class ContainerManagerImpl implements ContainerManager { .setWriteCount(container.getWriteCount()) .setReadBytes(container.getReadBytes()) .setWriteBytes(container.getWriteBytes()) - .setState(getState(containerId)); + .setState(getState(containerId)) + .setDeleteTransactionId(container.getDeleteTransactionId()); crBuilder.addReports(ciBuilder.build()); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java deleted file mode 100644 index f1d3f7f1691..00000000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - -import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.container.common.interfaces - .ContainerReportManager; -import org.apache.hadoop.util.Time; - -import java.util.concurrent.TimeUnit; - -import static org.apache.hadoop.hdds.scm.HddsServerUtil.getScmHeartbeatInterval; - -/** - * Class wraps the container report operations on datanode. - * // TODO: support incremental/delta container report - */ -public class ContainerReportManagerImpl implements ContainerReportManager { - // Last non-empty container report time - private long lastContainerReportTime; - private final long containerReportInterval; - private final long heartbeatInterval; - - public ContainerReportManagerImpl(Configuration config) { - this.lastContainerReportTime = -1; - this.containerReportInterval = config.getTimeDuration( - OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL, - OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, - TimeUnit.MILLISECONDS); - this.heartbeatInterval = getScmHeartbeatInterval(config); - } - - public boolean shouldSendContainerReport() { - if (lastContainerReportTime < 0) { - return true; - } - // Add a random delay (0~30s) on top of the container report - // interval (60s) so tha the SCM is overwhelmed by the container reports - // sent in sync. - if (Time.monotonicNow() - lastContainerReportTime > - (containerReportInterval + getRandomReportDelay())) { - return true; - } - return false; - } - - private long getRandomReportDelay() { - return RandomUtils.nextLong(0, heartbeatInterval); - } -} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java index 0ca73545f6f..40ae1c70a7d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java @@ -72,8 +72,10 @@ public class KeyManagerImpl implements KeyManager { */ @Override public void putKey(KeyData data) throws IOException { - Preconditions.checkNotNull(data, "KeyData cannot be null for put operation."); - Preconditions.checkState(data.getContainerID() >= 0, "Container ID cannot be negative"); + Preconditions.checkNotNull(data, + "KeyData cannot be null for put operation."); + Preconditions.checkState(data.getContainerID() >= 0, + "Container ID cannot be negative"); containerManager.readLock(); try { // We are not locking the key manager since LevelDb serializes all actions @@ -169,8 +171,10 @@ public class KeyManagerImpl implements KeyManager { public List listKey( long containerID, long startLocalID, int count) throws IOException { - Preconditions.checkState(containerID >= 0, "Container ID cannot be negative"); - Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be negative"); + Preconditions.checkState(containerID >= 0, + "Container ID cannot be negative"); + Preconditions.checkState(startLocalID >= 0, + "startLocal ID cannot be negative"); Preconditions.checkArgument(count > 0, "Count must be a positive number."); ContainerData cData = containerManager.readContainer(containerID); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java new file mode 100644 index 00000000000..ea2b987036a --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ContainerReportPublisher.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.ozone.OzoneConfigKeys; + +import java.util.concurrent.TimeUnit; + + +/** + * Publishes ContainerReport which will be sent to SCM as part of heartbeat. + * ContainerReport consist of the following information about each containers: + * - containerID + * - size + * - used + * - keyCount + * - readCount + * - writeCount + * - readBytes + * - writeBytes + * - finalHash + * - LifeCycleState + * + */ +public class ContainerReportPublisher extends + ReportPublisher { + + private Long containerReportInterval = null; + + @Override + protected long getReportFrequency() { + if (containerReportInterval == null) { + containerReportInterval = getConf().getTimeDuration( + OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL, + OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, + TimeUnit.MILLISECONDS); + } + // Add a random delay (0~30s) on top of the container report + // interval (60s) so tha the SCM is overwhelmed by the container reports + // sent in sync. + return containerReportInterval + getRandomReportDelay(); + } + + private long getRandomReportDelay() { + return RandomUtils.nextLong(0, containerReportInterval); + } + + @Override + protected ContainerReportsProto getReport() { + return ContainerReportsProto.getDefaultInstance(); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java new file mode 100644 index 00000000000..704b1f5b19d --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/NodeReportPublisher.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; + +/** + * Publishes NodeReport which will be sent to SCM as part of heartbeat. + * NodeReport consist of: + * - NodeIOStats + * - VolumeReports + */ +public class NodeReportPublisher extends ReportPublisher { + + @Override + protected long getReportFrequency() { + return 90000L; + } + + @Override + protected NodeReportProto getReport() { + return NodeReportProto.getDefaultInstance(); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java new file mode 100644 index 00000000000..8097cd6364e --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportManager.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.apache.hadoop.util.concurrent.HadoopExecutors; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; + +/** + * ReportManager is responsible for managing all the {@link ReportPublisher} + * and also provides {@link ScheduledExecutorService} to ReportPublisher + * which should be used for scheduling the reports. + */ +public final class ReportManager { + + private final StateContext context; + private final List publishers; + private final ScheduledExecutorService executorService; + + /** + * Construction of {@link ReportManager} should be done via + * {@link ReportManager.Builder}. + * + * @param context StateContext which holds the report + * @param publishers List of publishers which generates report + */ + private ReportManager(StateContext context, + List publishers) { + this.context = context; + this.publishers = publishers; + this.executorService = HadoopExecutors.newScheduledThreadPool( + publishers.size(), + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("Datanode ReportManager Thread - %d").build()); + } + + /** + * Initializes ReportManager, also initializes all the configured + * report publishers. + */ + public void init() { + for (ReportPublisher publisher : publishers) { + publisher.init(context, executorService); + } + } + + /** + * Shutdown the ReportManager. + */ + public void shutdown() { + executorService.shutdown(); + } + + /** + * Returns new {@link ReportManager.Builder} which can be used to construct. + * {@link ReportManager} + * @param conf - Conf + * @return builder - Builder. + */ + public static Builder newBuilder(Configuration conf) { + return new Builder(conf); + } + + /** + * Builder to construct {@link ReportManager}. + */ + public static final class Builder { + + private StateContext stateContext; + private List reportPublishers; + private ReportPublisherFactory publisherFactory; + + + private Builder(Configuration conf) { + this.reportPublishers = new ArrayList<>(); + this.publisherFactory = new ReportPublisherFactory(conf); + } + + /** + * Sets the {@link StateContext}. + * + * @param context StateContext + + * @return ReportManager.Builder + */ + public Builder setStateContext(StateContext context) { + stateContext = context; + return this; + } + + /** + * Adds publisher for the corresponding report. + * + * @param report report for which publisher needs to be added + * + * @return ReportManager.Builder + */ + public Builder addPublisherFor(Class report) { + reportPublishers.add(publisherFactory.getPublisherFor(report)); + return this; + } + + /** + * Adds new ReportPublisher to the ReportManager. + * + * @param publisher ReportPublisher + * + * @return ReportManager.Builder + */ + public Builder addPublisher(ReportPublisher publisher) { + reportPublishers.add(publisher); + return this; + } + + /** + * Build and returns ReportManager. + * + * @return {@link ReportManager} + */ + public ReportManager build() { + Preconditions.checkNotNull(stateContext); + return new ReportManager(stateContext, reportPublishers); + } + + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java new file mode 100644 index 00000000000..4ff47a05232 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisher.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ozone.container.common.statemachine + .DatanodeStateMachine.DatanodeStates; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Abstract class responsible for scheduling the reports based on the + * configured interval. All the ReportPublishers should extend this class. + */ +public abstract class ReportPublisher + implements Configurable, Runnable { + + private Configuration config; + private StateContext context; + private ScheduledExecutorService executor; + + /** + * Initializes ReportPublisher with stateContext and executorService. + * + * @param stateContext Datanode state context + * @param executorService ScheduledExecutorService to schedule reports + */ + public void init(StateContext stateContext, + ScheduledExecutorService executorService) { + this.context = stateContext; + this.executor = executorService; + this.executor.schedule(this, + getReportFrequency(), TimeUnit.MILLISECONDS); + } + + @Override + public void setConf(Configuration conf) { + config = conf; + } + + @Override + public Configuration getConf() { + return config; + } + + @Override + public void run() { + publishReport(); + if (!executor.isShutdown() || + !(context.getState() == DatanodeStates.SHUTDOWN)) { + executor.schedule(this, + getReportFrequency(), TimeUnit.MILLISECONDS); + } + } + + /** + * Generates and publishes the report to datanode state context. + */ + private void publishReport() { + context.addReport(getReport()); + } + + /** + * Returns the frequency in which this particular report has to be scheduled. + * + * @return report interval in milliseconds + */ + protected abstract long getReportFrequency(); + + /** + * Generate and returns the report which has to be sent as part of heartbeat. + * + * @return datanode report + */ + protected abstract T getReport(); + +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java new file mode 100644 index 00000000000..dc246d9428c --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/ReportPublisherFactory.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.util.ReflectionUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * Factory class to construct {@link ReportPublisher} for a report. + */ +public class ReportPublisherFactory { + + private final Configuration conf; + private final Map, + Class> report2publisher; + + /** + * Constructs {@link ReportPublisherFactory} instance. + * + * @param conf Configuration to be passed to the {@link ReportPublisher} + */ + public ReportPublisherFactory(Configuration conf) { + this.conf = conf; + this.report2publisher = new HashMap<>(); + + report2publisher.put(NodeReportProto.class, NodeReportPublisher.class); + report2publisher.put(ContainerReportsProto.class, + ContainerReportPublisher.class); + } + + /** + * Returns the ReportPublisher for the corresponding report. + * + * @param report report + * + * @return report publisher + */ + public ReportPublisher getPublisherFor( + Class report) { + Class publisherClass = + report2publisher.get(report); + if (publisherClass == null) { + throw new RuntimeException("No publisher found for report " + report); + } + return ReflectionUtils.newInstance(publisherClass, conf); + } + +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/package-info.java new file mode 100644 index 00000000000..404b37a7b08 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/report/package-info.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.common.report; +/** + * Datanode Reports: As part of heartbeat, datanode has to share its current + * state with SCM. The state of datanode is split into multiple reports which + * are sent along with heartbeat in a configured frequency. + * + * This package contains code which is responsible for sending reports from + * datanode to SCM. + * + * ReportPublisherFactory: Given a report this constructs corresponding + * {@link org.apache.hadoop.ozone.container.common.report.ReportPublisher}. + * + * ReportManager: Manages and initializes all the available ReportPublishers. + * + * ReportPublisher: Abstract class responsible for scheduling the reports + * based on the configured interval. All the ReportPublishers should extend + * {@link org.apache.hadoop.ozone.container.common.report.ReportPublisher} + * + * How to add new report: + * + * 1. Create a new ReportPublisher class which extends + * {@link org.apache.hadoop.ozone.container.common.report.ReportPublisher}. + * + * 2. Add a mapping Report to ReportPublisher entry in ReportPublisherFactory. + * + * 3. In DatanodeStateMachine add the report to ReportManager instance. + * + * + * + * Datanode Reports State Diagram: + * + * DatanodeStateMachine ReportManager ReportPublisher SCM + * | | | | + * | | | | + * | construct | | | + * |----------------->| | | + * | | | | + * | init | | | + * |----------------->| | | + * | | init | | + * | |------------->| | + * | | | | + * +--------+------------------+--------------+--------------------+------+ + * |loop | | | | | + * | | | publish | | | + * | |<-----------------+--------------| | | + * | | | report | | | + * | | | | | | + * | | | | | | + * | | heartbeat(rpc) | | | | + * | |------------------+--------------+------------------->| | + * | | | | | | + * | | | | | | + * +--------+------------------+--------------+--------------------+------+ + * | | | | + * | | | | + * | | | | + * | shutdown | | | + * |----------------->| | | + * | | | | + * | | | | + * - - - - + */ \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index d0a4217245e..dc4e673126b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -21,7 +21,13 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CloseContainerCommandHandler; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.ozone.container.common.report.ReportManager; +import org.apache.hadoop.ozone.container.common.statemachine.commandhandler + .CloseContainerCommandHandler; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler .CommandDispatcher; import org.apache.hadoop.ozone.container.common.statemachine.commandhandler @@ -56,6 +62,7 @@ public class DatanodeStateMachine implements Closeable { private final OzoneContainer container; private DatanodeDetails datanodeDetails; private final CommandDispatcher commandDispatcher; + private final ReportManager reportManager; private long commandsHandled; private AtomicLong nextHB; private Thread stateMachineThread = null; @@ -92,6 +99,12 @@ public class DatanodeStateMachine implements Closeable { .setContainer(container) .setContext(context) .build(); + + reportManager = ReportManager.newBuilder(conf) + .setStateContext(context) + .addPublisherFor(NodeReportProto.class) + .addPublisherFor(ContainerReportsProto.class) + .build(); } /** @@ -125,12 +138,12 @@ public class DatanodeStateMachine implements Closeable { long now = 0; container.start(); + reportManager.init(); initCommandHandlerThread(conf); while (context.getState() != DatanodeStates.SHUTDOWN) { try { LOG.debug("Executing cycle Number : {}", context.getExecutionCount()); nextHB.set(Time.monotonicNow() + heartbeatFrequency); - context.setNodeReport(container.getNodeReport()); context.execute(executorService, heartbeatFrequency, TimeUnit.MILLISECONDS); now = Time.monotonicNow(); @@ -307,6 +320,7 @@ public class DatanodeStateMachine implements Closeable { public synchronized void stopDaemon() { try { context.setState(DatanodeStates.SHUTDOWN); + reportManager.shutdown(); this.close(); LOG.info("Ozone container server stopped."); } catch (IOException e) { @@ -389,4 +403,13 @@ public class DatanodeStateMachine implements Closeable { public long getCommandHandled() { return commandsHandled; } + + /** + * returns the Command Dispatcher. + * @return CommandDispatcher + */ + @VisibleForTesting + public CommandDispatcher getCommandDispatcher() { + return commandDispatcher; + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java index 4e3c610f778..98eb7a05f64 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java @@ -16,9 +16,8 @@ */ package org.apache.hadoop.ozone.container.common.statemachine; +import com.google.protobuf.GeneratedMessage; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.NodeReportProto; import org.apache.hadoop.ozone.container.common.states.DatanodeState; import org.apache.hadoop.ozone.container.common.states.datanode .InitDatanodeState; @@ -28,7 +27,9 @@ import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.LinkedList; +import java.util.List; import java.util.Queue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -51,8 +52,8 @@ public class StateContext { private final DatanodeStateMachine parent; private final AtomicLong stateExecutionCount; private final Configuration conf; + private final Queue reports; private DatanodeStateMachine.DatanodeStates state; - private NodeReportProto dnReport; /** * Constructs a StateContext. @@ -67,9 +68,9 @@ public class StateContext { this.state = state; this.parent = parent; commandQueue = new LinkedList<>(); + reports = new LinkedList<>(); lock = new ReentrantLock(); stateExecutionCount = new AtomicLong(0); - dnReport = NodeReportProto.getDefaultInstance(); } /** @@ -141,19 +142,53 @@ public class StateContext { } /** - * Returns the node report of the datanode state context. - * @return the node report. + * Adds the report to report queue. + * + * @param report report to be added */ - public NodeReportProto getNodeReport() { - return dnReport; + public void addReport(GeneratedMessage report) { + synchronized (reports) { + reports.add(report); + } } /** - * Sets the storage location report of the datanode state context. - * @param nodeReport node report + * Returns the next report, or null if the report queue is empty. + * + * @return report */ - public void setNodeReport(NodeReportProto nodeReport) { - this.dnReport = nodeReport; + public GeneratedMessage getNextReport() { + synchronized (reports) { + return reports.poll(); + } + } + + /** + * Returns all the available reports from the report queue, or empty list if + * the queue is empty. + * + * @return List + */ + public List getAllAvailableReports() { + return getReports(Integer.MAX_VALUE); + } + + /** + * Returns available reports from the report queue with a max limit on + * list size, or empty list if the queue is empty. + * + * @return List + */ + public List getReports(int maxLimit) { + List results = new ArrayList<>(); + synchronized (reports) { + GeneratedMessage report = reports.poll(); + while(results.size() < maxLimit && report != null) { + results.add(report); + report = reports.poll(); + } + } + return results; } /** diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java index 99845fa1e40..63f57b48458 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java @@ -175,8 +175,8 @@ public class BlockDeletingService extends BackgroundService{ // Scan container's db and get list of under deletion blocks MetadataStore meta = KeyUtils.getDB(containerData, conf); // # of blocks to delete is throttled - KeyPrefixFilter filter = new KeyPrefixFilter( - OzoneConsts.DELETING_KEY_PREFIX); + KeyPrefixFilter filter = + new KeyPrefixFilter().addFilter(OzoneConsts.DELETING_KEY_PREFIX); List> toDeleteBlocks = meta.getSequentialRangeKVs(null, blockLimitPerTask, filter); if (toDeleteBlocks.isEmpty()) { @@ -214,10 +214,16 @@ public class BlockDeletingService extends BackgroundService{ } }); - // Once files are deleted ... clean up DB + // Once files are deleted... replace deleting entries with deleted entries BatchOperation batch = new BatchOperation(); - succeedBlocks.forEach(entry -> - batch.delete(DFSUtil.string2Bytes(entry))); + succeedBlocks.forEach(entry -> { + String blockId = + entry.substring(OzoneConsts.DELETING_KEY_PREFIX.length()); + String deletedEntry = OzoneConsts.DELETED_KEY_PREFIX + blockId; + batch.put(DFSUtil.string2Bytes(deletedEntry), + DFSUtil.string2Bytes(blockId)); + batch.delete(DFSUtil.string2Bytes(entry)); + }); meta.writeBatch(batch); // update count of pending deletion blocks in in-memory container status containerManager.decrPendingDeletionBlocks(succeedBlocks.size(), diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java index e8c602d1ad3..45f2bbd145f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java @@ -16,6 +16,8 @@ */ package org.apache.hadoop.ozone.container.common.statemachine.commandhandler; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto @@ -29,6 +31,8 @@ import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.UUID; + /** * Handler for close container command received from SCM. */ @@ -67,8 +71,23 @@ public class CloseContainerCommandHandler implements CommandHandler { CloseContainerCommandProto .parseFrom(command.getProtoBufMessage()); containerID = closeContainerProto.getContainerID(); + HddsProtos.ReplicationType replicationType = + closeContainerProto.getReplicationType(); - container.getContainerManager().closeContainer(containerID); + ContainerProtos.CloseContainerRequestProto.Builder closeRequest = + ContainerProtos.CloseContainerRequestProto.newBuilder(); + closeRequest.setContainerID(containerID); + + ContainerProtos.ContainerCommandRequestProto.Builder request = + ContainerProtos.ContainerCommandRequestProto.newBuilder(); + request.setCmdType(ContainerProtos.Type.CloseContainer); + request.setCloseContainer(closeRequest); + request.setTraceID(UUID.randomUUID().toString()); + request.setDatanodeUuid( + context.getParent().getDatanodeDetails().getUuidString()); + // submit the close container request for the XceiverServer to handle + container.submitContainerRequest( + request.build(), replicationType); } catch (Exception e) { LOG.error("Can't close container " + containerID, e); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java index aedd78fe4bd..5163d9851a2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java @@ -77,6 +77,10 @@ public final class CommandDispatcher { } } + public CommandHandler getCloseContainerHandler() { + return handlerMap.get(Type.closeContainerCommand); + } + /** * Dispatch the command to the correct handler. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java index ab69bdc38a6..f954d98f939 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java @@ -186,6 +186,9 @@ public class DeleteBlocksCommandHandler implements CommandHandler { LOG.debug("Block {} not found or already under deletion in" + " container {}, skip deleting it.", blk, containerId); } + containerDB.put(DFSUtil.string2Bytes( + OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()), + Longs.toByteArray(delTX.getTxID())); } // update pending deletion blocks count in in-memory container status diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index 337cdfbcf8b..1ee6375a562 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -19,6 +19,8 @@ package org.apache.hadoop.ozone.container.common.states.endpoint; import com.google.common.base.Preconditions; +import com.google.protobuf.Descriptors; +import com.google.protobuf.GeneratedMessage; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; @@ -99,13 +101,13 @@ public class HeartbeatEndpointTask try { Preconditions.checkState(this.datanodeDetailsProto != null); - SCMHeartbeatRequestProto request = SCMHeartbeatRequestProto.newBuilder() - .setDatanodeDetails(datanodeDetailsProto) - .setNodeReport(context.getNodeReport()) - .build(); + SCMHeartbeatRequestProto.Builder requestBuilder = + SCMHeartbeatRequestProto.newBuilder() + .setDatanodeDetails(datanodeDetailsProto); + addReports(requestBuilder); SCMHeartbeatResponseProto reponse = rpcEndpoint.getEndPoint() - .sendHeartbeat(request); + .sendHeartbeat(requestBuilder.build()); processResponse(reponse, datanodeDetailsProto); rpcEndpoint.setLastSuccessfulHeartbeat(ZonedDateTime.now()); rpcEndpoint.zeroMissedCount(); @@ -117,6 +119,24 @@ public class HeartbeatEndpointTask return rpcEndpoint.getState(); } + /** + * Adds all the available reports to heartbeat. + * + * @param requestBuilder builder to which the report has to be added. + */ + private void addReports(SCMHeartbeatRequestProto.Builder requestBuilder) { + for (GeneratedMessage report : context.getAllAvailableReports()) { + String reportName = report.getDescriptorForType().getFullName(); + for (Descriptors.FieldDescriptor descriptor : + SCMHeartbeatRequestProto.getDescriptor().getFields()) { + String heartbeatFieldName = descriptor.getMessageType().getFullName(); + if (heartbeatFieldName.equals(reportName)) { + requestBuilder.setField(descriptor, report); + } + } + } + } + /** * Returns a builder class for HeartbeatEndpointTask task. * @return Builder. diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java index 12b48abb10f..b3d2b620091 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.container.common.states.endpoint; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.ozone.container.common.statemachine diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java index 455df4977e3..3a469de1fdd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.container.common.transport.server; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.ratis.shaded.io.netty.bootstrap.ServerBootstrap; import org.apache.ratis.shaded.io.netty.channel.Channel; import org.apache.ratis.shaded.io.netty.channel.EventLoopGroup; @@ -129,4 +130,10 @@ public final class XceiverServer implements XceiverServerSpi { channel.close().awaitUninterruptibly(); } } + + @Override + public void submitRequest( + ContainerProtos.ContainerCommandRequestProto request) throws IOException { + storageContainer.dispatch(request); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java index 550fe416178..0a9e1dbba48 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java @@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.container.common.transport.server; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; @@ -44,6 +45,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi { LOG = LoggerFactory.getLogger(XceiverServerGrpc.class); private int port; private Server server; + private final ContainerDispatcher storageContainer; /** * Constructs a Grpc server class. @@ -77,6 +79,7 @@ public final class XceiverServerGrpc implements XceiverServerSpi { .maxMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE) .addService(new GrpcXceiverService(dispatcher)) .build(); + storageContainer = dispatcher; } @Override @@ -103,4 +106,10 @@ public final class XceiverServerGrpc implements XceiverServerSpi { public void stop() { server.shutdown(); } + + @Override + public void submitRequest( + ContainerProtos.ContainerCommandRequestProto request) throws IOException { + storageContainer.dispatch(request); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java index dad9e9f3fde..49579f2829b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.container.common.transport.server; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import java.io.IOException; @@ -40,4 +41,10 @@ public interface XceiverServerSpi { */ HddsProtos.ReplicationType getServerType(); + /** + * submits a containerRequest to be performed by the replication pipeline. + * @param request ContainerCommandRequest + */ + void submitRequest(ContainerProtos.ContainerCommandRequestProto request) + throws IOException; } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 56c52bb80e1..176407d66d2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -265,7 +265,8 @@ public class ContainerStateMachine extends BaseStateMachine { Message message = runCommand(requestProto); if (cmdType == ContainerProtos.Type.CreateContainer) { long containerID = - requestProto.getCreateContainer().getContainerData().getContainerID(); + requestProto.getCreateContainer() + .getContainerData().getContainerID(); createContainerFutureMap.remove(containerID).complete(message); } return CompletableFuture.completedFuture(message); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 33c25eaf2ef..b9c7cae4938 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; @@ -33,10 +35,12 @@ import org.apache.ratis.client.RaftClientConfigKeys; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.grpc.GrpcConfigKeys; import org.apache.ratis.netty.NettyConfigKeys; +import org.apache.ratis.protocol.*; import org.apache.ratis.rpc.RpcType; import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.shaded.proto.RaftProtos; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; @@ -49,8 +53,10 @@ import java.net.ServerSocket; import java.net.SocketAddress; import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * Creates a ratis server endpoint that acts as the communication layer for @@ -58,6 +64,12 @@ import java.util.concurrent.TimeUnit; */ public final class XceiverServerRatis implements XceiverServerSpi { static final Logger LOG = LoggerFactory.getLogger(XceiverServerRatis.class); + private static final AtomicLong callIdCounter = new AtomicLong(); + + private static long nextCallId() { + return callIdCounter.getAndIncrement() & Long.MAX_VALUE; + } + private final int port; private final RaftServer server; private ThreadPoolExecutor writeChunkExecutor; @@ -241,4 +253,46 @@ public final class XceiverServerRatis implements XceiverServerSpi { public HddsProtos.ReplicationType getServerType() { return HddsProtos.ReplicationType.RATIS; } -} + + @VisibleForTesting + public RaftServer getServer() { + return server; + } + + private void processReply(RaftClientReply reply) { + + // NotLeader exception is thrown only when the raft server to which the + // request is submitted is not the leader. The request will be rejected + // and will eventually be executed once the request comnes via the leader + // node. + NotLeaderException notLeaderException = reply.getNotLeaderException(); + if (notLeaderException != null) { + LOG.info(reply.getNotLeaderException().getLocalizedMessage()); + } + StateMachineException stateMachineException = + reply.getStateMachineException(); + if (stateMachineException != null) { + // In case the request could not be completed, StateMachine Exception + // will be thrown. For now, Just log the message. + // If the container could not be closed, SCM will come to know + // via containerReports. CloseContainer should be re tried via SCM. + LOG.error(stateMachineException.getLocalizedMessage()); + } + } + + @Override + public void submitRequest( + ContainerProtos.ContainerCommandRequestProto request) throws IOException { + ClientId clientId = ClientId.randomId(); + RaftClientRequest raftClientRequest = + new RaftClientRequest(clientId, server.getId(), + RatisHelper.emptyRaftGroup().getGroupId(), nextCallId(), 0, + Message.valueOf(request.toByteString()), RaftClientRequest + // ReplicationLevel.ALL ensures the transactions corresponding to + // the request here are applied on all the raft servers. + .writeRequestType(RaftProtos.ReplicationLevel.ALL)); + CompletableFuture reply = + server.submitClientRequestAsync(raftClientRequest); + reply.thenAccept(this::processReply); + } +} \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java index d9dd3605246..c63eb731c7a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java @@ -128,7 +128,8 @@ public final class ContainerCache extends LRUMap { public MetadataStore getDB(long containerID, String containerDBType, String containerDBPath) throws IOException { - Preconditions.checkState(containerID >= 0, "Container ID cannot be negative."); + Preconditions.checkState(containerID >= 0, + "Container ID cannot be negative."); lock.lock(); try { MetadataStore db = (MetadataStore) this.get(containerID); @@ -157,7 +158,8 @@ public final class ContainerCache extends LRUMap { * @param containerID - ID of the container. */ public void removeDB(long containerID) { - Preconditions.checkState(containerID >= 0, "Container ID cannot be negative."); + Preconditions.checkState(containerID >= 0, + "Container ID cannot be negative."); lock.lock(); try { MetadataStore db = (MetadataStore)this.get(containerID); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index b357fefb79d..4156f5a4e0c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -18,7 +18,9 @@ package org.apache.hadoop.ozone.container.ozoneimpl; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -72,7 +74,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT; * layer. */ public class OzoneContainer { - private static final Logger LOG = + public static final Logger LOG = LoggerFactory.getLogger(OzoneContainer.class); private final Configuration ozoneConfig; @@ -269,9 +271,65 @@ public class OzoneContainer { return this.manager.getClosedContainerReports(); } + private XceiverServerSpi getRatisSerer() { + for (XceiverServerSpi serverInstance : server) { + if (serverInstance instanceof XceiverServerRatis) { + return serverInstance; + } + } + return null; + } + + private XceiverServerSpi getStandaAloneSerer() { + for (XceiverServerSpi serverInstance : server) { + if (!(serverInstance instanceof XceiverServerRatis)) { + return serverInstance; + } + } + return null; + } + @VisibleForTesting public ContainerManager getContainerManager() { return this.manager; } -} + public void submitContainerRequest( + ContainerProtos.ContainerCommandRequestProto request, + HddsProtos.ReplicationType replicationType) throws IOException { + XceiverServerSpi serverInstance; + long containerId = getContainerIdForCmd(request); + if (replicationType == HddsProtos.ReplicationType.RATIS) { + serverInstance = getRatisSerer(); + Preconditions.checkNotNull(serverInstance); + serverInstance.submitRequest(request); + LOG.info("submitting {} request over RATIS server for container {}", + request.getCmdType(), containerId); + } else { + serverInstance = getStandaAloneSerer(); + Preconditions.checkNotNull(serverInstance); + getStandaAloneSerer().submitRequest(request); + LOG.info( + "submitting {} request over STAND_ALONE server for container {}", + request.getCmdType(), containerId); + } + + } + + private long getContainerIdForCmd( + ContainerProtos.ContainerCommandRequestProto request) + throws IllegalArgumentException { + ContainerProtos.Type type = request.getCmdType(); + switch (type) { + case CloseContainer: + return request.getCloseContainer().getContainerID(); + // Right now, we handle only closeContainer via queuing it over the + // over the XceiVerServer. For all other commands we throw Illegal + // argument exception here. Will need to extend the switch cases + // in case we want add another commands here. + default: + throw new IllegalArgumentException("Cmd " + request.getCmdType() + + " not supported over HearBeat Response"); + } + } +} \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java index 4f4f82b6002..c7d8df57150 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.protocol.commands; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMCommandProto; import org.apache.hadoop.hdds.protocol.proto @@ -31,9 +32,12 @@ public class CloseContainerCommand extends SCMCommand { private long containerID; + private HddsProtos.ReplicationType replicationType; - public CloseContainerCommand(long containerID) { + public CloseContainerCommand(long containerID, + HddsProtos.ReplicationType replicationType) { this.containerID = containerID; + this.replicationType = replicationType; } /** @@ -58,13 +62,15 @@ public class CloseContainerCommand public CloseContainerCommandProto getProto() { return CloseContainerCommandProto.newBuilder() - .setContainerID(containerID).build(); + .setContainerID(containerID) + .setReplicationType(replicationType).build(); } public static CloseContainerCommand getFromProtobuf( CloseContainerCommandProto closeContainerProto) { Preconditions.checkNotNull(closeContainerProto); - return new CloseContainerCommand(closeContainerProto.getContainerID()); + return new CloseContainerCommand(closeContainerProto.getContainerID(), + closeContainerProto.getReplicationType()); } diff --git a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto index ac2314e2c02..f6aba05636c 100644 --- a/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdds/container-service/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -160,6 +160,7 @@ message ContainerInfo { optional int64 writeBytes = 8; optional string finalhash = 9; optional hadoop.hdds.LifeCycleState state = 10; + optional int64 deleteTransactionId = 11; } /* @@ -222,6 +223,7 @@ This command asks the datanode to close a specific container. */ message CloseContainerCommandProto { required int64 containerID = 1; + required hadoop.hdds.ReplicationType replicationType = 2; } /** diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java index 0ee6321c992..14da9601e45 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.container.common; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.hdds.scm.VersionInfo; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -179,6 +180,7 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol { List cmdResponses = new LinkedList<>(); return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses) + .setDatanodeUUID(heartbeat.getDatanodeDetails().getUuid()) .build(); } @@ -197,6 +199,7 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol { throws IOException { rpcCount.incrementAndGet(); updateNodeReport(datanodeDetailsProto, nodeReport); + updateContainerReport(containerReportsRequestProto, datanodeDetailsProto); sleepIfNeeded(); return StorageContainerDatanodeProtocolProtos.SCMRegisteredResponseProto .newBuilder().setClusterID(UUID.randomUUID().toString()) @@ -227,6 +230,35 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol { } + /** + * Update the cotainerReport. + * + * @param reports Container report + * @param datanodeDetails DataNode Info + * @throws IOException + */ + public void updateContainerReport( + StorageContainerDatanodeProtocolProtos.ContainerReportsProto reports, + DatanodeDetailsProto datanodeDetails) throws IOException { + Preconditions.checkNotNull(reports); + containerReportsCount.incrementAndGet(); + DatanodeDetails datanode = DatanodeDetails.getFromProtoBuf( + datanodeDetails); + if (reports.getReportsCount() > 0) { + Map containers = nodeContainers.get(datanode); + if (containers == null) { + containers = new LinkedHashMap(); + nodeContainers.put(datanode, containers); + } + + for (StorageContainerDatanodeProtocolProtos.ContainerInfo report : reports + .getReportsList()) { + containers.put(report.getContainerID(), report); + } + } + } + + /** * Return the number of StorageReports of a datanode. * @param datanodeDetails diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/package-info.java new file mode 100644 index 00000000000..ca3d29dada1 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +/** + * SCM Testing and Mocking Utils. + */ +package org.apache.hadoop.ozone.container.common; \ No newline at end of file diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java new file mode 100644 index 00000000000..aae388dd5a1 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportManager.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.ScheduledExecutorService; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Test cases to test {@link ReportManager}. + */ +public class TestReportManager { + + @Test + public void testReportManagerInit() { + Configuration conf = new OzoneConfiguration(); + StateContext dummyContext = Mockito.mock(StateContext.class); + ReportPublisher dummyPublisher = Mockito.mock(ReportPublisher.class); + ReportManager.Builder builder = ReportManager.newBuilder(conf); + builder.setStateContext(dummyContext); + builder.addPublisher(dummyPublisher); + ReportManager reportManager = builder.build(); + reportManager.init(); + verify(dummyPublisher, times(1)).init(eq(dummyContext), + any(ScheduledExecutorService.class)); + + } +} diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java new file mode 100644 index 00000000000..5fd9cf60479 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisher.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.protobuf.Descriptors; +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; +import org.apache.hadoop.ozone.container.common.statemachine.StateContext; +import org.apache.hadoop.util.concurrent.HadoopExecutors; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Test cases to test {@link ReportPublisher}. + */ +public class TestReportPublisher { + + /** + * Dummy report publisher for testing. + */ + private class DummyReportPublisher extends ReportPublisher { + + private final long frequency; + private int getReportCount = 0; + + DummyReportPublisher(long frequency) { + this.frequency = frequency; + } + + @Override + protected long getReportFrequency() { + return frequency; + } + + @Override + protected GeneratedMessage getReport() { + getReportCount++; + return null; + } + } + + @Test + public void testReportPublisherInit() { + ReportPublisher publisher = new DummyReportPublisher(0); + StateContext dummyContext = Mockito.mock(StateContext.class); + ScheduledExecutorService dummyExecutorService = Mockito.mock( + ScheduledExecutorService.class); + publisher.init(dummyContext, dummyExecutorService); + verify(dummyExecutorService, times(1)).schedule(publisher, + 0, TimeUnit.MILLISECONDS); + } + + @Test + public void testScheduledReport() throws InterruptedException { + ReportPublisher publisher = new DummyReportPublisher(100); + StateContext dummyContext = Mockito.mock(StateContext.class); + ScheduledExecutorService executorService = HadoopExecutors + .newScheduledThreadPool(1, + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("Unit test ReportManager Thread - %d").build()); + publisher.init(dummyContext, executorService); + Thread.sleep(150); + Assert.assertEquals(1, ((DummyReportPublisher)publisher).getReportCount); + Thread.sleep(150); + Assert.assertEquals(2, ((DummyReportPublisher)publisher).getReportCount); + executorService.shutdown(); + } + + @Test + public void testPublishReport() throws InterruptedException { + ReportPublisher publisher = new DummyReportPublisher(100); + StateContext dummyContext = Mockito.mock(StateContext.class); + ScheduledExecutorService executorService = HadoopExecutors + .newScheduledThreadPool(1, + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("Unit test ReportManager Thread - %d").build()); + publisher.init(dummyContext, executorService); + Thread.sleep(150); + executorService.shutdown(); + Assert.assertEquals(1, ((DummyReportPublisher)publisher).getReportCount); + verify(dummyContext, times(1)).addReport(null); + + } + + @Test + public void testAddingReportToHeartbeat() { + Configuration conf = new OzoneConfiguration(); + ReportPublisherFactory factory = new ReportPublisherFactory(conf); + ReportPublisher nodeReportPublisher = factory.getPublisherFor( + NodeReportProto.class); + ReportPublisher containerReportPubliser = factory.getPublisherFor( + ContainerReportsProto.class); + GeneratedMessage nodeReport = nodeReportPublisher.getReport(); + GeneratedMessage containerReport = containerReportPubliser.getReport(); + SCMHeartbeatRequestProto.Builder heartbeatBuilder = + SCMHeartbeatRequestProto.newBuilder(); + heartbeatBuilder.setDatanodeDetails( + getDatanodeDetails().getProtoBufMessage()); + addReport(heartbeatBuilder, nodeReport); + addReport(heartbeatBuilder, containerReport); + SCMHeartbeatRequestProto heartbeat = heartbeatBuilder.build(); + Assert.assertTrue(heartbeat.hasNodeReport()); + Assert.assertTrue(heartbeat.hasContainerReport()); + } + + /** + * Get a datanode details. + * + * @return DatanodeDetails + */ + private static DatanodeDetails getDatanodeDetails() { + String uuid = UUID.randomUUID().toString(); + Random random = new Random(); + String ipAddress = + random.nextInt(256) + "." + random.nextInt(256) + "." + random + .nextInt(256) + "." + random.nextInt(256); + + DatanodeDetails.Port containerPort = DatanodeDetails.newPort( + DatanodeDetails.Port.Name.STANDALONE, 0); + DatanodeDetails.Port ratisPort = DatanodeDetails.newPort( + DatanodeDetails.Port.Name.RATIS, 0); + DatanodeDetails.Port restPort = DatanodeDetails.newPort( + DatanodeDetails.Port.Name.REST, 0); + DatanodeDetails.Builder builder = DatanodeDetails.newBuilder(); + builder.setUuid(uuid) + .setHostName("localhost") + .setIpAddress(ipAddress) + .addPort(containerPort) + .addPort(ratisPort) + .addPort(restPort); + return builder.build(); + } + + /** + * Adds the report to heartbeat. + * + * @param requestBuilder builder to which the report has to be added. + * @param report the report to be added. + */ + private static void addReport(SCMHeartbeatRequestProto.Builder requestBuilder, + GeneratedMessage report) { + String reportName = report.getDescriptorForType().getFullName(); + for (Descriptors.FieldDescriptor descriptor : + SCMHeartbeatRequestProto.getDescriptor().getFields()) { + String heartbeatFieldName = descriptor.getMessageType().getFullName(); + if (heartbeatFieldName.equals(reportName)) { + requestBuilder.setField(descriptor, report); + } + } + } + +} diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java new file mode 100644 index 00000000000..f8c5fe5e275 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/TestReportPublisherFactory.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.common.report; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +/** + * Test cases to test ReportPublisherFactory. + */ +public class TestReportPublisherFactory { + + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testGetContainerReportPublisher() { + Configuration conf = new OzoneConfiguration(); + ReportPublisherFactory factory = new ReportPublisherFactory(conf); + ReportPublisher publisher = factory + .getPublisherFor(ContainerReportsProto.class); + Assert.assertEquals(ContainerReportPublisher.class, publisher.getClass()); + Assert.assertEquals(conf, publisher.getConf()); + } + + @Test + public void testGetNodeReportPublisher() { + Configuration conf = new OzoneConfiguration(); + ReportPublisherFactory factory = new ReportPublisherFactory(conf); + ReportPublisher publisher = factory + .getPublisherFor(NodeReportProto.class); + Assert.assertEquals(NodeReportPublisher.class, publisher.getClass()); + Assert.assertEquals(conf, publisher.getConf()); + } + + @Test + public void testInvalidReportPublisher() { + Configuration conf = new OzoneConfiguration(); + ReportPublisherFactory factory = new ReportPublisherFactory(conf); + exception.expect(RuntimeException.class); + exception.expectMessage("No publisher found for report"); + factory.getPublisherFor(HddsProtos.DatanodeDetailsProto.class); + } +} diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/package-info.java similarity index 71% rename from hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java rename to hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/package-info.java index 6d7557b94aa..37615bc7536 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/report/package-info.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,16 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.container.common.interfaces; - +package org.apache.hadoop.ozone.container.common.report; /** - * Interface for container report manager operations. - */ -public interface ContainerReportManager { - - /** - * Check if we have to send container report. - * @return true if container report has to be sent. - */ - boolean shouldSendContainerReport(); -} + * This package has test cases for all the report publishers which generates + * reports that are sent to SCM via heartbeat. + */ \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java index d17d6c07f06..7cfbdabe1a9 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java @@ -361,13 +361,10 @@ public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean { } } - // We update SCM DB first, so if this step fails, we end up here, - // nothing gets into the delLog so no blocks will be accidentally - // removed. If we write the log first, once log is written, the - // async deleting service will start to scan and might be picking - // up some blocks to do real deletions, that might cause data loss. try { - deletedBlockLog.addTransactions(containerBlocks); + Map deleteTransactionsMap = + deletedBlockLog.addTransactions(containerBlocks); + containerManager.updateDeleteTransactionId(deleteTransactionsMap); } catch (IOException e) { throw new IOException( "Skip writing the deleted blocks info to" diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java index cc32b35a4da..4f4c75563b7 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java @@ -108,9 +108,10 @@ public interface DeletedBlockLog extends Closeable { * number of containers) together (on success) or non (on failure). * * @param containerBlocksMap a map of containerBlocks. + * @return Mapping from containerId to latest transactionId for the container. * @throws IOException */ - void addTransactions(Map> containerBlocksMap) + Map addTransactions(Map> containerBlocksMap) throws IOException; /** diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java index cedc5060352..48fa2eb1112 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java @@ -36,6 +36,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -306,12 +307,15 @@ public class DeletedBlockLogImpl implements DeletedBlockLog { * {@inheritDoc} * * @param containerBlocksMap a map of containerBlocks. + * @return Mapping from containerId to latest transactionId for the container. * @throws IOException */ @Override - public void addTransactions(Map> containerBlocksMap) + public Map addTransactions( + Map> containerBlocksMap) throws IOException { BatchOperation batch = new BatchOperation(); + Map deleteTransactionsMap = new HashMap<>(); lock.lock(); try { long currentLatestID = lastTxID; @@ -321,11 +325,13 @@ public class DeletedBlockLogImpl implements DeletedBlockLog { byte[] key = Longs.toByteArray(currentLatestID); DeletedBlocksTransaction tx = constructNewTransaction(currentLatestID, entry.getKey(), entry.getValue()); + deleteTransactionsMap.put(entry.getKey(), currentLatestID); batch.put(key, tx.toByteArray()); } lastTxID = currentLatestID; batch.put(LATEST_TXID, Longs.toByteArray(lastTxID)); deletedStore.writeBatch(batch); + return deleteTransactionsMap; } finally { lock.unlock(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java new file mode 100644 index 00000000000..16e84a3c5f3 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ *

http://www.apache.org/licenses/LICENSE-2.0 + *

+ *

Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.hdds.server.events.TypedEvent; +import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * In case of a node failure, volume failure, volume out of spapce, node + * out of space etc, CLOSE_CONTAINER_EVENT will be triggered. + * CloseContainerEventHandler is the handler for CLOSE_CONTAINER_EVENT. + * When a close container event is fired, a close command for the container + * should be sent to all the datanodes in the pipeline and containerStateManager + * needs to update the container state to Closing. + */ +public class CloseContainerEventHandler implements EventHandler { + + public static final Logger LOG = + LoggerFactory.getLogger(CloseContainerEventHandler.class); + + public static final TypedEvent CLOSE_CONTAINER_EVENT = + new TypedEvent<>(ContainerID.class); + + private final Mapping containerManager; + + public CloseContainerEventHandler(Mapping containerManager) { + this.containerManager = containerManager; + } + + @Override + public void onMessage(ContainerID containerID, EventPublisher publisher) { + + LOG.info("Close container Event triggered for container : {}", + containerID.getId()); + ContainerStateManager stateManager = containerManager.getStateManager(); + ContainerInfo info = stateManager.getContainer(containerID); + if (info == null) { + LOG.info("Container with id : {} does not exist", containerID.getId()); + return; + } + if (info.getState() == HddsProtos.LifeCycleState.OPEN) { + for (DatanodeDetails datanode : info.getPipeline().getMachines()) { + containerManager.getNodeManager().addDatanodeCommand(datanode.getUuid(), + new CloseContainerCommand(containerID.getId(), + info.getPipeline().getType())); + } + try { + // Finalize event will make sure the state of the container transitions + // from OPEN to CLOSING in containerStateManager. + stateManager + .updateContainerState(info, HddsProtos.LifeCycleEvent.FINALIZE); + } catch (SCMException ex) { + LOG.error("Failed to update the container state for container : {}" + + containerID); + } + } else { + LOG.info("container with id : {} is in {} state and need not be closed.", + containerID.getId(), info.getState()); + } + + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java index f5fe46a3a53..b563e90e765 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java @@ -341,6 +341,39 @@ public class ContainerMapping implements Mapping { } } + /** + * Update deleteTransactionId according to deleteTransactionMap. + * + * @param deleteTransactionMap Maps the containerId to latest delete + * transaction id for the container. + * @throws IOException + */ + public void updateDeleteTransactionId(Map deleteTransactionMap) + throws IOException { + lock.lock(); + try { + for (Map.Entry entry : deleteTransactionMap.entrySet()) { + long containerID = entry.getKey(); + byte[] dbKey = Longs.toByteArray(containerID); + byte[] containerBytes = containerStore.get(dbKey); + if (containerBytes == null) { + throw new SCMException( + "Failed to increment number of deleted blocks for container " + + containerID + ", reason : " + "container doesn't exist.", + SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER); + } + ContainerInfo containerInfo = ContainerInfo.fromProtobuf( + HddsProtos.SCMContainerInfo.parseFrom(containerBytes)); + containerInfo.updateDeleteTransactionId(entry.getValue()); + containerStore.put(dbKey, containerInfo.getProtobuf().toByteArray()); + containerStateManager + .updateDeleteTransactionId(containerID, entry.getValue()); + } + } finally { + lock.unlock(); + } + } + /** * Returns the container State Manager. * @@ -441,6 +474,7 @@ public class ContainerMapping implements Mapping { builder.setState(knownState.getState()); builder.setStateEnterTime(knownState.getStateEnterTime()); builder.setContainerID(knownState.getContainerID()); + builder.setDeleteTransactionId(knownState.getDeleteTransactionId()); if (knownState.getOwner() != null) { builder.setOwner(knownState.getOwner()); } @@ -571,6 +605,7 @@ public class ContainerMapping implements Mapping { .setPipeline(oldInfo.getPipeline()) .setState(oldInfo.getState()) .setUsedBytes(oldInfo.getUsedBytes()) + .setDeleteTransactionId(oldInfo.getDeleteTransactionId()) .build(); containerStore.put(dbKey, newInfo.getProtobuf().toByteArray()); } else { @@ -588,6 +623,11 @@ public class ContainerMapping implements Mapping { } } + @Override + public NodeManager getNodeManager() { + return nodeManager; + } + @VisibleForTesting public MetadataStore getContainerStore() { return containerStore; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java index 4895b786f92..08733bd707d 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java @@ -304,6 +304,7 @@ public class ContainerStateManager implements Closeable { .setStateEnterTime(Time.monotonicNow()) .setOwner(owner) .setContainerID(containerCount.incrementAndGet()) + .setDeleteTransactionId(0) .build(); Preconditions.checkNotNull(containerInfo); containers.addContainer(containerInfo); @@ -351,6 +352,17 @@ public class ContainerStateManager implements Closeable { return containers.getContainerInfo(info); } + /** + * Update deleteTransactionId for a container. + * + * @param containerID ContainerID of the container whose delete + * transactionId needs to be updated. + * @param transactionId latest transactionId to be updated for the container + */ + public void updateDeleteTransactionId(Long containerID, long transactionId) { + containers.getContainerMap().get(ContainerID.valueof(containerID)) + .updateDeleteTransactionId(transactionId); + } /** * Return a container matching the attributes specified. @@ -445,6 +457,15 @@ public class ContainerStateManager implements Closeable { factor, type); } + /** + * Returns the containerInfo for the given container id. + * @param containerID id of the container + * @return ContainerInfo containerInfo + * @throws IOException + */ + public ContainerInfo getContainer(ContainerID containerID) { + return containers.getContainerInfo(containerID.getId()); + } @Override public void close() throws IOException { } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java index ee8e344d0df..e77a4b60f25 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java @@ -21,10 +21,12 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.scm.node.NodeManager; import java.io.Closeable; import java.io.IOException; import java.util.List; +import java.util.Map; /** * Mapping class contains the mapping from a name to a pipeline mapping. This is @@ -103,4 +105,19 @@ public interface Mapping extends Closeable { ContainerReportsProto reports) throws IOException; + /** + * Update deleteTransactionId according to deleteTransactionMap. + * + * @param deleteTransactionMap Maps the containerId to latest delete + * transaction id for the container. + * @throws IOException + */ + void updateDeleteTransactionId(Map deleteTransactionMap) + throws IOException; + + /** + * Returns the nodeManager. + * @return NodeManager + */ + NodeManager getNodeManager(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java index 75ec8e103c1..937076cfb77 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/closer/ContainerCloser.java @@ -127,11 +127,12 @@ public class ContainerCloser { // to SCM. In that case also, data node will ignore this command. HddsProtos.Pipeline pipeline = info.getPipeline(); - for (HddsProtos.DatanodeDetailsProto datanodeDetails : - pipeline.getPipelineChannel().getMembersList()) { + for (HddsProtos.DatanodeDetailsProto datanodeDetails : pipeline + .getPipelineChannel().getMembersList()) { nodeManager.addDatanodeCommand( DatanodeDetails.getFromProtoBuf(datanodeDetails).getUuid(), - new CloseContainerCommand(info.getContainerID())); + new CloseContainerCommand(info.getContainerID(), + pipeline.getPipelineChannel().getType())); } if (!commandIssued.containsKey(info.getContainerID())) { commandIssued.put(info.getContainerID(), diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index 1b1645db93c..7d16161e595 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -69,7 +69,7 @@ import static org.apache.hadoop.hdds.protocol.proto import org.apache.hadoop.hdds.scm.HddsServerUtil; -import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat; +import org.apache.hadoop.hdds.scm.server.report.SCMDatanodeHeartbeatDispatcher; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; @@ -114,6 +114,7 @@ public class SCMDatanodeProtocolServer implements private final StorageContainerManager scm; private final InetSocketAddress datanodeRpcAddress; + private final SCMDatanodeHeartbeatDispatcher heartbeatDispatcher; public SCMDatanodeProtocolServer(final OzoneConfiguration conf, StorageContainerManager scm) throws IOException { @@ -148,16 +149,24 @@ public class SCMDatanodeProtocolServer implements updateRPCListenAddress( conf, OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr, datanodeRpcServer); + + heartbeatDispatcher = SCMDatanodeHeartbeatDispatcher.newBuilder(conf, scm) + .addHandlerFor(NodeReportProto.class) + .addHandlerFor(ContainerReportsProto.class) + .build(); + } + + public void start() { + LOG.info( + StorageContainerManager.buildRpcServerStartMessage( + "RPC server for DataNodes", datanodeRpcAddress)); + datanodeRpcServer.start(); } public InetSocketAddress getDatanodeRpcAddress() { return datanodeRpcAddress; } - public RPC.Server getDatanodeRpcServer() { - return datanodeRpcServer; - } - @Override public SCMVersionResponseProto getVersion(SCMVersionRequestProto versionRequest) @@ -166,25 +175,6 @@ public class SCMDatanodeProtocolServer implements .getProtobufMessage(); } - @Override - public SCMHeartbeatResponseProto sendHeartbeat( - SCMHeartbeatRequestProto heartbeat) - throws IOException { - // TODO: Add a heartbeat dispatcher. - DatanodeDetails datanodeDetails = DatanodeDetails - .getFromProtoBuf(heartbeat.getDatanodeDetails()); - NodeReportProto nodeReport = heartbeat.getNodeReport(); - List commands = - scm.getScmNodeManager().sendHeartbeat(datanodeDetails, nodeReport); - List cmdResponses = new LinkedList<>(); - for (SCMCommand cmd : commands) { - cmdResponses.add(getCommandResponse(cmd)); - } - return SCMHeartbeatResponseProto.newBuilder() - .setDatanodeUUID(datanodeDetails.getUuidString()) - .addAllCommands(cmdResponses).build(); - } - @Override public SCMRegisteredResponseProto register( HddsProtos.DatanodeDetailsProto datanodeDetailsProto, @@ -216,36 +206,27 @@ public class SCMDatanodeProtocolServer implements .build(); } - public void processContainerReports(DatanodeDetails datanodeDetails, - ContainerReportsProto reports) + @Override + public SCMHeartbeatResponseProto sendHeartbeat( + SCMHeartbeatRequestProto heartbeat) throws IOException { - updateContainerReportMetrics(datanodeDetails, reports); - // should we process container reports async? - scm.getScmContainerManager() - .processContainerReports(datanodeDetails, reports); - } + heartbeatDispatcher.dispatch(heartbeat); - private void updateContainerReportMetrics(DatanodeDetails datanodeDetails, - ContainerReportsProto reports) { - ContainerStat newStat = new ContainerStat(); - for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports - .getReportsList()) { - newStat.add(new ContainerStat(info.getSize(), info.getUsed(), - info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(), - info.getReadCount(), info.getWriteCount())); + // TODO: Remove the below code after SCM refactoring. + DatanodeDetails datanodeDetails = DatanodeDetails + .getFromProtoBuf(heartbeat.getDatanodeDetails()); + NodeReportProto nodeReport = heartbeat.getNodeReport(); + List commands = + scm.getScmNodeManager().sendHeartbeat(datanodeDetails, nodeReport); + List cmdResponses = new LinkedList<>(); + for (SCMCommand cmd : commands) { + cmdResponses.add(getCommandResponse(cmd)); } - // update container metrics - StorageContainerManager.getMetrics().setLastContainerStat(newStat); - - // Update container stat entry, this will trigger a removal operation if it - // exists in cache. - String datanodeUuid = datanodeDetails.getUuidString(); - scm.getContainerReportCache().put(datanodeUuid, newStat); - // update global view container metrics - StorageContainerManager.getMetrics().incrContainerStat(newStat); + return SCMHeartbeatResponseProto.newBuilder() + .setDatanodeUUID(datanodeDetails.getUuidString()) + .addAllCommands(cmdResponses).build(); } - @Override public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK( ContainerBlocksDeletionACKProto acks) throws IOException { @@ -271,28 +252,6 @@ public class SCMDatanodeProtocolServer implements .getDefaultInstanceForType(); } - public void start() { - LOG.info( - StorageContainerManager.buildRpcServerStartMessage( - "RPC server for DataNodes", getDatanodeRpcAddress())); - getDatanodeRpcServer().start(); - } - - public void stop() { - try { - LOG.info("Stopping the RPC server for DataNodes"); - datanodeRpcServer.stop(); - } catch (Exception ex) { - LOG.error(" datanodeRpcServer stop failed.", ex); - } - IOUtils.cleanupWithLogger(LOG, scm.getScmNodeManager()); - } - - public void join() throws InterruptedException { - LOG.trace("Join RPC server for DataNodes"); - datanodeRpcServer.join(); - } - /** * Returns a SCMCommandRepose from the SCM Command. * @@ -338,4 +297,22 @@ public class SCMDatanodeProtocolServer implements throw new IllegalArgumentException("Not implemented"); } } + + + public void join() throws InterruptedException { + LOG.trace("Join RPC server for DataNodes"); + datanodeRpcServer.join(); + } + + public void stop() { + try { + LOG.info("Stopping the RPC server for DataNodes"); + datanodeRpcServer.stop(); + heartbeatDispatcher.shutdown(); + } catch (Exception ex) { + LOG.error(" datanodeRpcServer stop failed.", ex); + } + IOUtils.cleanupWithLogger(LOG, scm.getScmNodeManager()); + } + } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java new file mode 100644 index 00000000000..00ce94d7f58 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeContainerReportHandler.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.scm.container.placement.metrics.ContainerStat; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * Handler for Datanode Container Report. + */ +public class SCMDatanodeContainerReportHandler extends + SCMDatanodeReportHandler { + + private static final Logger LOG = LoggerFactory.getLogger( + SCMDatanodeContainerReportHandler.class); + + @Override + public void processReport(DatanodeDetails datanodeDetails, + ContainerReportsProto report) throws IOException { + LOG.trace("Processing container report from {}.", datanodeDetails); + updateContainerReportMetrics(datanodeDetails, report); + getSCM().getScmContainerManager() + .processContainerReports(datanodeDetails, report); + } + + /** + * Updates container report metrics in SCM. + * + * @param datanodeDetails Datanode Information + * @param reports Container Reports + */ + private void updateContainerReportMetrics(DatanodeDetails datanodeDetails, + ContainerReportsProto reports) { + ContainerStat newStat = new ContainerStat(); + for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports + .getReportsList()) { + newStat.add(new ContainerStat(info.getSize(), info.getUsed(), + info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(), + info.getReadCount(), info.getWriteCount())); + } + // update container metrics + StorageContainerManager.getMetrics().setLastContainerStat(newStat); + + // Update container stat entry, this will trigger a removal operation if it + // exists in cache. + String datanodeUuid = datanodeDetails.getUuidString(); + getSCM().getContainerReportCache().put(datanodeUuid, newStat); + // update global view container metrics + StorageContainerManager.getMetrics().incrContainerStat(newStat); + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java new file mode 100644 index 00000000000..d50edff7c50 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeHeartbeatDispatcher.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.util.concurrent.HadoopExecutors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +/** + * This class is responsible for dispatching heartbeat from datanode to + * appropriate ReportHandlers at SCM. + * Only one handler per report is supported now, it's very easy to support + * multiple handlers for a report. + */ +public final class SCMDatanodeHeartbeatDispatcher { + + private static final Logger LOG = LoggerFactory.getLogger( + SCMDatanodeHeartbeatDispatcher.class); + + /** + * This stores Report to Handler mapping. + */ + private final Map, + SCMDatanodeReportHandler> handlers; + + /** + * Executor service which will be used for processing reports. + */ + private final ExecutorService executorService; + + /** + * Constructs SCMDatanodeHeartbeatDispatcher instance with the given + * handlers. + * + * @param handlers report to report handler mapping + */ + private SCMDatanodeHeartbeatDispatcher(Map, + SCMDatanodeReportHandler> handlers) { + this.handlers = handlers; + this.executorService = HadoopExecutors.newCachedThreadPool( + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("SCMDatanode Heartbeat Dispatcher Thread - %d") + .build()); + } + + /** + * Dispatches heartbeat to registered handlers. + * + * @param heartbeat heartbeat to be dispatched. + */ + public void dispatch(SCMHeartbeatRequestProto heartbeat) { + DatanodeDetails datanodeDetails = DatanodeDetails + .getFromProtoBuf(heartbeat.getDatanodeDetails()); + if (heartbeat.hasNodeReport()) { + processReport(datanodeDetails, heartbeat.getNodeReport()); + } + if (heartbeat.hasContainerReport()) { + processReport(datanodeDetails, heartbeat.getContainerReport()); + } + } + + /** + * Invokes appropriate ReportHandler and submits the task to executor + * service for processing. + * + * @param datanodeDetails Datanode Information + * @param report Report to be processed + */ + @SuppressWarnings("unchecked") + private void processReport(DatanodeDetails datanodeDetails, + GeneratedMessage report) { + executorService.submit(() -> { + try { + SCMDatanodeReportHandler handler = handlers.get(report.getClass()); + handler.processReport(datanodeDetails, report); + } catch (IOException ex) { + LOG.error("Exception wile processing report {}, from {}", + report.getClass(), datanodeDetails, ex); + } + }); + } + + /** + * Shuts down SCMDatanodeHeartbeatDispatcher. + */ + public void shutdown() { + executorService.shutdown(); + } + + /** + * Returns a new Builder to construct {@link SCMDatanodeHeartbeatDispatcher}. + * + * @param conf Configuration to be used by SCMDatanodeHeartbeatDispatcher + * @param scm {@link StorageContainerManager} instance to be used by report + * handlers + * + * @return {@link SCMDatanodeHeartbeatDispatcher.Builder} instance + */ + public static Builder newBuilder(Configuration conf, + StorageContainerManager scm) { + return new Builder(conf, scm); + } + + /** + * Builder for SCMDatanodeHeartbeatDispatcher. + */ + public static class Builder { + + private final SCMDatanodeReportHandlerFactory reportHandlerFactory; + private final Map, + SCMDatanodeReportHandler> report2handler; + + /** + * Constructs SCMDatanodeHeartbeatDispatcher.Builder instance. + * + * @param conf Configuration object to be used. + * @param scm StorageContainerManager instance to be used for report + * handler initialization. + */ + private Builder(Configuration conf, StorageContainerManager scm) { + this.report2handler = new HashMap<>(); + this.reportHandlerFactory = + new SCMDatanodeReportHandlerFactory(conf, scm); + } + + /** + * Adds new report handler for the given report. + * + * @param report Report for which handler has to be added + * + * @return Builder + */ + public Builder addHandlerFor(Class report) { + report2handler.put(report, reportHandlerFactory.getHandlerFor(report)); + return this; + } + + /** + * Associates the given report handler for the given report. + * + * @param report Report to be associated with + * @param handler Handler to be used for the report + * + * @return Builder + */ + public Builder addHandler(Class report, + SCMDatanodeReportHandler handler) { + report2handler.put(report, handler); + return this; + } + + /** + * Builds and returns {@link SCMDatanodeHeartbeatDispatcher} instance. + * + * @return SCMDatanodeHeartbeatDispatcher + */ + public SCMDatanodeHeartbeatDispatcher build() { + return new SCMDatanodeHeartbeatDispatcher(report2handler); + } + } + +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java new file mode 100644 index 00000000000..fb89b022153 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeNodeReportHandler.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * Handles Datanode Node Report. + */ +public class SCMDatanodeNodeReportHandler extends + SCMDatanodeReportHandler { + + private static final Logger LOG = LoggerFactory.getLogger( + SCMDatanodeNodeReportHandler.class); + + @Override + public void processReport(DatanodeDetails datanodeDetails, + NodeReportProto report) throws IOException { + LOG.debug("Processing node report from {}.", datanodeDetails); + //TODO: add logic to process node report. + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java new file mode 100644 index 00000000000..d3386493c13 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandler.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; + +import java.io.IOException; + +/** + * Datanode Report handlers should implement this interface in order to get + * call back whenever the report is received from datanode. + * + * @param Type of report the handler is interested in. + */ +public abstract class SCMDatanodeReportHandler + implements Configurable { + + private Configuration config; + private StorageContainerManager scm; + + /** + * Initializes SCMDatanodeReportHandler and associates it with the given + * StorageContainerManager instance. + * + * @param storageContainerManager StorageContainerManager instance to be + * associated with. + */ + public void init(StorageContainerManager storageContainerManager) { + this.scm = storageContainerManager; + } + + /** + * Returns the associated StorageContainerManager instance. This will be + * used by the ReportHandler implementations. + * + * @return {@link StorageContainerManager} + */ + protected StorageContainerManager getSCM() { + return scm; + } + + @Override + public void setConf(Configuration conf) { + this.config = conf; + } + + @Override + public Configuration getConf() { + return config; + } + + /** + * Processes the report received from datanode. Each ReportHandler + * implementation is responsible for providing the logic to process the + * report it's interested in. + * + * @param datanodeDetails Datanode Information + * @param report Report to be processed + * + * @throws IOException In case of any exception + */ + abstract void processReport(DatanodeDetails datanodeDetails, T report) + throws IOException; +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java new file mode 100644 index 00000000000..e88495fc23f --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/SCMDatanodeReportHandlerFactory.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import com.google.protobuf.GeneratedMessage; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.util.ReflectionUtils; + +import java.util.HashMap; +import java.util.Map; + + +/** + * Factory class to construct {@link SCMDatanodeReportHandler} given a report. + */ +public class SCMDatanodeReportHandlerFactory { + + private final Configuration conf; + private final StorageContainerManager scm; + private final Map, + Class>> + report2handler; + + /** + * Constructs {@link SCMDatanodeReportHandler} instance. + * + * @param conf Configuration to be passed to the + * {@link SCMDatanodeReportHandler} + */ + public SCMDatanodeReportHandlerFactory(Configuration conf, + StorageContainerManager scm) { + this.conf = conf; + this.scm = scm; + this.report2handler = new HashMap<>(); + + report2handler.put(NodeReportProto.class, + SCMDatanodeNodeReportHandler.class); + report2handler.put(ContainerReportsProto.class, + SCMDatanodeContainerReportHandler.class); + } + + /** + * Returns the SCMDatanodeReportHandler for the corresponding report. + * + * @param report report + * + * @return report handler + */ + public SCMDatanodeReportHandler getHandlerFor( + Class report) { + Class> + handlerClass = report2handler.get(report); + if (handlerClass == null) { + throw new RuntimeException("No handler found for report " + report); + } + SCMDatanodeReportHandler instance = + ReflectionUtils.newInstance(handlerClass, conf); + instance.init(scm); + return instance; + } + +} \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java new file mode 100644 index 00000000000..fda39930967 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/report/package-info.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdds.scm.server.report; +/** + * Handling of all the datanode reports in SCM which are received through + * heartbeat is done here. + * + * SCM Datanode Report Processing State Diagram: + * + * SCMDatanode SCMDatanodeHeartbeat SCMDatanodeReport + * ProtocolServer Dispatcher Handler + * | | | + * | | | + * | construct | | + * |----------------------->| | + * | | | + * | | register | + * | |<-----------------------| + * | | | + * +------------+------------------------+------------------------+--------+ + * | loop | | | | + * | | | | | + * | | | | | + * | heartbeat | | | | + * - +----------->| | | | + * | from | heartbeat | | | + * | Datanode |----------------------->| | | + * | | | report | | + * | | |----------------------->| | + * | | | | | + * | DN | | | | + * <-+------------| | | | + * | commands | | | | + * | | | | | + * +------------+------------------------+------------------------+--------+ + * | | | + * | | | + * | shutdown | | + * |----------------------->| | + * | | | + * | | | + * - - - + */ \ No newline at end of file diff --git a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html b/hadoop-hdds/server-scm/src/main/webapps/scm/index.html index 3407f51be73..2c943b626ff 100644 --- a/hadoop-hdds/server-scm/src/main/webapps/scm/index.html +++ b/hadoop-hdds/server-scm/src/main/webapps/scm/index.html @@ -26,7 +26,7 @@ HDFS Storage Container Manager - + @@ -63,7 +63,7 @@ - + @@ -71,6 +71,6 @@ - + diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java index f3e42ea7316..9fbb9fa5f55 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java @@ -109,8 +109,24 @@ public class TestBlockManager { AllocatedBlock block = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, containerOwner); Assert.assertNotNull(block); + long transactionId = + mapping.getContainer(block.getBlockID().getContainerID()) + .getDeleteTransactionId(); + Assert.assertEquals(0, transactionId); blockManager.deleteBlocks(Collections.singletonList( block.getBlockID())); + Assert.assertEquals(++transactionId, + mapping.getContainer(block.getBlockID().getContainerID()) + .getDeleteTransactionId()); + + block = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, + type, factor, containerOwner); + Assert.assertNotNull(block); + blockManager.deleteBlocks(Collections.singletonList( + block.getBlockID())); + Assert.assertEquals(++transactionId, + mapping.getContainer(block.getBlockID().getContainerID()) + .getDeleteTransactionId()); } @Test diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java new file mode 100644 index 00000000000..09ade3ea630 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.container; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; +import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.container.common.SCMTestUtils; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Random; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATE; +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent.CREATED; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB; + +/** + * Tests the closeContainerEventHandler class. + */ +public class TestCloseContainerEventHandler { + + private static Configuration configuration; + private static MockNodeManager nodeManager; + private static ContainerMapping mapping; + private static long size; + private static File testDir; + private static EventQueue eventQueue; + + @BeforeClass + public static void setUp() throws Exception { + configuration = SCMTestUtils.getConf(); + size = configuration + .getLong(OZONE_SCM_CONTAINER_SIZE_GB, OZONE_SCM_CONTAINER_SIZE_DEFAULT) + * 1024 * 1024 * 1024; + testDir = GenericTestUtils + .getTestDir(TestCloseContainerEventHandler.class.getSimpleName()); + configuration + .set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath()); + nodeManager = new MockNodeManager(true, 10); + mapping = new ContainerMapping(configuration, nodeManager, 128); + eventQueue = new EventQueue(); + eventQueue.addHandler(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, + new CloseContainerEventHandler(mapping)); + } + + @AfterClass + public static void tearDown() throws Exception { + if (mapping != null) { + mapping.close(); + } + FileUtil.fullyDelete(testDir); + } + + @Test + public void testIfCloseContainerEventHadnlerInvoked() { + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(CloseContainerEventHandler.LOG); + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, + new ContainerID(Math.abs(new Random().nextLong()))); + eventQueue.processAll(1000); + Assert.assertTrue(logCapturer.getOutput() + .contains("Close container Event triggered for container")); + } + + @Test + public void testCloseContainerEventWithInvalidContainer() { + long id = Math.abs(new Random().nextLong()); + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(CloseContainerEventHandler.LOG); + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, + new ContainerID(id)); + eventQueue.processAll(1000); + Assert.assertTrue(logCapturer.getOutput() + .contains("Container with id : " + id + " does not exist")); + } + + @Test + public void testCloseContainerEventWithValidContainers() throws IOException { + + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(CloseContainerEventHandler.LOG); + ContainerInfo info = mapping + .allocateContainer(HddsProtos.ReplicationType.STAND_ALONE, + HddsProtos.ReplicationFactor.ONE, "ozone"); + ContainerID id = new ContainerID(info.getContainerID()); + DatanodeDetails datanode = info.getPipeline().getLeader(); + int closeCount = nodeManager.getCommandCount(datanode); + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, id); + eventQueue.processAll(1000); + // At this point of time, the allocated container is not in open + // state, so firing close container event should not queue CLOSE + // command in the Datanode + Assert.assertEquals(0, nodeManager.getCommandCount(datanode)); + // Make sure the information is logged + Assert.assertTrue(logCapturer.getOutput().contains( + "container with id : " + id.getId() + + " is in ALLOCATED state and need not be closed")); + //Execute these state transitions so that we can close the container. + mapping.updateContainerState(id.getId(), CREATE); + mapping.updateContainerState(id.getId(), CREATED); + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, + new ContainerID(info.getContainerID())); + eventQueue.processAll(1000); + Assert.assertEquals(closeCount + 1, nodeManager.getCommandCount(datanode)); + Assert.assertEquals(HddsProtos.LifeCycleState.CLOSING, + mapping.getStateManager().getContainer(id).getState()); + } + + @Test + public void testCloseContainerEventWithRatis() throws IOException { + + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer + .captureLogs(CloseContainerEventHandler.LOG); + ContainerInfo info = mapping + .allocateContainer(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE, "ozone"); + ContainerID id = new ContainerID(info.getContainerID()); + int[] closeCount = new int[3]; + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, id); + eventQueue.processAll(1000); + int i = 0; + for (DatanodeDetails details : info.getPipeline().getMachines()) { + closeCount[i] = nodeManager.getCommandCount(details); + i++; + } + i = 0; + for (DatanodeDetails details : info.getPipeline().getMachines()) { + Assert.assertEquals(closeCount[i], nodeManager.getCommandCount(details)); + i++; + } + // Make sure the information is logged + Assert.assertTrue(logCapturer.getOutput().contains( + "container with id : " + id.getId() + + " is in ALLOCATED state and need not be closed")); + //Execute these state transitions so that we can close the container. + mapping.updateContainerState(id.getId(), CREATE); + mapping.updateContainerState(id.getId(), CREATED); + eventQueue.fireEvent(CloseContainerEventHandler.CLOSE_CONTAINER_EVENT, + new ContainerID(info.getContainerID())); + eventQueue.processAll(1000); + i = 0; + // Make sure close is queued for each datanode on the pipeline + for (DatanodeDetails details : info.getPipeline().getMachines()) { + Assert.assertEquals(closeCount[i] + 1, + nodeManager.getCommandCount(details)); + Assert.assertEquals(HddsProtos.LifeCycleState.CLOSING, + mapping.getStateManager().getContainer(id).getState()); + i++; + } + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java index ba2ab64a766..eefb639d59e 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java @@ -203,7 +203,8 @@ public class TestContainerMapping { .setWriteCount(100000000L) .setReadBytes(2000000000L) .setWriteBytes(2000000000L) - .setContainerID(info.getContainerID()); + .setContainerID(info.getContainerID()) + .setDeleteTransactionId(0); reports.add(ciBuilder.build()); @@ -237,7 +238,8 @@ public class TestContainerMapping { .setWriteCount(500000000L) .setReadBytes(5368705120L) .setWriteBytes(5368705120L) - .setContainerID(info.getContainerID()); + .setContainerID(info.getContainerID()) + .setDeleteTransactionId(0); reports.add(ciBuilder.build()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java index 0a3efda8302..0d7848f7987 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java @@ -212,7 +212,8 @@ public class TestContainerCloser { .setReadCount(100000000L) .setWriteCount(100000000L) .setReadBytes(2000000000L) - .setWriteBytes(2000000000L); + .setWriteBytes(2000000000L) + .setDeleteTransactionId(0); reports.addReports(ciBuilder); mapping.processContainerReports(TestUtils.getDatanodeDetails(), reports.build()); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java new file mode 100644 index 00000000000..776ae887541 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeContainerReportHandler.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test cases to verify SCMDatanodeContainerReportHandler's behavior. + */ +public class TestSCMDatanodeContainerReportHandler { + + //TODO: add test cases to verify SCMDatanodeContainerReportHandler. + + @Test + public void dummyTest() { + Assert.assertTrue(true); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java new file mode 100644 index 00000000000..5d086471c11 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeHeartbeatDispatcher.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.apache.hadoop.hdds.scm.TestUtils; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * This class tests the behavior of SCMDatanodeHeartbeatDispatcher. + */ +public class TestSCMDatanodeHeartbeatDispatcher { + + @Test + public void testSCMDatanodeHeartbeatDispatcherBuilder() { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeHeartbeatDispatcher dispatcher = + SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null) + .addHandlerFor(NodeReportProto.class) + .addHandlerFor(ContainerReportsProto.class) + .build(); + Assert.assertNotNull(dispatcher); + } + + @Test + public void testNodeReportDispatcher() throws IOException { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeNodeReportHandler nodeReportHandler = + Mockito.mock(SCMDatanodeNodeReportHandler.class); + SCMDatanodeHeartbeatDispatcher dispatcher = + SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null) + .addHandler(NodeReportProto.class, nodeReportHandler) + .build(); + + DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(); + NodeReportProto nodeReport = NodeReportProto.getDefaultInstance(); + SCMHeartbeatRequestProto heartbeat = + SCMHeartbeatRequestProto.newBuilder() + .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) + .setNodeReport(nodeReport) + .build(); + dispatcher.dispatch(heartbeat); + verify(nodeReportHandler, + times(1)) + .processReport(any(DatanodeDetails.class), eq(nodeReport)); + } + + @Test + public void testContainerReportDispatcher() throws IOException { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeContainerReportHandler containerReportHandler = + Mockito.mock(SCMDatanodeContainerReportHandler.class); + SCMDatanodeHeartbeatDispatcher dispatcher = + SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null) + .addHandler(ContainerReportsProto.class, containerReportHandler) + .build(); + + DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(); + ContainerReportsProto containerReport = + ContainerReportsProto.getDefaultInstance(); + SCMHeartbeatRequestProto heartbeat = + SCMHeartbeatRequestProto.newBuilder() + .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) + .setContainerReport(containerReport) + .build(); + dispatcher.dispatch(heartbeat); + verify(containerReportHandler, + times(1)) + .processReport(any(DatanodeDetails.class), + any(ContainerReportsProto.class)); + } + + @Test + public void testNodeAndContainerReportDispatcher() throws IOException { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeNodeReportHandler nodeReportHandler = + Mockito.mock(SCMDatanodeNodeReportHandler.class); + SCMDatanodeContainerReportHandler containerReportHandler = + Mockito.mock(SCMDatanodeContainerReportHandler.class); + SCMDatanodeHeartbeatDispatcher dispatcher = + SCMDatanodeHeartbeatDispatcher.newBuilder(conf, null) + .addHandler(NodeReportProto.class, nodeReportHandler) + .addHandler(ContainerReportsProto.class, containerReportHandler) + .build(); + + DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(); + NodeReportProto nodeReport = NodeReportProto.getDefaultInstance(); + ContainerReportsProto containerReport = + ContainerReportsProto.getDefaultInstance(); + SCMHeartbeatRequestProto heartbeat = + SCMHeartbeatRequestProto.newBuilder() + .setDatanodeDetails(datanodeDetails.getProtoBufMessage()) + .setNodeReport(nodeReport) + .setContainerReport(containerReport) + .build(); + dispatcher.dispatch(heartbeat); + verify(nodeReportHandler, + times(1)) + .processReport(any(DatanodeDetails.class), any(NodeReportProto.class)); + verify(containerReportHandler, + times(1)) + .processReport(any(DatanodeDetails.class), + any(ContainerReportsProto.class)); + } + +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java new file mode 100644 index 00000000000..30a753c0240 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeNodeReportHandler.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test cases to verify TestSCMDatanodeNodeReportHandler's behavior. + */ +public class TestSCMDatanodeNodeReportHandler { + + + //TODO: add test cases to verify SCMDatanodeNodeReportHandler. + + @Test + public void dummyTest() { + Assert.assertTrue(true); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java new file mode 100644 index 00000000000..4b918f76c7b --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/TestSCMDatanodeReportHandlerFactory.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; +import org.apache.hadoop.hdds.protocol.proto + .StorageContainerDatanodeProtocolProtos.NodeReportProto; +import org.junit.Assert; +import org.junit.Test; + +/** + * Test cases to verify the functionality of SCMDatanodeReportHandlerFactory. + */ +public class TestSCMDatanodeReportHandlerFactory { + + @Test + public void testNodeReportHandlerConstruction() { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeReportHandlerFactory factory = + new SCMDatanodeReportHandlerFactory(conf, null); + Assert.assertTrue(factory.getHandlerFor(NodeReportProto.class) + instanceof SCMDatanodeNodeReportHandler); + } + + @Test + public void testContainerReporttHandlerConstruction() { + Configuration conf = new OzoneConfiguration(); + SCMDatanodeReportHandlerFactory factory = + new SCMDatanodeReportHandlerFactory(conf, null); + Assert.assertTrue(factory.getHandlerFor(ContainerReportsProto.class) + instanceof SCMDatanodeContainerReportHandler); + } +} diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java new file mode 100644 index 00000000000..4a3f59f0167 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/report/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.hdds.scm.server.report; +/** + * Contains test-cases to test Datanode report handlers in SCM. + */ \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/CannotObtainBlockLengthException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/CannotObtainBlockLengthException.java new file mode 100644 index 00000000000..6da1d67b9ae --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/CannotObtainBlockLengthException.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; + +import java.io.IOException; + +/** + * This exception is thrown when the length of a LocatedBlock instance + * can not be obtained. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public class CannotObtainBlockLengthException extends IOException { + private static final long serialVersionUID = 1L; + + public CannotObtainBlockLengthException() { + super(); + } + + public CannotObtainBlockLengthException(String message){ + super(message); + } + + /** + * Constructs an {@code CannotObtainBlockLengthException} with the + * specified LocatedBlock that failed to obtain block length. + * + * @param locatedBlock + * The LocatedBlock instance which block length can not be obtained + */ + public CannotObtainBlockLengthException(LocatedBlock locatedBlock) { + super("Cannot obtain block length for " + locatedBlock); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 5f1b2bb1721..96c45053fc4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2942,6 +2942,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } } + void updateFileSystemECReadStats(int nRead) { + if (stats != null) { + stats.incrementBytesReadErasureCoded(nRead); + } + } + /** * Create hedged reads thread pool, HEDGED_READ_THREAD_POOL, if * it does not already exist. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java index b38e6299030..573b860fab1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java @@ -61,6 +61,7 @@ import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks; import org.apache.hadoop.hdfs.client.impl.BlockReaderFactory; import org.apache.hadoop.hdfs.client.impl.DfsClientConf; +import org.apache.hadoop.hdfs.protocol.BlockType; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; @@ -359,7 +360,7 @@ public class DFSInputStream extends FSInputStream return 0; } - throw new IOException("Cannot obtain block length for " + locatedblock); + throw new CannotObtainBlockLengthException(locatedblock); } public long getFileLength() { @@ -593,8 +594,8 @@ public class DFSInputStream extends FSInputStream fetchBlockAt(target); } else { connectFailedOnce = true; - DFSClient.LOG.warn("Failed to connect to {} for block {}, " + - "add to deadNodes and continue. ", targetAddr, + DFSClient.LOG.warn("Failed to connect to {} for file {} for block " + + "{}, add to deadNodes and continue. ", targetAddr, src, targetBlock.getBlock(), ex); // Put chosen node into dead list, continue addToDeadNodes(chosenNode); @@ -1082,6 +1083,9 @@ public class DFSInputStream extends FSInputStream IOUtilsClient.updateReadStatistics(readStatistics, nread, reader); dfsClient.updateFileSystemReadStats( reader.getNetworkDistance(), nread); + if (readStatistics.getBlockType() == BlockType.STRIPED) { + dfsClient.updateFileSystemECReadStats(nread); + } if (nread != len) { throw new IOException("truncated return from reader.read(): " + "excpected " + len + ", got " + nread); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java index c984c3b7322..39ad2ff4a9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReaderStrategy.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs; +import org.apache.hadoop.hdfs.protocol.BlockType; + import java.io.IOException; import java.nio.ByteBuffer; import static org.apache.hadoop.hdfs.util.IOUtilsClient.updateReadStatistics; @@ -121,6 +123,9 @@ class ByteArrayStrategy implements ReaderStrategy { updateReadStatistics(readStatistics, nRead, blockReader); dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(), nRead); + if (readStatistics.getBlockType() == BlockType.STRIPED) { + dfsClient.updateFileSystemECReadStats(nRead); + } offset += nRead; } return nRead; @@ -188,6 +193,9 @@ class ByteBufferStrategy implements ReaderStrategy { updateReadStatistics(readStatistics, nRead, blockReader); dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(), nRead); + if (readStatistics.getBlockType() == BlockType.STRIPED) { + dfsClient.updateFileSystemECReadStats(nRead); + } } return nRead; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java index e9c8791c5c5..96722fcfab3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/ConfiguredFailoverProxyProvider.java @@ -52,11 +52,11 @@ public class ConfiguredFailoverProxyProvider extends protected final Configuration conf; protected final List> proxies = new ArrayList>(); - private final UserGroupInformation ugi; + protected final UserGroupInformation ugi; protected final Class xface; private int currentProxyIndex = 0; - private final HAProxyFactory factory; + protected final HAProxyFactory factory; public ConfiguredFailoverProxyProvider(Configuration conf, URI uri, Class xface, HAProxyFactory factory) { @@ -94,9 +94,7 @@ public class ConfiguredFailoverProxyProvider extends proxies.add(new AddressRpcProxyPair(address)); } // Randomize the list to prevent all clients pointing to the same one - boolean randomized = conf.getBoolean( - HdfsClientConfigKeys.Failover.RANDOM_ORDER, - HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + boolean randomized = getRandomOrder(conf, uri); if (randomized) { Collections.shuffle(proxies); } @@ -111,6 +109,31 @@ public class ConfiguredFailoverProxyProvider extends } } + /** + * Check whether random order is configured for failover proxy provider + * for the namenode/nameservice. + * + * @param conf Configuration + * @param nameNodeUri The URI of namenode/nameservice + * @return random order configuration + */ + private static boolean getRandomOrder( + Configuration conf, URI nameNodeUri) { + String host = nameNodeUri.getHost(); + String configKeyWithHost = HdfsClientConfigKeys.Failover.RANDOM_ORDER + + "." + host; + + if (conf.get(configKeyWithHost) != null) { + return conf.getBoolean( + configKeyWithHost, + HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + } + + return conf.getBoolean( + HdfsClientConfigKeys.Failover.RANDOM_ORDER, + HdfsClientConfigKeys.Failover.RANDOM_ORDER_DEFAULT); + } + @Override public Class getInterface() { return xface; @@ -122,6 +145,10 @@ public class ConfiguredFailoverProxyProvider extends @Override public synchronized ProxyInfo getProxy() { AddressRpcProxyPair current = proxies.get(currentProxyIndex); + return getProxy(current); + } + + protected ProxyInfo getProxy(AddressRpcProxyPair current) { if (current.namenode == null) { try { current.namenode = factory.createProxy(conf, @@ -147,7 +174,7 @@ public class ConfiguredFailoverProxyProvider extends * A little pair object to store the address and connected RPC proxy object to * an NN. Note that {@link AddressRpcProxyPair#namenode} may be null. */ - private static class AddressRpcProxyPair { + protected static class AddressRpcProxyPair { public final InetSocketAddress address; public T namenode; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier similarity index 100% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer similarity index 100% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer rename to hadoop-hdfs-project/hadoop-hdfs-client/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandlerFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandlerFactory.java index 910fee2b071..53cd557541a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandlerFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/fs/TestUrlStreamHandlerFactory.java @@ -18,14 +18,19 @@ package org.apache.hadoop.fs; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; +import java.io.File; +import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; /** * Test of the URL stream handler factory. @@ -35,7 +40,9 @@ public class TestUrlStreamHandlerFactory { private static final int RUNS = 20; private static final int THREADS = 10; private static final int TASKS = 200; - private static final int TIMEOUT = 30; + + @Rule + public Timeout globalTimeout = new Timeout(30000); @Test public void testConcurrency() throws Exception { @@ -62,12 +69,6 @@ public class TestUrlStreamHandlerFactory { } executor.shutdown(); - try { - executor.awaitTermination(TIMEOUT, TimeUnit.SECONDS); - executor.shutdownNow(); - } catch (InterruptedException e) { - // pass - } // check for exceptions for (Future future : futures) { @@ -77,4 +78,23 @@ public class TestUrlStreamHandlerFactory { future.get(); } } + + @Test + public void testFsUrlStreamHandlerFactory() throws IOException { + File myFile = new File(GenericTestUtils.getTestDir(), "foo bar.txt"); + myFile.createNewFile(); + + // Create URL directly from File (JRE builds it). + URL myUrl = myFile.toURI().toURL(); + + // Succeeds. + myUrl.openStream().close(); + + // Replace handling of file: scheme with FsUrlStreamHandler. + URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); + + URL myUrl2 = myFile.toURI().toURL(); + + myUrl2.openStream(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java new file mode 100644 index 00000000000..d7a5db6b893 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestConfiguredFailoverProxyProvider.java @@ -0,0 +1,264 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode.ha; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.slf4j.event.Level; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test {@link ConfiguredFailoverProxyProvider}. + * This manages failover logic for a given set of nameservices/namenodes + * (aka proxies). + */ +public class TestConfiguredFailoverProxyProvider { + private Configuration conf; + private int rpcPort = 8020; + private URI ns1Uri; + private URI ns2Uri; + private String ns1; + private String ns1nn1Hostname = "machine1.foo.bar"; + private InetSocketAddress ns1nn1 = + new InetSocketAddress(ns1nn1Hostname, rpcPort); + private String ns1nn2Hostname = "machine2.foo.bar"; + private InetSocketAddress ns1nn2 = + new InetSocketAddress(ns1nn2Hostname, rpcPort); + private String ns2; + private String ns2nn1Hostname = "router1.foo.bar"; + private InetSocketAddress ns2nn1 = + new InetSocketAddress(ns2nn1Hostname, rpcPort); + private String ns2nn2Hostname = "router2.foo.bar"; + private InetSocketAddress ns2nn2 = + new InetSocketAddress(ns2nn2Hostname, rpcPort); + private String ns2nn3Hostname = "router3.foo.bar"; + private InetSocketAddress ns2nn3 = + new InetSocketAddress(ns2nn3Hostname, rpcPort); + private static final int NUM_ITERATIONS = 50; + + @BeforeClass + public static void setupClass() throws Exception { + GenericTestUtils.setLogLevel(RequestHedgingProxyProvider.LOG, Level.TRACE); + } + + @Before + public void setup() throws URISyntaxException { + ns1 = "mycluster-1-" + Time.monotonicNow(); + ns1Uri = new URI("hdfs://" + ns1); + conf = new Configuration(); + conf.set( + HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns1, + "nn1,nn2,nn3"); + conf.set( + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns1 + ".nn1", + ns1nn1Hostname + ":" + rpcPort); + conf.set( + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns1 + ".nn2", + ns1nn2Hostname + ":" + rpcPort); + conf.set( + HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + ns1, + ConfiguredFailoverProxyProvider.class.getName()); + conf.setBoolean( + HdfsClientConfigKeys.Failover.RANDOM_ORDER + "." + ns1, + false); + + ns2 = "myroutercluster-2-" + Time.monotonicNow(); + ns2Uri = new URI("hdfs://" + ns2); + conf.set( + HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns2, + "nn1,nn2,nn3"); + conf.set( + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns2 + ".nn1", + ns2nn1Hostname + ":" + rpcPort); + conf.set( + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns2 + ".nn2", + ns2nn2Hostname + ":" + rpcPort); + conf.set( + HdfsClientConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + ns2 + ".nn3", + ns2nn3Hostname + ":" + rpcPort); + conf.set( + HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + "." + ns2, + ConfiguredFailoverProxyProvider.class.getName()); + conf.setBoolean( + HdfsClientConfigKeys.Failover.RANDOM_ORDER + "." + ns2, + true); + + conf.set(HdfsClientConfigKeys.DFS_NAMESERVICES, ns1 + "," + ns2); + conf.set("fs.defaultFS", "hdfs://" + ns1); + } + + /** + * Tests getProxy with random.order configuration set to false. + * This expects the proxy order to be consistent every time a new + * ConfiguredFailoverProxyProvider is created. + */ + @Test + public void testNonRandomGetProxy() throws Exception { + final AtomicInteger nn1Count = new AtomicInteger(0); + final AtomicInteger nn2Count = new AtomicInteger(0); + + Map proxyMap = new HashMap<>(); + + final ClientProtocol nn1Mock = mock(ClientProtocol.class); + when(nn1Mock.getStats()).thenAnswer(createAnswer(nn1Count, 1)); + proxyMap.put(ns1nn1, nn1Mock); + + final ClientProtocol nn2Mock = mock(ClientProtocol.class); + when(nn2Mock.getStats()).thenAnswer(createAnswer(nn2Count, 2)); + proxyMap.put(ns1nn2, nn2Mock); + + ConfiguredFailoverProxyProvider provider1 = + new ConfiguredFailoverProxyProvider<>(conf, ns1Uri, + ClientProtocol.class, createFactory(proxyMap)); + ClientProtocol proxy1 = provider1.getProxy().proxy; + proxy1.getStats(); + assertEquals(1, nn1Count.get()); + assertEquals(0, nn2Count.get()); + proxy1.getStats(); + assertEquals(2, nn1Count.get()); + assertEquals(0, nn2Count.get()); + nn1Count.set(0); + nn2Count.set(0); + + for (int i = 0; i < NUM_ITERATIONS; i++) { + ConfiguredFailoverProxyProvider provider2 = + new ConfiguredFailoverProxyProvider<>(conf, ns1Uri, + ClientProtocol.class, createFactory(proxyMap)); + ClientProtocol proxy2 = provider2.getProxy().proxy; + proxy2.getStats(); + } + assertEquals(NUM_ITERATIONS, nn1Count.get()); + assertEquals(0, nn2Count.get()); + } + + /** + * Tests getProxy with random.order configuration set to true. + * This expects the proxy order to be random every time a new + * ConfiguredFailoverProxyProvider is created. + */ + @Test + public void testRandomGetProxy() throws Exception { + final AtomicInteger nn1Count = new AtomicInteger(0); + final AtomicInteger nn2Count = new AtomicInteger(0); + final AtomicInteger nn3Count = new AtomicInteger(0); + + Map proxyMap = new HashMap<>(); + + final ClientProtocol nn1Mock = mock(ClientProtocol.class); + when(nn1Mock.getStats()).thenAnswer(createAnswer(nn1Count, 1)); + proxyMap.put(ns2nn1, nn1Mock); + + final ClientProtocol nn2Mock = mock(ClientProtocol.class); + when(nn2Mock.getStats()).thenAnswer(createAnswer(nn2Count, 2)); + proxyMap.put(ns2nn2, nn2Mock); + + final ClientProtocol nn3Mock = mock(ClientProtocol.class); + when(nn3Mock.getStats()).thenAnswer(createAnswer(nn3Count, 3)); + proxyMap.put(ns2nn3, nn3Mock); + + + for (int i = 0; i < NUM_ITERATIONS; i++) { + ConfiguredFailoverProxyProvider provider = + new ConfiguredFailoverProxyProvider<>(conf, ns2Uri, + ClientProtocol.class, createFactory(proxyMap)); + ClientProtocol proxy = provider.getProxy().proxy; + proxy.getStats(); + } + + assertTrue(nn1Count.get() < NUM_ITERATIONS && nn1Count.get() > 0); + assertTrue(nn2Count.get() < NUM_ITERATIONS && nn2Count.get() > 0); + assertTrue(nn3Count.get() < NUM_ITERATIONS && nn3Count.get() > 0); + assertEquals(NUM_ITERATIONS, + nn1Count.get() + nn2Count.get() + nn3Count.get()); + } + + /** + * createAnswer creates an Answer for using with the ClientProtocol mocks. + * @param counter counter to increment + * @param retVal return value from answer + * @return + */ + private Answer createAnswer(final AtomicInteger counter, + final long retVal) { + return new Answer() { + @Override + public long[] answer(InvocationOnMock invocation) throws Throwable { + counter.incrementAndGet(); + return new long[]{retVal}; + } + }; + } + + /** + * createFactory returns a HAProxyFactory for tests. + * This uses a map of name node address to ClientProtocol to route calls to + * different ClientProtocol objects. The tests could create ClientProtocol + * mocks and create name node mappings to use with + * ConfiguredFailoverProxyProvider. + */ + private HAProxyFactory createFactory( + final Map proxies) { + final Map proxyMap = proxies; + return new HAProxyFactory() { + @Override + public ClientProtocol createProxy(Configuration cfg, + InetSocketAddress nnAddr, Class xface, + UserGroupInformation ugi, boolean withRetries, + AtomicBoolean fallbackToSimpleAuth) throws IOException { + if (proxyMap.containsKey(nnAddr)) { + return proxyMap.get(nnAddr); + } else { + throw new IOException("Name node address not found"); + } + } + + @Override + public ClientProtocol createProxy(Configuration cfg, + InetSocketAddress nnAddr, Class xface, + UserGroupInformation ugi, boolean withRetries) throws IOException { + if (proxyMap.containsKey(nnAddr)) { + return proxyMap.get(nnAddr); + } else { + throw new IOException("Name node address not found"); + } + } + }; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h index 9805bad1c0f..a6ec97ad491 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/ioservice.h @@ -58,6 +58,7 @@ #ifndef INCLUDE_HDFSPP_IOSERVICE_H_ #define INCLUDE_HDFSPP_IOSERVICE_H_ +#include #include // forward decl diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h index 2b36f590b4a..ea176532f23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h @@ -59,7 +59,7 @@ public: void Connect(std::function dn)> handler) override {(void)handler; /*TODO: Handshaking goes here*/}; - void Cancel(); + void Cancel() override; private: DataTransferSaslStream(const DataTransferSaslStream &) = delete; DataTransferSaslStream &operator=(const DataTransferSaslStream &) = delete; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc index 0957ea377c9..6fc04f754f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc @@ -91,8 +91,10 @@ void SaslProtocol::Authenticate(std::function resp_msg = std::make_shared(); auto self(shared_from_this()); connection->AsyncRpc_locked(SASL_METHOD_NAME, req_msg.get(), resp_msg, - [self, req_msg, resp_msg] (const Status & status) { - self->OnServerResponse(status, resp_msg.get()); } ); + [self, req_msg, resp_msg, connection] (const Status & status) { + assert(connection); + self->OnServerResponse(status, resp_msg.get()); + }); } // authenticate() method AuthInfo::AuthMethod ParseMethod(const std::string & method) @@ -340,9 +342,10 @@ bool SaslProtocol::SendSaslMessage(RpcSaslProto & message) std::shared_ptr resp_msg = std::make_shared(); auto self(shared_from_this()); connection->AsyncRpc(SASL_METHOD_NAME, &message, resp_msg, - [self, resp_msg] (const Status & status) { - self->OnServerResponse(status, resp_msg.get()); - } ); + [self, resp_msg, connection] (const Status & status) { + assert(connection); + self->OnServerResponse(status, resp_msg.get()); + }); return true; } // SendSaslMessage() method @@ -370,7 +373,9 @@ bool SaslProtocol::AuthComplete(const Status & status, const AuthInfo & auth_inf void SaslProtocol::OnServerResponse(const Status & status, const hadoop::common::RpcSaslProto * response) { + std::lock_guard state_lock(sasl_state_lock_); + LOG_TRACE(kRPC, << "Received SASL response: " << status.ToString()); if (status.ok()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h index de234efd8d8..82db7604213 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h @@ -87,7 +87,7 @@ public: virtual void cancel() {} virtual void close() {} protected: - virtual ProducerResult Produce() = 0; + ProducerResult Produce() override = 0; ::asio::io_service *io_service_; private: diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc index 4b909b23942..3997e64be5c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc @@ -82,7 +82,7 @@ public: this->MockConnectionBase::async_write_some(buf, handler); } - void Cancel() { + void Cancel() override { /* no-op, declared pure virtual */ } }; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java index 6b416dd25e5..5fcde5b2e85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ConnectionPool.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.net.SocketFactory; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -148,6 +149,14 @@ public class ConnectionPool { return this.connectionPoolId; } + /** + * Get the clientIndex used to calculate index for lookup. + */ + @VisibleForTesting + public AtomicInteger getClientIndex() { + return this.clientIndex; + } + /** * Return the next connection round-robin. * @@ -161,7 +170,8 @@ public class ConnectionPool { ConnectionContext conn = null; List tmpConnections = this.connections; int size = tmpConnections.size(); - int threadIndex = this.clientIndex.getAndIncrement(); + // Inc and mask off sign bit, lookup index should be non-negative int + int threadIndex = this.clientIndex.getAndIncrement() & 0x7FFFFFFF; for (int i=0; i zkAcl; @Override @@ -83,6 +86,7 @@ public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl { try { this.zkManager = new ZKCuratorManager(conf); this.zkManager.start(); + this.zkAcl = ZKCuratorManager.getZKAcls(conf); } catch (IOException e) { LOG.error("Cannot initialize the ZK connection", e); return false; @@ -95,7 +99,7 @@ public class StateStoreZooKeeperImpl extends StateStoreSerializableImpl { String className, Class clazz) { try { String checkPath = getNodePath(baseZNode, className); - zkManager.createRootDirRecursively(checkPath); + zkManager.createRootDirRecursively(checkPath, zkAcl); return true; } catch (Exception e) { LOG.error("Cannot initialize ZK node for {}: {}", diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html index 37fcb928121..7359087d63c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html @@ -19,7 +19,7 @@ - + Router Information @@ -425,9 +425,9 @@ - + - + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java index a7316481d7e..0e1eb40783f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java @@ -32,6 +32,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assert.assertNotNull; /** * Test functionalities of {@link ConnectionManager}, which manages a pool @@ -149,6 +150,18 @@ public class TestConnectionManager { assertTrue(cc.isActive()); } + @Test + public void testValidClientIndex() throws Exception { + ConnectionPool pool = new ConnectionPool( + conf, TEST_NN_ADDRESS, TEST_USER1, 2, 2, ClientProtocol.class); + for(int i = -3; i <= 3; i++) { + pool.getClientIndex().set(i); + ConnectionContext conn = pool.getConnection(); + assertNotNull(conn); + assertTrue(conn.isUsable()); + } + } + @Test public void getGetConnectionNamenodeProtocol() throws Exception { Map poolMap = connManager.getPools(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.3.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.3.xml new file mode 100644 index 00000000000..eea6c599a22 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/jdiff/Apache_Hadoop_HDFS_3.0.3.xml @@ -0,0 +1,322 @@ + + + + + + + + + + + A distributed implementation of {@link +org.apache.hadoop.fs.FileSystem}. This is loosely modelled after +Google's GFS.

+ +

The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.

]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + This method must return as quickly as possible, since it's called + in a critical section of the NameNode's operation. + + @param succeeded Whether authorization succeeded. + @param userName Name of the user executing the request. + @param addr Remote address of the request. + @param cmd The requested command. + @param src Path of affected source file. + @param dst Path of affected destination file (if any). + @param stat File information for operations that change the file's + metadata (permissions, owner, times, etc).]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index f8b17223e3e..eaf9361e9f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -392,11 +392,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> src/main/webapps/secondary/robots.txt src/contrib/** src/site/resources/images/* - src/main/webapps/static/bootstrap-3.0.2/** + src/main/webapps/static/bootstrap-3.3.7/** src/main/webapps/static/moment.min.js src/main/webapps/static/dust-full-2.0.0.min.js src/main/webapps/static/dust-helpers-1.1.1.min.js - src/main/webapps/static/jquery-1.10.2.min.js + src/main/webapps/static/jquery-3.3.1.min.js src/main/webapps/static/jquery.dataTables.min.js src/main/webapps/static/json-bignum.js src/main/webapps/static/dataTables.bootstrap.css diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 518e62c6b2e..c94232fbcb1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -485,9 +485,13 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { throws NotEnoughReplicasException { final int numOfResults = results.size(); if (numOfResults == 0) { - writer = chooseLocalStorage(writer, excludedNodes, blocksize, - maxNodesPerRack, results, avoidStaleNodes, storageTypes, true) - .getDatanodeDescriptor(); + DatanodeStorageInfo storageInfo = chooseLocalStorage(writer, + excludedNodes, blocksize, maxNodesPerRack, results, avoidStaleNodes, + storageTypes, true); + + writer = (storageInfo != null) ? storageInfo.getDatanodeDescriptor() + : null; + if (--numOfReplicas == 0) { return writer; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index e6cd513881f..9ebc693a235 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1834,7 +1834,7 @@ public class DatanodeManager { } public void markAllDatanodesStale() { - LOG.info("Marking all datandoes as stale"); + LOG.info("Marking all datanodes as stale"); synchronized (this) { for (DatanodeDescriptor dn : datanodeMap.values()) { for(DatanodeStorageInfo storage : dn.getStorageInfos()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java index 6ab6425ad79..3889e2317c2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/DatasetVolumeChecker.java @@ -46,6 +46,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; @@ -103,6 +104,8 @@ public class DatasetVolumeChecker { private static final VolumeCheckContext IGNORED_CONTEXT = new VolumeCheckContext(); + private final ExecutorService checkVolumeResultHandlerExecutorService; + /** * @param conf Configuration object. * @param timer {@link Timer} object used for throttling checks. @@ -163,6 +166,12 @@ public class DatasetVolumeChecker { .setNameFormat("DataNode DiskChecker thread %d") .setDaemon(true) .build())); + + checkVolumeResultHandlerExecutorService = Executors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setNameFormat("VolumeCheck ResultHandler thread %d") + .setDaemon(true) + .build()); } /** @@ -292,7 +301,9 @@ public class DatasetVolumeChecker { numVolumeChecks.incrementAndGet(); Futures.addCallback(olf.get(), new ResultHandler(volumeReference, new HashSet<>(), new HashSet<>(), - new AtomicLong(1), callback)); + new AtomicLong(1), callback), + checkVolumeResultHandlerExecutorService + ); return true; } else { IOUtils.cleanup(null, volumeReference); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 3a32db45875..7160b861f77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.XAttr; @@ -344,16 +345,28 @@ final class FSDirErasureCodingOp { } /** - * Check if the file or directory has an erasure coding policy. + * Get the erasure coding policy information for specified path and policy + * name. If ec policy name is given, it will be parsed and the corresponding + * policy will be returned. Otherwise, get the policy from the parents of the + * iip. * * @param fsn namespace + * @param ecPolicyName the ec policy name * @param iip inodes in the path containing the file - * @return Whether the file or directory has an erasure coding policy. + * @return {@link ErasureCodingPolicy}, or null if no policy is found * @throws IOException */ - static boolean hasErasureCodingPolicy(final FSNamesystem fsn, - final INodesInPath iip) throws IOException { - return unprotectedGetErasureCodingPolicy(fsn, iip) != null; + static ErasureCodingPolicy getErasureCodingPolicy(FSNamesystem fsn, + String ecPolicyName, INodesInPath iip) throws IOException { + ErasureCodingPolicy ecPolicy; + if (!StringUtils.isEmpty(ecPolicyName)) { + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( + fsn, ecPolicyName); + } else { + ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( + fsn, iip); + } + return ecPolicy; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index 8f34e1c6217..03c349c3dee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.base.Preconditions; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.hdfs.AddBlockFlag; @@ -543,13 +542,8 @@ class FSDirWriteFileOp { boolean isStriped = false; ErasureCodingPolicy ecPolicy = null; if (!shouldReplicate) { - if (!StringUtils.isEmpty(ecPolicyName)) { - ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicyByName( - fsd.getFSNamesystem(), ecPolicyName); - } else { - ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy( - fsd.getFSNamesystem(), existing); - } + ecPolicy = FSDirErasureCodingOp.getErasureCodingPolicy( + fsd.getFSNamesystem(), ecPolicyName, existing); if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) { isStriped = true; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index b0fe60a77bb..82e35bd353e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -138,7 +138,7 @@ public class FSEditLogLoader { long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId) throws IOException { - return loadFSEdits(edits, expectedStartingTxId, null, null); + return loadFSEdits(edits, expectedStartingTxId, Long.MAX_VALUE, null, null); } /** @@ -147,6 +147,7 @@ public class FSEditLogLoader { * along. */ long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId, + long maxTxnsToRead, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { StartupProgress prog = NameNode.getStartupProgress(); Step step = createStartupProgressStep(edits); @@ -154,9 +155,10 @@ public class FSEditLogLoader { fsNamesys.writeLock(); try { long startTime = monotonicNow(); - FSImage.LOG.info("Start loading edits file " + edits.getName()); + FSImage.LOG.info("Start loading edits file " + edits.getName() + + " maxTxnsToRead = " + maxTxnsToRead); long numEdits = loadEditRecords(edits, false, expectedStartingTxId, - startOpt, recovery); + maxTxnsToRead, startOpt, recovery); FSImage.LOG.info("Edits file " + edits.getName() + " of size " + edits.length() + " edits # " + numEdits + " loaded in " + (monotonicNow()-startTime)/1000 + " seconds"); @@ -171,8 +173,13 @@ public class FSEditLogLoader { long loadEditRecords(EditLogInputStream in, boolean closeOnExit, long expectedStartingTxId, StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { - FSDirectory fsDir = fsNamesys.dir; + return loadEditRecords(in, closeOnExit, expectedStartingTxId, + Long.MAX_VALUE, startOpt, recovery); + } + long loadEditRecords(EditLogInputStream in, boolean closeOnExit, + long expectedStartingTxId, long maxTxnsToRead, StartupOption startOpt, + MetaRecoveryContext recovery) throws IOException { EnumMap> opCounts = new EnumMap>(FSEditLogOpCodes.class); @@ -181,6 +188,7 @@ public class FSEditLogLoader { } fsNamesys.writeLock(); + FSDirectory fsDir = fsNamesys.dir; fsDir.writeLock(); long recentOpcodeOffsets[] = new long[4]; @@ -285,6 +293,9 @@ public class FSEditLogLoader { } numEdits++; totalEdits++; + if(numEdits >= maxTxnsToRead) { + break; + } } catch (RollingUpgradeOp.RollbackException e) { LOG.info("Stopped at OP_START_ROLLING_UPGRADE for rollback."); break; @@ -308,7 +319,11 @@ public class FSEditLogLoader { if (FSImage.LOG.isDebugEnabled()) { dumpOpCounts(opCounts); + FSImage.LOG.debug("maxTxnsToRead = " + maxTxnsToRead + + " actual edits read = " + numEdits); } + assert numEdits <= maxTxnsToRead || numEdits == 1 : + "should read at least one txn, but not more than the configured max"; } return numEdits; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index dd7df5ad6b4..5cfc0176f1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -742,7 +742,8 @@ public class FSImage implements Closeable { prog.endPhase(Phase.LOADING_FSIMAGE); if (!rollingRollback) { - long txnsAdvanced = loadEdits(editStreams, target, startOpt, recovery); + long txnsAdvanced = loadEdits(editStreams, target, Long.MAX_VALUE, + startOpt, recovery); needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(), txnsAdvanced); } else { @@ -866,11 +867,12 @@ public class FSImage implements Closeable { */ public long loadEdits(Iterable editStreams, FSNamesystem target) throws IOException { - return loadEdits(editStreams, target, null, null); + return loadEdits(editStreams, target, Long.MAX_VALUE, null, null); } - private long loadEdits(Iterable editStreams, - FSNamesystem target, StartupOption startOpt, MetaRecoveryContext recovery) + public long loadEdits(Iterable editStreams, + FSNamesystem target, long maxTxnsToRead, + StartupOption startOpt, MetaRecoveryContext recovery) throws IOException { LOG.debug("About to load edits:\n " + Joiner.on("\n ").join(editStreams)); StartupProgress prog = NameNode.getStartupProgress(); @@ -885,14 +887,16 @@ public class FSImage implements Closeable { LOG.info("Reading " + editIn + " expecting start txid #" + (lastAppliedTxId + 1)); try { - loader.loadFSEdits(editIn, lastAppliedTxId + 1, startOpt, recovery); + loader.loadFSEdits(editIn, lastAppliedTxId + 1, maxTxnsToRead, + startOpt, recovery); } finally { // Update lastAppliedTxId even in case of error, since some ops may // have been successfully applied before the error. lastAppliedTxId = loader.getLastAppliedTxId(); } // If we are in recovery mode, we may have skipped over some txids. - if (editIn.getLastTxId() != HdfsServerConstants.INVALID_TXID) { + if (editIn.getLastTxId() != HdfsServerConstants.INVALID_TXID + && recovery != null) { lastAppliedTxId = editIn.getLastTxId(); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 752c830f958..a8c1926051a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1952,6 +1952,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (!isInSafeMode() && res.updateAccessTime()) { String src = srcArg; + checkOperation(OperationCategory.WRITE); writeLock(); final long now = now(); try { @@ -2034,6 +2035,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, FileStatus stat = null; boolean success = false; final FSPermissionChecker pc = getPermissionChecker(); + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -2137,6 +2139,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, throw new UnsupportedOperationException("Symlinks not supported"); } FileStatus auditStat = null; + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -2400,11 +2403,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, iip = FSDirWriteFileOp.resolvePathForStartFile( dir, pc, src, flag, createParent); - if (shouldReplicate || - (org.apache.commons.lang.StringUtils.isEmpty(ecPolicyName) && - !FSDirErasureCodingOp.hasErasureCodingPolicy(this, iip))) { - blockManager.verifyReplication(src, replication, clientMachine); - } if (blockSize < minBlockSize) { throw new IOException("Specified block size is less than configured" + @@ -2412,8 +2410,24 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, + "): " + blockSize + " < " + minBlockSize); } + if (shouldReplicate) { + blockManager.verifyReplication(src, replication, clientMachine); + } else { + final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp + .getErasureCodingPolicy(this, ecPolicyName, iip); + if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) { + if (blockSize < ecPolicy.getCellSize()) { + throw new IOException("Specified block size (" + blockSize + + ") is less than the cell size (" + ecPolicy.getCellSize() + +") of the erasure coding policy (" + ecPolicy + ")."); + } + } else { + blockManager.verifyReplication(src, replication, clientMachine); + } + } + FileEncryptionInfo feInfo = null; - if (provider != null) { + if (!iip.isRaw() && provider != null) { EncryptionKeyInfo ezInfo = FSDirEncryptionZoneOp.getEncryptionKeyInfo( this, iip, supportedVersions); // if the path has an encryption zone, the lock was released while @@ -5306,6 +5320,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, String clientName, ExtendedBlock oldBlock, ExtendedBlock newBlock, DatanodeID[] newNodes, String[] newStorageIDs, boolean logRetryCache) throws IOException { + checkOperation(OperationCategory.WRITE); LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + ", newGS=" + newBlock.getGenerationStamp() + ", newLength=" + newBlock.getNumBytes() @@ -6605,6 +6620,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, String rootPath = null; BlocksMapUpdateInfo blocksToBeDeleted = null; final FSPermissionChecker pc = getPermissionChecker(); + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -6645,6 +6661,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkOperation(OperationCategory.READ); readLock(); try { + checkOperation(OperationCategory.READ); if (!isRollingUpgrade()) { return null; } @@ -6897,6 +6914,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (!flags.contains(CacheFlag.FORCE)) { cacheManager.waitForRescanIfNeeded(); } + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -6928,6 +6946,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (!flags.contains(CacheFlag.FORCE)) { cacheManager.waitForRescanIfNeeded(); } + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -6953,6 +6972,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final String operationName = "removeCacheDirective"; boolean success = false; String idStr = "{id: " + Long.toString(id) + "}"; + checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); @@ -6997,6 +7017,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, void addCachePool(CachePoolInfo req, boolean logRetryCache) throws IOException { final String operationName = "addCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; String poolInfoStr = null; @@ -7021,6 +7042,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, void modifyCachePool(CachePoolInfo req, boolean logRetryCache) throws IOException { final String operationName = "modifyCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; String poolNameStr = "{poolName: " + @@ -7047,6 +7069,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, void removeCachePool(String cachePoolName, boolean logRetryCache) throws IOException { final String operationName = "removeCachePool"; + checkOperation(OperationCategory.WRITE); writeLock(); boolean success = false; String poolNameStr = "{poolName: " + cachePoolName + "}"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java index 0b284b9a7d7..354b4e364aa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java @@ -44,7 +44,7 @@ import org.apache.hadoop.security.UserGroupInformation; * The state of this class need not be synchronized as it has data structures that * are read-only. * - * Some of the helper methods are gaurded by {@link FSNamesystem#readLock()}. + * Some of the helper methods are guarded by {@link FSNamesystem#readLock()}. */ public class FSPermissionChecker implements AccessControlEnforcer { static final Log LOG = LogFactory.getLog(UserGroupInformation.class); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java index f57cb4bd939..2003f94f5f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java @@ -62,6 +62,7 @@ import static org.apache.hadoop.util.ExitUtil.terminate; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import org.apache.hadoop.util.Time; /** @@ -73,7 +74,19 @@ import com.google.common.base.Preconditions; @InterfaceStability.Evolving public class EditLogTailer { public static final Log LOG = LogFactory.getLog(EditLogTailer.class); - + + /** + * StandbyNode will hold namesystem lock to apply at most this many journal + * transactions. + * It will then release the lock and re-acquire it to load more transactions. + * By default the write lock is held for the entire journal segment. + * Fine-grained locking allows read requests to get through. + */ + public static final String DFS_HA_TAILEDITS_MAX_TXNS_PER_LOCK_KEY = + "dfs.ha.tail-edits.max-txns-per-lock"; + public static final long DFS_HA_TAILEDITS_MAX_TXNS_PER_LOCK_DEFAULT = + Long.MAX_VALUE; + private final EditLogTailerThread tailerThread; private final Configuration conf; @@ -138,6 +151,12 @@ public class EditLogTailer { */ private final boolean inProgressOk; + /** + * Release the namesystem lock after loading this many transactions. + * Then re-acquire the lock to load more edits. + */ + private final long maxTxnsPerLock; + public EditLogTailer(FSNamesystem namesystem, Configuration conf) { this.tailerThread = new EditLogTailerThread(); this.conf = conf; @@ -198,6 +217,10 @@ public class EditLogTailer { DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT); + this.maxTxnsPerLock = conf.getLong( + DFS_HA_TAILEDITS_MAX_TXNS_PER_LOCK_KEY, + DFS_HA_TAILEDITS_MAX_TXNS_PER_LOCK_DEFAULT); + nnCount = nns.size(); // setup the iterator to endlessly loop the nns this.nnLookup = Iterators.cycle(nns); @@ -270,6 +293,7 @@ public class EditLogTailer { LOG.debug("lastTxnId: " + lastTxnId); } Collection streams; + long startTime = Time.monotonicNow(); try { streams = editLog.selectInputStreams(lastTxnId + 1, 0, null, inProgressOk, true); @@ -280,6 +304,9 @@ public class EditLogTailer { LOG.warn("Edits tailer failed to find any streams. Will try again " + "later.", ioe); return; + } finally { + NameNode.getNameNodeMetrics().addEditLogFetchTime( + Time.monotonicNow() - startTime); } if (LOG.isDebugEnabled()) { LOG.debug("edit streams to load from: " + streams.size()); @@ -290,7 +317,8 @@ public class EditLogTailer { // disk are ignored. long editsLoaded = 0; try { - editsLoaded = image.loadEdits(streams, namesystem); + editsLoaded = image.loadEdits( + streams, namesystem, maxTxnsPerLock, null, null); } catch (EditLogInputException elie) { editsLoaded = elie.getNumEditsLoaded(); throw elie; @@ -299,6 +327,7 @@ public class EditLogTailer { LOG.debug(String.format("Loaded %d edits starting from txid %d ", editsLoaded, lastTxnId)); } + NameNode.getNameNodeMetrics().addNumEditLogLoaded(editsLoaded); } if (editsLoaded > 0) { @@ -423,10 +452,15 @@ public class EditLogTailer { // name system lock will be acquired to further block even the block // state updates. namesystem.cpLockInterruptibly(); + long startTime = Time.monotonicNow(); try { + NameNode.getNameNodeMetrics().addEditLogTailInterval( + startTime - lastLoadTimeMs); doTailEdits(); } finally { namesystem.cpUnlock(); + NameNode.getNameNodeMetrics().addEditLogTailTime( + Time.monotonicNow() - startTime); } //Update NameDirSize Metric namesystem.getFSImage().getStorage().updateNameDirSize(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java index 94c5e9e92b7..5d43ebf95fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java @@ -32,6 +32,7 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableQuantiles; import org.apache.hadoop.metrics2.lib.MutableRate; +import org.apache.hadoop.metrics2.lib.MutableStat; import org.apache.hadoop.metrics2.source.JvmMetrics; /** @@ -134,6 +135,18 @@ public class NameNodeMetrics { @Metric("Time loading FS Image at startup in msec") MutableGaugeInt fsImageLoadTime; + @Metric("Time tailing edit logs in msec") + MutableRate editLogTailTime; + private final MutableQuantiles[] editLogTailTimeQuantiles; + @Metric MutableRate editLogFetchTime; + private final MutableQuantiles[] editLogFetchTimeQuantiles; + @Metric(value = "Number of edits loaded", valueName = "Count") + MutableStat numEditLogLoaded; + private final MutableQuantiles[] numEditLogLoadedQuantiles; + @Metric("Time between edit log tailing in msec") + MutableRate editLogTailInterval; + private final MutableQuantiles[] editLogTailIntervalQuantiles; + @Metric("GetImageServlet getEdit") MutableRate getEdit; @Metric("GetImageServlet getImage") @@ -156,7 +169,11 @@ public class NameNodeMetrics { generateEDEKTimeQuantiles = new MutableQuantiles[len]; warmUpEDEKTimeQuantiles = new MutableQuantiles[len]; resourceCheckTimeQuantiles = new MutableQuantiles[len]; - + editLogTailTimeQuantiles = new MutableQuantiles[len]; + editLogFetchTimeQuantiles = new MutableQuantiles[len]; + numEditLogLoadedQuantiles = new MutableQuantiles[len]; + editLogTailIntervalQuantiles = new MutableQuantiles[len]; + for (int i = 0; i < len; i++) { int interval = intervals[i]; syncsQuantiles[i] = registry.newQuantiles( @@ -181,6 +198,18 @@ public class NameNodeMetrics { resourceCheckTimeQuantiles[i] = registry.newQuantiles( "resourceCheckTime" + interval + "s", "resource check time", "ops", "latency", interval); + editLogTailTimeQuantiles[i] = registry.newQuantiles( + "editLogTailTime" + interval + "s", + "Edit log tailing time", "ops", "latency", interval); + editLogFetchTimeQuantiles[i] = registry.newQuantiles( + "editLogFetchTime" + interval + "s", + "Edit log fetch time", "ops", "latency", interval); + numEditLogLoadedQuantiles[i] = registry.newQuantiles( + "numEditLogLoaded" + interval + "s", + "Number of edits loaded", "ops", "count", interval); + editLogTailIntervalQuantiles[i] = registry.newQuantiles( + "editLogTailInterval" + interval + "s", + "Edit log tailing interval", "ops", "latency", interval); } } @@ -389,4 +418,32 @@ public class NameNodeMetrics { q.add(latency); } } + + public void addEditLogTailTime(long elapsed) { + editLogTailTime.add(elapsed); + for (MutableQuantiles q : editLogTailTimeQuantiles) { + q.add(elapsed); + } + } + + public void addEditLogFetchTime(long elapsed) { + editLogFetchTime.add(elapsed); + for (MutableQuantiles q : editLogFetchTimeQuantiles) { + q.add(elapsed); + } + } + + public void addNumEditLogLoaded(long loaded) { + numEditLogLoaded.add(loaded); + for (MutableQuantiles q : numEditLogLoadedQuantiles) { + q.add(loaded); + } + } + + public void addEditLogTailInterval(long elapsed) { + editLogTailInterval.add(elapsed); + for (MutableQuantiles q : editLogTailIntervalQuantiles) { + q.add(elapsed); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index f7935572182..99a8e3e7886 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -1333,6 +1333,8 @@ public class DFSAdmin extends FsShell { System.out.println(evictWriters); } else if ("getDatanodeInfo".equalsIgnoreCase(cmd)) { System.out.println(getDatanodeInfo); + } else if ("triggerBlockReport".equalsIgnoreCase(cmd)) { + System.out.println(triggerBlockReport); } else if ("listOpenFiles".equalsIgnoreCase(cmd)) { System.out.println(listOpenFiles); } else if ("help".equals(cmd)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java index 0f2ac81d8d5..e4afa994614 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java @@ -72,6 +72,7 @@ public class OfflineImageViewerPB { + " rather than a number of bytes. (false by default)\n" + " * Web: Run a viewer to expose read-only WebHDFS API.\n" + " -addr specifies the address to listen. (localhost:5978 by default)\n" + + " It does not support secure mode nor HTTPS.\n" + " * Delimited (experimental): Generate a text file with all of the elements common\n" + " to both inodes and inodes-under-construction, separated by a\n" + " delimiter. The default delimiter is \\t, though this may be\n" @@ -200,7 +201,7 @@ public class OfflineImageViewerPB { case "WEB": String addr = cmd.getOptionValue("addr", "localhost:5978"); try (WebImageViewer viewer = - new WebImageViewer(NetUtils.createSocketAddr(addr))) { + new WebImageViewer(NetUtils.createSocketAddr(addr), conf)) { viewer.start(inputFile); } break; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java index 087972f94cc..a50e828e4a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java @@ -34,6 +34,9 @@ import io.netty.handler.codec.string.StringEncoder; import io.netty.util.concurrent.GlobalEventExecutor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.UserGroupInformation; import java.io.Closeable; import java.io.IOException; @@ -53,8 +56,12 @@ public class WebImageViewer implements Closeable { private final EventLoopGroup bossGroup; private final EventLoopGroup workerGroup; private final ChannelGroup allChannels; + private final Configuration conf; public WebImageViewer(InetSocketAddress address) { + this(address, new Configuration()); + } + public WebImageViewer(InetSocketAddress address, Configuration conf) { this.address = address; this.bossGroup = new NioEventLoopGroup(); this.workerGroup = new NioEventLoopGroup(); @@ -62,15 +69,25 @@ public class WebImageViewer implements Closeable { this.bootstrap = new ServerBootstrap() .group(bossGroup, workerGroup) .channel(NioServerSocketChannel.class); + this.conf = conf; + UserGroupInformation.setConfiguration(conf); } /** * Start WebImageViewer and wait until the thread is interrupted. * @param fsimage the fsimage to load. * @throws IOException if failed to load the fsimage. + * @throws RuntimeException if security is enabled in configuration. */ public void start(String fsimage) throws IOException { try { + if (UserGroupInformation.isSecurityEnabled()) { + throw new RuntimeException( + "WebImageViewer does not support secure mode. To start in " + + "non-secure mode, pass -D" + + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION + + "=simple"); + } initServer(fsimage); channel.closeFuture().await(); } catch (InterruptedException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 7a437abcfa5..b55421c162e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -3634,6 +3634,18 @@ + + dfs.client.failover.random.order + false + + Determines if the failover proxies are picked in random order instead of the + configured order. The prefix can be used with an optional nameservice ID + (of form dfs.client.failover.random.order[.nameservice]) in case multiple + nameservices exist and random order should be enabled for specific + nameservices. + + + dfs.client.key.provider.cache.expiry 864000000 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html index ed8f8fc9b4a..1d66f5591f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html @@ -19,7 +19,7 @@ - + DataNode Information @@ -167,8 +167,8 @@ {/dn.VolumeInfo} - - + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index a9284252156..4495b99b1fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -19,7 +19,7 @@ - + Namenode information @@ -467,9 +467,9 @@ There are no reported volume failures. - - - + @@ -65,6 +65,6 @@ - + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java index de4a85ac46b..e6158bddd0c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/ksm/TestChunkStreams.java @@ -16,7 +16,7 @@ */ package org.apache.hadoop.ozone.ksm; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream; import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream; import org.apache.hadoop.hdds.scm.storage.ChunkInputStream; diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index f605da2ac92..cffef14e6e5 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -143,8 +143,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> src/main/webapps/router/robots.txt src/contrib/** src/site/resources/images/* - webapps/static/bootstrap-3.0.2/** - webapps/static/jquery-1.10.2.min.js + webapps/static/bootstrap-3.3.7/** + webapps/static/jquery-3.3.1.min.js webapps/static/jquery.dataTables.min.js webapps/static/nvd3-1.8.5.min.css.map webapps/static/nvd3-1.8.5.min.js @@ -155,6 +155,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> webapps/static/nvd3-1.8.5.min.js.map webapps/static/angular-1.6.4.min.js webapps/static/d3-3.5.17.min.js + static/OzoneOverview.svg + themes/ozonedoc/static/js/jquery.min.js + themes/ozonedoc/static/js/bootstrap.min.js + themes/ozonedoc/static/css/bootstrap.min.css + themes/ozonedoc/static/css/bootstrap.min.css.map + themes/ozonedoc/static/css/bootstrap-theme.min.css + themes/ozonedoc/static/css/bootstrap-theme.min.css.map + themes/ozonedoc/static/fonts/glyphicons-halflings-regular.svg + themes/ozonedoc/layouts/index.html + themes/ozonedoc/theme.toml diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index d933e6f2e70..ab52b86c23a 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -29,9 +29,9 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.commons.lang.time.DurationFormatUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hdds.client.OzoneQuota; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java index e3f6cc95d89..1b1153b18a1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkContainerStateMap.java @@ -64,9 +64,14 @@ public class BenchMarkContainerStateMap { .setPipeline(pipeline) // This is bytes allocated for blocks inside container, not the // container size - .setAllocatedBytes(0).setUsedBytes(0).setNumberOfKeys(0) - .setStateEnterTime(Time.monotonicNow()).setOwner("OZONE") - .setContainerID(x).build(); + .setAllocatedBytes(0) + .setUsedBytes(0) + .setNumberOfKeys(0) + .setStateEnterTime(Time.monotonicNow()) + .setOwner("OZONE") + .setContainerID(x) + .setDeleteTransactionId(0) + .build(); stateMap.addContainer(containerInfo); currentCount++; } catch (SCMException e) { @@ -80,9 +85,14 @@ public class BenchMarkContainerStateMap { .setPipeline(pipeline) // This is bytes allocated for blocks inside container, not the // container size - .setAllocatedBytes(0).setUsedBytes(0).setNumberOfKeys(0) - .setStateEnterTime(Time.monotonicNow()).setOwner("OZONE") - .setContainerID(y).build(); + .setAllocatedBytes(0) + .setUsedBytes(0) + .setNumberOfKeys(0) + .setStateEnterTime(Time.monotonicNow()) + .setOwner("OZONE") + .setContainerID(y) + .setDeleteTransactionId(0) + .build(); stateMap.addContainer(containerInfo); currentCount++; } catch (SCMException e) { @@ -95,9 +105,14 @@ public class BenchMarkContainerStateMap { .setPipeline(pipeline) // This is bytes allocated for blocks inside container, not the // container size - .setAllocatedBytes(0).setUsedBytes(0).setNumberOfKeys(0) - .setStateEnterTime(Time.monotonicNow()).setOwner("OZONE") - .setContainerID(currentCount++).build(); + .setAllocatedBytes(0) + .setUsedBytes(0) + .setNumberOfKeys(0) + .setStateEnterTime(Time.monotonicNow()) + .setOwner("OZONE") + .setContainerID(currentCount++) + .setDeleteTransactionId(0) + .build(); stateMap.addContainer(containerInfo); } catch (SCMException e) { e.printStackTrace(); @@ -155,9 +170,14 @@ public class BenchMarkContainerStateMap { .setPipeline(pipeline) // This is bytes allocated for blocks inside container, not the // container size - .setAllocatedBytes(0).setUsedBytes(0).setNumberOfKeys(0) - .setStateEnterTime(Time.monotonicNow()).setOwner("OZONE") - .setContainerID(cid).build(); + .setAllocatedBytes(0) + .setUsedBytes(0) + .setNumberOfKeys(0) + .setStateEnterTime(Time.monotonicNow()) + .setOwner("OZONE") + .setContainerID(cid) + .setDeleteTransactionId(0) + .build(); state.stateMap.addContainer(containerInfo); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java index a5d268d5089..13b04c31ee5 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.genesis; import org.apache.ratis.shaded.com.google.protobuf.ByteString; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.client.BlockID; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java index fc3dbcb2305..05e7920dbc4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreReads.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.genesis; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.utils.MetadataStore; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Param; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java index f496a7d514f..4321287e185 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkMetadataStoreWrites.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.genesis; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.utils.MetadataStore; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Param; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java index c4c6f9eed6f..5f4e035cd3e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkRocksDbStore.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.genesis; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.utils.MetadataStore; import org.apache.hadoop.utils.RocksDBStore; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index 7f864ae6bf4..f4999254d50 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.genesis; -import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 5f83da3ec1f..5e21b4a5971 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -145,7 +145,7 @@ false - 2.9.1 + 3.0.3 -unstable diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 59a9bd2b891..8cb5bfc48b4 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -69,7 +69,7 @@ 1.9.13 - 2.9.4 + 2.9.5 1.7.25 @@ -450,6 +450,13 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-yarn-services-core + ${hadoop.version} + test-jar + + org.apache.hadoop hadoop-mapreduce-client-jobclient @@ -1326,7 +1333,7 @@ com.microsoft.azure azure-storage - 5.4.0 + 7.0.0 diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java index 754f3431426..e4ad70cedb2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemHelper.java @@ -87,8 +87,10 @@ final class NativeAzureFileSystemHelper { if (errorCode != null && (errorCode.equals(StorageErrorCodeStrings.BLOB_NOT_FOUND) || errorCode.equals(StorageErrorCodeStrings.RESOURCE_NOT_FOUND) + || errorCode.equals(StorageErrorCodeStrings.CONTAINER_NOT_FOUND) || errorCode.equals(StorageErrorCode.BLOB_NOT_FOUND.toString()) - || errorCode.equals(StorageErrorCode.RESOURCE_NOT_FOUND.toString()))) { + || errorCode.equals(StorageErrorCode.RESOURCE_NOT_FOUND.toString()) + || errorCode.equals(StorageErrorCode.CONTAINER_NOT_FOUND.toString()))) { return true; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java index cc3baf501d7..456e4b18bda 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestContainerChecks.java @@ -75,7 +75,7 @@ public class ITestContainerChecks extends AbstractWasbTestWithTimeout { assertTrue("Should've thrown.", false); } catch (FileNotFoundException ex) { assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); + ex.getMessage().contains("is not found")); } assertFalse(container.exists()); @@ -115,7 +115,7 @@ public class ITestContainerChecks extends AbstractWasbTestWithTimeout { assertTrue("Should've thrown.", false); } catch (FileNotFoundException ex) { assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); + ex.getMessage().contains("is not found")); } assertFalse(container.exists()); @@ -143,7 +143,7 @@ public class ITestContainerChecks extends AbstractWasbTestWithTimeout { assertTrue("Should've thrown.", false); } catch (FileNotFoundException ex) { assertTrue("Unexpected exception: " + ex, - ex.getMessage().contains("does not exist.")); + ex.getMessage().contains("is not found")); } assertFalse(container.exists()); @@ -165,6 +165,9 @@ public class ITestContainerChecks extends AbstractWasbTestWithTimeout { assertFalse(fs.rename(foo, bar)); assertFalse(container.exists()); + // Create a container outside of the WASB FileSystem + container.create(); + // But a write should. assertTrue(fs.createNewFile(foo)); assertTrue(container.exists()); diff --git a/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js b/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js index 38f046c739e..9b5206bcc60 100644 --- a/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js +++ b/hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js @@ -1,247 +1,173 @@ /*! - * jQuery JavaScript Library v1.10.2 - * http://jquery.com/ + * jQuery JavaScript Library v3.3.1 + * https://jquery.com/ * * Includes Sizzle.js - * http://sizzlejs.com/ + * https://sizzlejs.com/ * - * Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors + * Copyright JS Foundation and other contributors * Released under the MIT license - * http://jquery.org/license + * https://jquery.org/license * - * Date: 2013-07-03T13:48Z + * Date: 2018-01-20T17:24Z */ -(function( window, undefined ) { +( function( global, factory ) { + + "use strict"; + + if ( typeof module === "object" && typeof module.exports === "object" ) { + + // For CommonJS and CommonJS-like environments where a proper `window` + // is present, execute the factory and get jQuery. + // For environments that do not have a `window` with a `document` + // (such as Node.js), expose a factory as module.exports. + // This accentuates the need for the creation of a real `window`. + // e.g. var jQuery = require("jquery")(window); + // See ticket #14549 for more info. + module.exports = global.document ? + factory( global, true ) : + function( w ) { + if ( !w.document ) { + throw new Error( "jQuery requires a window with a document" ); + } + return factory( w ); + }; + } else { + factory( global ); + } + +// Pass this if window is not defined yet +} )( typeof window !== "undefined" ? window : this, function( window, noGlobal ) { + +// Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1 +// throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode +// arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common +// enough that all such attempts are guarded in a try block. +"use strict"; + +var arr = []; + +var document = window.document; + +var getProto = Object.getPrototypeOf; + +var slice = arr.slice; + +var concat = arr.concat; + +var push = arr.push; + +var indexOf = arr.indexOf; + +var class2type = {}; + +var toString = class2type.toString; + +var hasOwn = class2type.hasOwnProperty; + +var fnToString = hasOwn.toString; + +var ObjectFunctionString = fnToString.call( Object ); + +var support = {}; + +var isFunction = function isFunction( obj ) { + + // Support: Chrome <=57, Firefox <=52 + // In some browsers, typeof returns "function" for HTML elements + // (i.e., `typeof document.createElement( "object" ) === "function"`). + // We don't want to classify *any* DOM node as a function. + return typeof obj === "function" && typeof obj.nodeType !== "number"; + }; + + +var isWindow = function isWindow( obj ) { + return obj != null && obj === obj.window; + }; + + + + + var preservedScriptAttributes = { + type: true, + src: true, + noModule: true + }; + + function DOMEval( code, doc, node ) { + doc = doc || document; + + var i, + script = doc.createElement( "script" ); + + script.text = code; + if ( node ) { + for ( i in preservedScriptAttributes ) { + if ( node[ i ] ) { + script[ i ] = node[ i ]; + } + } + } + doc.head.appendChild( script ).parentNode.removeChild( script ); + } + + +function toType( obj ) { + if ( obj == null ) { + return obj + ""; + } + + // Support: Android <=2.3 only (functionish RegExp) + return typeof obj === "object" || typeof obj === "function" ? + class2type[ toString.call( obj ) ] || "object" : + typeof obj; +} +/* global Symbol */ +// Defining this global in .eslintrc.json would create a danger of using the global +// unguarded in another place, it seems safer to define global only for this module + + -// Can't do this because several apps including ASP.NET trace -// the stack via arguments.caller.callee and Firefox dies if -// you try to trace through "use strict" call chains. (#13335) -// Support: Firefox 18+ -//"use strict"; var - // The deferred used on DOM ready - readyList, - - // A central reference to the root jQuery(document) - rootjQuery, - - // Support: IE<10 - // For `typeof xmlNode.method` instead of `xmlNode.method !== undefined` - core_strundefined = typeof undefined, - - // Use the correct document accordingly with window argument (sandbox) - location = window.location, - document = window.document, - docElem = document.documentElement, - - // Map over jQuery in case of overwrite - _jQuery = window.jQuery, - - // Map over the $ in case of overwrite - _$ = window.$, - - // [[Class]] -> type pairs - class2type = {}, - - // List of deleted data cache ids, so we can reuse them - core_deletedIds = [], - - core_version = "1.10.2", - - // Save a reference to some core methods - core_concat = core_deletedIds.concat, - core_push = core_deletedIds.push, - core_slice = core_deletedIds.slice, - core_indexOf = core_deletedIds.indexOf, - core_toString = class2type.toString, - core_hasOwn = class2type.hasOwnProperty, - core_trim = core_version.trim, + version = "3.3.1", // Define a local copy of jQuery jQuery = function( selector, context ) { + // The jQuery object is actually just the init constructor 'enhanced' - return new jQuery.fn.init( selector, context, rootjQuery ); + // Need init if jQuery is called (just allow error to be thrown if not included) + return new jQuery.fn.init( selector, context ); }, - // Used for matching numbers - core_pnum = /[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source, - - // Used for splitting on whitespace - core_rnotwhite = /\S+/g, - - // Make sure we trim BOM and NBSP (here's looking at you, Safari 5.0 and IE) - rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, - - // A simple way to check for HTML strings - // Prioritize #id over to avoid XSS via location.hash (#9521) - // Strict HTML recognition (#11290: must start with <) - rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/, - - // Match a standalone tag - rsingleTag = /^<(\w+)\s*\/?>(?:<\/\1>|)$/, - - // JSON RegExp - rvalidchars = /^[\],:{}\s]*$/, - rvalidbraces = /(?:^|:|,)(?:\s*\[)+/g, - rvalidescape = /\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g, - rvalidtokens = /"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g, - - // Matches dashed string for camelizing - rmsPrefix = /^-ms-/, - rdashAlpha = /-([\da-z])/gi, - - // Used by jQuery.camelCase as callback to replace() - fcamelCase = function( all, letter ) { - return letter.toUpperCase(); - }, - - // The ready event handler - completed = function( event ) { - - // readyState === "complete" is good enough for us to call the dom ready in oldIE - if ( document.addEventListener || event.type === "load" || document.readyState === "complete" ) { - detach(); - jQuery.ready(); - } - }, - // Clean-up method for dom ready events - detach = function() { - if ( document.addEventListener ) { - document.removeEventListener( "DOMContentLoaded", completed, false ); - window.removeEventListener( "load", completed, false ); - - } else { - document.detachEvent( "onreadystatechange", completed ); - window.detachEvent( "onload", completed ); - } - }; + // Support: Android <=4.0 only + // Make sure we trim BOM and NBSP + rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g; jQuery.fn = jQuery.prototype = { + // The current version of jQuery being used - jquery: core_version, + jquery: version, constructor: jQuery, - init: function( selector, context, rootjQuery ) { - var match, elem; - - // HANDLE: $(""), $(null), $(undefined), $(false) - if ( !selector ) { - return this; - } - - // Handle HTML strings - if ( typeof selector === "string" ) { - if ( selector.charAt(0) === "<" && selector.charAt( selector.length - 1 ) === ">" && selector.length >= 3 ) { - // Assume that strings that start and end with <> are HTML and skip the regex check - match = [ null, selector, null ]; - - } else { - match = rquickExpr.exec( selector ); - } - - // Match html or make sure no context is specified for #id - if ( match && (match[1] || !context) ) { - - // HANDLE: $(html) -> $(array) - if ( match[1] ) { - context = context instanceof jQuery ? context[0] : context; - - // scripts is true for back-compat - jQuery.merge( this, jQuery.parseHTML( - match[1], - context && context.nodeType ? context.ownerDocument || context : document, - true - ) ); - - // HANDLE: $(html, props) - if ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) { - for ( match in context ) { - // Properties of context are called as methods if possible - if ( jQuery.isFunction( this[ match ] ) ) { - this[ match ]( context[ match ] ); - - // ...and otherwise set as attributes - } else { - this.attr( match, context[ match ] ); - } - } - } - - return this; - - // HANDLE: $(#id) - } else { - elem = document.getElementById( match[2] ); - - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - if ( elem && elem.parentNode ) { - // Handle the case where IE and Opera return items - // by name instead of ID - if ( elem.id !== match[2] ) { - return rootjQuery.find( selector ); - } - - // Otherwise, we inject the element directly into the jQuery object - this.length = 1; - this[0] = elem; - } - - this.context = document; - this.selector = selector; - return this; - } - - // HANDLE: $(expr, $(...)) - } else if ( !context || context.jquery ) { - return ( context || rootjQuery ).find( selector ); - - // HANDLE: $(expr, context) - // (which is just equivalent to: $(context).find(expr) - } else { - return this.constructor( context ).find( selector ); - } - - // HANDLE: $(DOMElement) - } else if ( selector.nodeType ) { - this.context = this[0] = selector; - this.length = 1; - return this; - - // HANDLE: $(function) - // Shortcut for document ready - } else if ( jQuery.isFunction( selector ) ) { - return rootjQuery.ready( selector ); - } - - if ( selector.selector !== undefined ) { - this.selector = selector.selector; - this.context = selector.context; - } - - return jQuery.makeArray( selector, this ); - }, - - // Start with an empty selector - selector: "", // The default length of a jQuery object is 0 length: 0, toArray: function() { - return core_slice.call( this ); + return slice.call( this ); }, // Get the Nth element in the matched element set OR // Get the whole matched element set as a clean array get: function( num ) { - return num == null ? - // Return a 'clean' array - this.toArray() : + // Return all the elements in a clean array + if ( num == null ) { + return slice.call( this ); + } - // Return just the object - ( num < 0 ? this[ this.length + num ] : this[ num ] ); + // Return just the one element from the set + return num < 0 ? this[ num + this.length ] : this[ num ]; }, // Take an array of elements and push it onto the stack @@ -253,28 +179,24 @@ jQuery.fn = jQuery.prototype = { // Add the old object onto the stack (as a reference) ret.prevObject = this; - ret.context = this.context; // Return the newly-formed element set return ret; }, // Execute a callback for every element in the matched set. - // (You can seed the arguments with an array of args, but this is - // only used internally.) - each: function( callback, args ) { - return jQuery.each( this, callback, args ); + each: function( callback ) { + return jQuery.each( this, callback ); }, - ready: function( fn ) { - // Add the callback - jQuery.ready.promise().done( fn ); - - return this; + map: function( callback ) { + return this.pushStack( jQuery.map( this, function( elem, i ) { + return callback.call( elem, i, elem ); + } ) ); }, slice: function() { - return this.pushStack( core_slice.apply( this, arguments ) ); + return this.pushStack( slice.apply( this, arguments ) ); }, first: function() { @@ -288,32 +210,23 @@ jQuery.fn = jQuery.prototype = { eq: function( i ) { var len = this.length, j = +i + ( i < 0 ? len : 0 ); - return this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] ); - }, - - map: function( callback ) { - return this.pushStack( jQuery.map(this, function( elem, i ) { - return callback.call( elem, i, elem ); - })); + return this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] ); }, end: function() { - return this.prevObject || this.constructor(null); + return this.prevObject || this.constructor(); }, // For internal use only. // Behaves like an Array's method, not like a jQuery method. - push: core_push, - sort: [].sort, - splice: [].splice + push: push, + sort: arr.sort, + splice: arr.splice }; -// Give the init function the jQuery prototype for later instantiation -jQuery.fn.init.prototype = jQuery.fn; - jQuery.extend = jQuery.fn.extend = function() { - var src, copyIsArray, copy, name, options, clone, - target = arguments[0] || {}, + var options, name, src, copy, copyIsArray, clone, + target = arguments[ 0 ] || {}, i = 1, length = arguments.length, deep = false; @@ -321,25 +234,28 @@ jQuery.extend = jQuery.fn.extend = function() { // Handle a deep copy situation if ( typeof target === "boolean" ) { deep = target; - target = arguments[1] || {}; - // skip the boolean and the target - i = 2; + + // Skip the boolean and the target + target = arguments[ i ] || {}; + i++; } // Handle case when target is a string or something (possible in deep copy) - if ( typeof target !== "object" && !jQuery.isFunction(target) ) { + if ( typeof target !== "object" && !isFunction( target ) ) { target = {}; } - // extend jQuery itself if only one argument is passed - if ( length === i ) { + // Extend jQuery itself if only one argument is passed + if ( i === length ) { target = this; - --i; + i--; } for ( ; i < length; i++ ) { + // Only deal with non-null/undefined values - if ( (options = arguments[ i ]) != null ) { + if ( ( options = arguments[ i ] ) != null ) { + // Extend the base object for ( name in options ) { src = target[ name ]; @@ -351,13 +267,15 @@ jQuery.extend = jQuery.fn.extend = function() { } // Recurse if we're merging plain objects or arrays - if ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) { + if ( deep && copy && ( jQuery.isPlainObject( copy ) || + ( copyIsArray = Array.isArray( copy ) ) ) ) { + if ( copyIsArray ) { copyIsArray = false; - clone = src && jQuery.isArray(src) ? src : []; + clone = src && Array.isArray( src ) ? src : []; } else { - clone = src && jQuery.isPlainObject(src) ? src : {}; + clone = src && jQuery.isPlainObject( src ) ? src : {}; } // Never move original objects, clone them @@ -375,298 +293,72 @@ jQuery.extend = jQuery.fn.extend = function() { return target; }; -jQuery.extend({ +jQuery.extend( { + // Unique for each copy of jQuery on the page - // Non-digits removed to match rinlinejQuery - expando: "jQuery" + ( core_version + Math.random() ).replace( /\D/g, "" ), + expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), - noConflict: function( deep ) { - if ( window.$ === jQuery ) { - window.$ = _$; - } + // Assume jQuery is ready without the ready module + isReady: true, - if ( deep && window.jQuery === jQuery ) { - window.jQuery = _jQuery; - } - - return jQuery; + error: function( msg ) { + throw new Error( msg ); }, - // Is the DOM ready to be used? Set to true once it occurs. - isReady: false, - - // A counter to track how many items to wait for before - // the ready event fires. See #6781 - readyWait: 1, - - // Hold (or release) the ready event - holdReady: function( hold ) { - if ( hold ) { - jQuery.readyWait++; - } else { - jQuery.ready( true ); - } - }, - - // Handle when the DOM is ready - ready: function( wait ) { - - // Abort if there are pending holds or we're already ready - if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { - return; - } - - // Make sure body exists, at least, in case IE gets a little overzealous (ticket #5443). - if ( !document.body ) { - return setTimeout( jQuery.ready ); - } - - // Remember that the DOM is ready - jQuery.isReady = true; - - // If a normal DOM Ready event fired, decrement, and wait if need be - if ( wait !== true && --jQuery.readyWait > 0 ) { - return; - } - - // If there are functions bound, to execute - readyList.resolveWith( document, [ jQuery ] ); - - // Trigger any bound ready events - if ( jQuery.fn.trigger ) { - jQuery( document ).trigger("ready").off("ready"); - } - }, - - // See test/unit/core.js for details concerning isFunction. - // Since version 1.3, DOM methods and functions like alert - // aren't supported. They return false on IE (#2968). - isFunction: function( obj ) { - return jQuery.type(obj) === "function"; - }, - - isArray: Array.isArray || function( obj ) { - return jQuery.type(obj) === "array"; - }, - - isWindow: function( obj ) { - /* jshint eqeqeq: false */ - return obj != null && obj == obj.window; - }, - - isNumeric: function( obj ) { - return !isNaN( parseFloat(obj) ) && isFinite( obj ); - }, - - type: function( obj ) { - if ( obj == null ) { - return String( obj ); - } - return typeof obj === "object" || typeof obj === "function" ? - class2type[ core_toString.call(obj) ] || "object" : - typeof obj; - }, + noop: function() {}, isPlainObject: function( obj ) { - var key; + var proto, Ctor; - // Must be an Object. - // Because of IE, we also have to check the presence of the constructor property. - // Make sure that DOM nodes and window objects don't pass through, as well - if ( !obj || jQuery.type(obj) !== "object" || obj.nodeType || jQuery.isWindow( obj ) ) { + // Detect obvious negatives + // Use toString instead of jQuery.type to catch host objects + if ( !obj || toString.call( obj ) !== "[object Object]" ) { return false; } - try { - // Not own constructor property must be Object - if ( obj.constructor && - !core_hasOwn.call(obj, "constructor") && - !core_hasOwn.call(obj.constructor.prototype, "isPrototypeOf") ) { - return false; - } - } catch ( e ) { - // IE8,9 Will throw exceptions on certain host objects #9897 - return false; + proto = getProto( obj ); + + // Objects with no prototype (e.g., `Object.create( null )`) are plain + if ( !proto ) { + return true; } - // Support: IE<9 - // Handle iteration over inherited properties before own properties. - if ( jQuery.support.ownLast ) { - for ( key in obj ) { - return core_hasOwn.call( obj, key ); - } - } - - // Own properties are enumerated firstly, so to speed up, - // if last one is own, then all properties are own. - for ( key in obj ) {} - - return key === undefined || core_hasOwn.call( obj, key ); + // Objects with prototype are plain iff they were constructed by a global Object function + Ctor = hasOwn.call( proto, "constructor" ) && proto.constructor; + return typeof Ctor === "function" && fnToString.call( Ctor ) === ObjectFunctionString; }, isEmptyObject: function( obj ) { + + /* eslint-disable no-unused-vars */ + // See https://github.com/eslint/eslint/issues/6125 var name; + for ( name in obj ) { return false; } return true; }, - error: function( msg ) { - throw new Error( msg ); - }, - - // data: string of html - // context (optional): If specified, the fragment will be created in this context, defaults to document - // keepScripts (optional): If true, will include scripts passed in the html string - parseHTML: function( data, context, keepScripts ) { - if ( !data || typeof data !== "string" ) { - return null; - } - if ( typeof context === "boolean" ) { - keepScripts = context; - context = false; - } - context = context || document; - - var parsed = rsingleTag.exec( data ), - scripts = !keepScripts && []; - - // Single tag - if ( parsed ) { - return [ context.createElement( parsed[1] ) ]; - } - - parsed = jQuery.buildFragment( [ data ], context, scripts ); - if ( scripts ) { - jQuery( scripts ).remove(); - } - return jQuery.merge( [], parsed.childNodes ); - }, - - parseJSON: function( data ) { - // Attempt to parse using the native JSON parser first - if ( window.JSON && window.JSON.parse ) { - return window.JSON.parse( data ); - } - - if ( data === null ) { - return data; - } - - if ( typeof data === "string" ) { - - // Make sure leading/trailing whitespace is removed (IE can't handle it) - data = jQuery.trim( data ); - - if ( data ) { - // Make sure the incoming data is actual JSON - // Logic borrowed from http://json.org/json2.js - if ( rvalidchars.test( data.replace( rvalidescape, "@" ) - .replace( rvalidtokens, "]" ) - .replace( rvalidbraces, "")) ) { - - return ( new Function( "return " + data ) )(); - } - } - } - - jQuery.error( "Invalid JSON: " + data ); - }, - - // Cross-browser xml parsing - parseXML: function( data ) { - var xml, tmp; - if ( !data || typeof data !== "string" ) { - return null; - } - try { - if ( window.DOMParser ) { // Standard - tmp = new DOMParser(); - xml = tmp.parseFromString( data , "text/xml" ); - } else { // IE - xml = new ActiveXObject( "Microsoft.XMLDOM" ); - xml.async = "false"; - xml.loadXML( data ); - } - } catch( e ) { - xml = undefined; - } - if ( !xml || !xml.documentElement || xml.getElementsByTagName( "parsererror" ).length ) { - jQuery.error( "Invalid XML: " + data ); - } - return xml; - }, - - noop: function() {}, - // Evaluates a script in a global context - // Workarounds based on findings by Jim Driscoll - // http://weblogs.java.net/blog/driscoll/archive/2009/09/08/eval-javascript-global-context - globalEval: function( data ) { - if ( data && jQuery.trim( data ) ) { - // We use execScript on Internet Explorer - // We use an anonymous function so that context is window - // rather than jQuery in Firefox - ( window.execScript || function( data ) { - window[ "eval" ].call( window, data ); - } )( data ); - } + globalEval: function( code ) { + DOMEval( code ); }, - // Convert dashed to camelCase; used by the css and data modules - // Microsoft forgot to hump their vendor prefix (#9572) - camelCase: function( string ) { - return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); - }, + each: function( obj, callback ) { + var length, i = 0; - nodeName: function( elem, name ) { - return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); - }, - - // args is for internal usage only - each: function( obj, callback, args ) { - var value, - i = 0, - length = obj.length, - isArray = isArraylike( obj ); - - if ( args ) { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.apply( obj[ i ], args ); - - if ( value === false ) { - break; - } + if ( isArrayLike( obj ) ) { + length = obj.length; + for ( ; i < length; i++ ) { + if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { + break; } } - - // A special, fast, case for the most common use of each } else { - if ( isArray ) { - for ( ; i < length; i++ ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } - } - } else { - for ( i in obj ) { - value = callback.call( obj[ i ], i, obj[ i ] ); - - if ( value === false ) { - break; - } + for ( i in obj ) { + if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { + break; } } } @@ -674,33 +366,25 @@ jQuery.extend({ return obj; }, - // Use native String.trim function wherever possible - trim: core_trim && !core_trim.call("\uFEFF\xA0") ? - function( text ) { - return text == null ? - "" : - core_trim.call( text ); - } : - - // Otherwise use our own trimming functionality - function( text ) { - return text == null ? - "" : - ( text + "" ).replace( rtrim, "" ); - }, + // Support: Android <=4.0 only + trim: function( text ) { + return text == null ? + "" : + ( text + "" ).replace( rtrim, "" ); + }, // results is for internal usage only makeArray: function( arr, results ) { var ret = results || []; if ( arr != null ) { - if ( isArraylike( Object(arr) ) ) { + if ( isArrayLike( Object( arr ) ) ) { jQuery.merge( ret, typeof arr === "string" ? [ arr ] : arr ); } else { - core_push.call( ret, arr ); + push.call( ret, arr ); } } @@ -708,40 +392,18 @@ jQuery.extend({ }, inArray: function( elem, arr, i ) { - var len; - - if ( arr ) { - if ( core_indexOf ) { - return core_indexOf.call( arr, elem, i ); - } - - len = arr.length; - i = i ? i < 0 ? Math.max( 0, len + i ) : i : 0; - - for ( ; i < len; i++ ) { - // Skip accessing in sparse arrays - if ( i in arr && arr[ i ] === elem ) { - return i; - } - } - } - - return -1; + return arr == null ? -1 : indexOf.call( arr, elem, i ); }, + // Support: Android <=4.0 only, PhantomJS 1 only + // push.apply(_, arraylike) throws on ancient WebKit merge: function( first, second ) { - var l = second.length, - i = first.length, - j = 0; + var len = +second.length, + j = 0, + i = first.length; - if ( typeof l === "number" ) { - for ( ; j < l; j++ ) { - first[ i++ ] = second[ j ]; - } - } else { - while ( second[j] !== undefined ) { - first[ i++ ] = second[ j++ ]; - } + for ( ; j < len; j++ ) { + first[ i++ ] = second[ j ]; } first.length = i; @@ -749,40 +411,39 @@ jQuery.extend({ return first; }, - grep: function( elems, callback, inv ) { - var retVal, - ret = [], + grep: function( elems, callback, invert ) { + var callbackInverse, + matches = [], i = 0, - length = elems.length; - inv = !!inv; + length = elems.length, + callbackExpect = !invert; // Go through the array, only saving the items // that pass the validator function for ( ; i < length; i++ ) { - retVal = !!callback( elems[ i ], i ); - if ( inv !== retVal ) { - ret.push( elems[ i ] ); + callbackInverse = !callback( elems[ i ], i ); + if ( callbackInverse !== callbackExpect ) { + matches.push( elems[ i ] ); } } - return ret; + return matches; }, // arg is for internal usage only map: function( elems, callback, arg ) { - var value, + var length, value, i = 0, - length = elems.length, - isArray = isArraylike( elems ), ret = []; - // Go through the array, translating each of the items to their - if ( isArray ) { + // Go through the array, translating each of the items to their new values + if ( isArrayLike( elems ) ) { + length = elems.length; for ( ; i < length; i++ ) { value = callback( elems[ i ], i, arg ); if ( value != null ) { - ret[ ret.length ] = value; + ret.push( value ); } } @@ -792,234 +453,73 @@ jQuery.extend({ value = callback( elems[ i ], i, arg ); if ( value != null ) { - ret[ ret.length ] = value; + ret.push( value ); } } } // Flatten any nested arrays - return core_concat.apply( [], ret ); + return concat.apply( [], ret ); }, // A global GUID counter for objects guid: 1, - // Bind a function to a context, optionally partially applying any - // arguments. - proxy: function( fn, context ) { - var args, proxy, tmp; + // jQuery.support is not used in Core but other projects attach their + // properties to it so it needs to exist. + support: support +} ); - if ( typeof context === "string" ) { - tmp = fn[ context ]; - context = fn; - fn = tmp; - } - - // Quick check to determine if target is callable, in the spec - // this throws a TypeError, but we will just return undefined. - if ( !jQuery.isFunction( fn ) ) { - return undefined; - } - - // Simulated bind - args = core_slice.call( arguments, 2 ); - proxy = function() { - return fn.apply( context || this, args.concat( core_slice.call( arguments ) ) ); - }; - - // Set the guid of unique handler to the same of original handler, so it can be removed - proxy.guid = fn.guid = fn.guid || jQuery.guid++; - - return proxy; - }, - - // Multifunctional method to get and set values of a collection - // The value/s can optionally be executed if it's a function - access: function( elems, fn, key, value, chainable, emptyGet, raw ) { - var i = 0, - length = elems.length, - bulk = key == null; - - // Sets many values - if ( jQuery.type( key ) === "object" ) { - chainable = true; - for ( i in key ) { - jQuery.access( elems, fn, i, key[i], true, emptyGet, raw ); - } - - // Sets one value - } else if ( value !== undefined ) { - chainable = true; - - if ( !jQuery.isFunction( value ) ) { - raw = true; - } - - if ( bulk ) { - // Bulk operations run against the entire set - if ( raw ) { - fn.call( elems, value ); - fn = null; - - // ...except when executing function values - } else { - bulk = fn; - fn = function( elem, key, value ) { - return bulk.call( jQuery( elem ), value ); - }; - } - } - - if ( fn ) { - for ( ; i < length; i++ ) { - fn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) ); - } - } - } - - return chainable ? - elems : - - // Gets - bulk ? - fn.call( elems ) : - length ? fn( elems[0], key ) : emptyGet; - }, - - now: function() { - return ( new Date() ).getTime(); - }, - - // A method for quickly swapping in/out CSS properties to get correct calculations. - // Note: this method belongs to the css module but it's needed here for the support module. - // If support gets modularized, this method should be moved back to the css module. - swap: function( elem, options, callback, args ) { - var ret, name, - old = {}; - - // Remember the old values, and insert the new ones - for ( name in options ) { - old[ name ] = elem.style[ name ]; - elem.style[ name ] = options[ name ]; - } - - ret = callback.apply( elem, args || [] ); - - // Revert the old values - for ( name in options ) { - elem.style[ name ] = old[ name ]; - } - - return ret; - } -}); - -jQuery.ready.promise = function( obj ) { - if ( !readyList ) { - - readyList = jQuery.Deferred(); - - // Catch cases where $(document).ready() is called after the browser event has already occurred. - // we once tried to use readyState "interactive" here, but it caused issues like the one - // discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15 - if ( document.readyState === "complete" ) { - // Handle it asynchronously to allow scripts the opportunity to delay ready - setTimeout( jQuery.ready ); - - // Standards-based browsers support DOMContentLoaded - } else if ( document.addEventListener ) { - // Use the handy event callback - document.addEventListener( "DOMContentLoaded", completed, false ); - - // A fallback to window.onload, that will always work - window.addEventListener( "load", completed, false ); - - // If IE event model is used - } else { - // Ensure firing before onload, maybe late but safe also for iframes - document.attachEvent( "onreadystatechange", completed ); - - // A fallback to window.onload, that will always work - window.attachEvent( "onload", completed ); - - // If IE and not a frame - // continually check to see if the document is ready - var top = false; - - try { - top = window.frameElement == null && document.documentElement; - } catch(e) {} - - if ( top && top.doScroll ) { - (function doScrollCheck() { - if ( !jQuery.isReady ) { - - try { - // Use the trick by Diego Perini - // http://javascript.nwbox.com/IEContentLoaded/ - top.doScroll("left"); - } catch(e) { - return setTimeout( doScrollCheck, 50 ); - } - - // detach all dom ready events - detach(); - - // and execute any waiting functions - jQuery.ready(); - } - })(); - } - } - } - return readyList.promise( obj ); -}; +if ( typeof Symbol === "function" ) { + jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; +} // Populate the class2type map -jQuery.each("Boolean Number String Function Array Date RegExp Object Error".split(" "), function(i, name) { +jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), +function( i, name ) { class2type[ "[object " + name + "]" ] = name.toLowerCase(); -}); +} ); -function isArraylike( obj ) { - var length = obj.length, - type = jQuery.type( obj ); +function isArrayLike( obj ) { - if ( jQuery.isWindow( obj ) ) { + // Support: real iOS 8.2 only (not reproducible in simulator) + // `in` check used to prevent JIT error (gh-2145) + // hasOwn isn't used here due to false negatives + // regarding Nodelist length in IE + var length = !!obj && "length" in obj && obj.length, + type = toType( obj ); + + if ( isFunction( obj ) || isWindow( obj ) ) { return false; } - if ( obj.nodeType === 1 && length ) { - return true; - } - - return type === "array" || type !== "function" && - ( length === 0 || - typeof length === "number" && length > 0 && ( length - 1 ) in obj ); + return type === "array" || length === 0 || + typeof length === "number" && length > 0 && ( length - 1 ) in obj; } - -// All jQuery objects should point back to these -rootjQuery = jQuery(document); +var Sizzle = /*! - * Sizzle CSS Selector Engine v1.10.2 - * http://sizzlejs.com/ + * Sizzle CSS Selector Engine v2.3.3 + * https://sizzlejs.com/ * - * Copyright 2013 jQuery Foundation, Inc. and other contributors + * Copyright jQuery Foundation and other contributors * Released under the MIT license * http://jquery.org/license * - * Date: 2013-07-03 + * Date: 2016-08-08 */ -(function( window, undefined ) { +(function( window ) { var i, support, - cachedruns, Expr, getText, isXML, + tokenize, compile, + select, outermostContext, sortInput, + hasDuplicate, // Local document vars setDocument, @@ -1032,26 +532,20 @@ var i, contains, // Instance-specific data - expando = "sizzle" + -(new Date()), + expando = "sizzle" + 1 * new Date(), preferredDoc = window.document, dirruns = 0, done = 0, classCache = createCache(), tokenCache = createCache(), compilerCache = createCache(), - hasDuplicate = false, sortOrder = function( a, b ) { if ( a === b ) { hasDuplicate = true; - return 0; } return 0; }, - // General-purpose constants - strundefined = typeof undefined, - MAX_NEGATIVE = 1 << 31, - // Instance methods hasOwn = ({}).hasOwnProperty, arr = [], @@ -1059,12 +553,13 @@ var i, push_native = arr.push, push = arr.push, slice = arr.slice, - // Use a stripped-down indexOf if we can't use a native one - indexOf = arr.indexOf || function( elem ) { + // Use a stripped-down indexOf as it's faster than native + // https://jsperf.com/thor-indexof-vs-for/5 + indexOf = function( list, elem ) { var i = 0, - len = this.length; + len = list.length; for ( ; i < len; i++ ) { - if ( this[i] === elem ) { + if ( list[i] === elem ) { return i; } } @@ -1075,44 +570,46 @@ var i, // Regular expressions - // Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace + // http://www.w3.org/TR/css3-selectors/#whitespace whitespace = "[\\x20\\t\\r\\n\\f]", - // http://www.w3.org/TR/css3-syntax/#characters - characterEncoding = "(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+", - // Loosely modeled on CSS identifier characters - // An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors - // Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier - identifier = characterEncoding.replace( "w", "w#" ), + // http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier + identifier = "(?:\\\\.|[\\w-]|[^\0-\\xa0])+", - // Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors - attributes = "\\[" + whitespace + "*(" + characterEncoding + ")" + whitespace + - "*(?:([*^$|!~]?=)" + whitespace + "*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|(" + identifier + ")|)|)" + whitespace + "*\\]", + // Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors + attributes = "\\[" + whitespace + "*(" + identifier + ")(?:" + whitespace + + // Operator (capture 2) + "*([*^$|!~]?=)" + whitespace + + // "Attribute values must be CSS identifiers [capture 5] or strings [capture 3 or capture 4]" + "*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|(" + identifier + "))|)" + whitespace + + "*\\]", - // Prefer arguments quoted, - // then not containing pseudos/brackets, - // then attribute selectors/non-parenthetical expressions, - // then anything else - // These preferences are here to reduce the number of selectors - // needing tokenize in the PSEUDO preFilter - pseudos = ":(" + characterEncoding + ")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|" + attributes.replace( 3, 8 ) + ")*)|.*)\\)|)", + pseudos = ":(" + identifier + ")(?:\\((" + + // To reduce the number of selectors needing tokenize in the preFilter, prefer arguments: + // 1. quoted (capture 3; capture 4 or capture 5) + "('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|" + + // 2. simple (capture 6) + "((?:\\\\.|[^\\\\()[\\]]|" + attributes + ")*)|" + + // 3. anything else (capture 2) + ".*" + + ")\\)|)", // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter + rwhitespace = new RegExp( whitespace + "+", "g" ), rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), - rsibling = new RegExp( whitespace + "*[+~]" ), - rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*)" + whitespace + "*\\]", "g" ), + rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), rpseudo = new RegExp( pseudos ), ridentifier = new RegExp( "^" + identifier + "$" ), matchExpr = { - "ID": new RegExp( "^#(" + characterEncoding + ")" ), - "CLASS": new RegExp( "^\\.(" + characterEncoding + ")" ), - "TAG": new RegExp( "^(" + characterEncoding.replace( "w", "w*" ) + ")" ), + "ID": new RegExp( "^#(" + identifier + ")" ), + "CLASS": new RegExp( "^\\.(" + identifier + ")" ), + "TAG": new RegExp( "^(" + identifier + "|[*])" ), "ATTR": new RegExp( "^" + attributes ), "PSEUDO": new RegExp( "^" + pseudos ), "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + @@ -1125,31 +622,66 @@ var i, whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) }, + rinputs = /^(?:input|select|textarea|button)$/i, + rheader = /^h\d$/i, + rnative = /^[^{]+\{\s*\[native \w/, // Easily-parseable/retrievable ID or TAG or CLASS selectors rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, - rinputs = /^(?:input|select|textarea|button)$/i, - rheader = /^h\d$/i, + rsibling = /[+~]/, - rescape = /'|\\/g, - - // CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters + // CSS escapes + // http://www.w3.org/TR/CSS21/syndata.html#escaped-characters runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), funescape = function( _, escaped, escapedWhitespace ) { var high = "0x" + escaped - 0x10000; // NaN means non-codepoint - // Support: Firefox + // Support: Firefox<24 // Workaround erroneous numeric interpretation of +"0x" return high !== high || escapedWhitespace ? escaped : - // BMP codepoint high < 0 ? + // BMP codepoint String.fromCharCode( high + 0x10000 ) : // Supplemental Plane codepoint (surrogate pair) String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); - }; + }, + + // CSS string/identifier serialization + // https://drafts.csswg.org/cssom/#common-serializing-idioms + rcssescape = /([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g, + fcssescape = function( ch, asCodePoint ) { + if ( asCodePoint ) { + + // U+0000 NULL becomes U+FFFD REPLACEMENT CHARACTER + if ( ch === "\0" ) { + return "\uFFFD"; + } + + // Control characters and (dependent upon position) numbers get escaped as code points + return ch.slice( 0, -1 ) + "\\" + ch.charCodeAt( ch.length - 1 ).toString( 16 ) + " "; + } + + // Other potentially-special ASCII characters get backslash-escaped + return "\\" + ch; + }, + + // Used for iframes + // See setDocument() + // Removing the function wrapper causes a "Permission Denied" + // error in IE + unloadHandler = function() { + setDocument(); + }, + + disabledAncestor = addCombinator( + function( elem ) { + return elem.disabled === true && ("form" in elem || "label" in elem); + }, + { dir: "parentNode", next: "legend" } + ); // Optimize for push.apply( _, NodeList ) try { @@ -1181,104 +713,128 @@ try { } function Sizzle( selector, context, results, seed ) { - var match, elem, m, nodeType, - // QSA vars - i, groups, old, nid, newContext, newSelector; + var m, i, elem, nid, match, groups, newSelector, + newContext = context && context.ownerDocument, - if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { - setDocument( context ); - } + // nodeType defaults to 9, since context defaults to document + nodeType = context ? context.nodeType : 9; - context = context || document; results = results || []; - if ( !selector || typeof selector !== "string" ) { + // Return early from calls with invalid selector or context + if ( typeof selector !== "string" || !selector || + nodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) { + return results; } - if ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) { - return []; - } + // Try to shortcut find operations (as opposed to filters) in HTML documents + if ( !seed ) { - if ( documentIsHTML && !seed ) { + if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { + setDocument( context ); + } + context = context || document; + + if ( documentIsHTML ) { + + // If the selector is sufficiently simple, try using a "get*By*" DOM method + // (excepting DocumentFragment context, where the methods don't exist) + if ( nodeType !== 11 && (match = rquickExpr.exec( selector )) ) { + + // ID selector + if ( (m = match[1]) ) { + + // Document context + if ( nodeType === 9 ) { + if ( (elem = context.getElementById( m )) ) { + + // Support: IE, Opera, Webkit + // TODO: identify versions + // getElementById can match elements by name instead of ID + if ( elem.id === m ) { + results.push( elem ); + return results; + } + } else { + return results; + } + + // Element context + } else { + + // Support: IE, Opera, Webkit + // TODO: identify versions + // getElementById can match elements by name instead of ID + if ( newContext && (elem = newContext.getElementById( m )) && + contains( context, elem ) && + elem.id === m ) { - // Shortcuts - if ( (match = rquickExpr.exec( selector )) ) { - // Speed-up: Sizzle("#ID") - if ( (m = match[1]) ) { - if ( nodeType === 9 ) { - elem = context.getElementById( m ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - if ( elem && elem.parentNode ) { - // Handle the case where IE, Opera, and Webkit return items - // by name instead of ID - if ( elem.id === m ) { results.push( elem ); return results; } - } else { - return results; } - } else { - // Context is not a document - if ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) && - contains( context, elem ) && elem.id === m ) { - results.push( elem ); - return results; - } - } - // Speed-up: Sizzle("TAG") - } else if ( match[2] ) { - push.apply( results, context.getElementsByTagName( selector ) ); - return results; - - // Speed-up: Sizzle(".CLASS") - } else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) { - push.apply( results, context.getElementsByClassName( m ) ); - return results; - } - } - - // QSA path - if ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { - nid = old = expando; - newContext = context; - newSelector = nodeType === 9 && selector; - - // qSA works strangely on Element-rooted queries - // We can work around this by specifying an extra ID on the root - // and working up from there (Thanks to Andrew Dupont for the technique) - // IE 8 doesn't work on object elements - if ( nodeType === 1 && context.nodeName.toLowerCase() !== "object" ) { - groups = tokenize( selector ); - - if ( (old = context.getAttribute("id")) ) { - nid = old.replace( rescape, "\\$&" ); - } else { - context.setAttribute( "id", nid ); - } - nid = "[id='" + nid + "'] "; - - i = groups.length; - while ( i-- ) { - groups[i] = nid + toSelector( groups[i] ); - } - newContext = rsibling.test( selector ) && context.parentNode || context; - newSelector = groups.join(","); - } - - if ( newSelector ) { - try { - push.apply( results, - newContext.querySelectorAll( newSelector ) - ); + // Type selector + } else if ( match[2] ) { + push.apply( results, context.getElementsByTagName( selector ) ); return results; - } catch(qsaError) { - } finally { - if ( !old ) { - context.removeAttribute("id"); + + // Class selector + } else if ( (m = match[3]) && support.getElementsByClassName && + context.getElementsByClassName ) { + + push.apply( results, context.getElementsByClassName( m ) ); + return results; + } + } + + // Take advantage of querySelectorAll + if ( support.qsa && + !compilerCache[ selector + " " ] && + (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { + + if ( nodeType !== 1 ) { + newContext = context; + newSelector = selector; + + // qSA looks outside Element context, which is not what we want + // Thanks to Andrew Dupont for this workaround technique + // Support: IE <=8 + // Exclude object elements + } else if ( context.nodeName.toLowerCase() !== "object" ) { + + // Capture the context ID, setting it first if necessary + if ( (nid = context.getAttribute( "id" )) ) { + nid = nid.replace( rcssescape, fcssescape ); + } else { + context.setAttribute( "id", (nid = expando) ); + } + + // Prefix every selector in the list + groups = tokenize( selector ); + i = groups.length; + while ( i-- ) { + groups[i] = "#" + nid + " " + toSelector( groups[i] ); + } + newSelector = groups.join( "," ); + + // Expand context for sibling selectors + newContext = rsibling.test( selector ) && testContext( context.parentNode ) || + context; + } + + if ( newSelector ) { + try { + push.apply( results, + newContext.querySelectorAll( newSelector ) + ); + return results; + } catch ( qsaError ) { + } finally { + if ( nid === expando ) { + context.removeAttribute( "id" ); + } } } } @@ -1291,7 +847,7 @@ function Sizzle( selector, context, results, seed ) { /** * Create key-value caches of limited size - * @returns {Function(string, Object)} Returns the Object data after storing it on itself with + * @returns {function(string, object)} Returns the Object data after storing it on itself with * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) * deleting the oldest entry */ @@ -1300,11 +856,11 @@ function createCache() { function cache( key, value ) { // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) - if ( keys.push( key += " " ) > Expr.cacheLength ) { + if ( keys.push( key + " " ) > Expr.cacheLength ) { // Only keep the most recent entries delete cache[ keys.shift() ]; } - return (cache[ key ] = value); + return (cache[ key + " " ] = value); } return cache; } @@ -1320,22 +876,22 @@ function markFunction( fn ) { /** * Support testing using an element - * @param {Function} fn Passed the created div and expects a boolean result + * @param {Function} fn Passed the created element and returns a boolean result */ function assert( fn ) { - var div = document.createElement("div"); + var el = document.createElement("fieldset"); try { - return !!fn( div ); + return !!fn( el ); } catch (e) { return false; } finally { // Remove from its parent by default - if ( div.parentNode ) { - div.parentNode.removeChild( div ); + if ( el.parentNode ) { + el.parentNode.removeChild( el ); } // release memory in IE - div = null; + el = null; } } @@ -1346,7 +902,7 @@ function assert( fn ) { */ function addHandle( attrs, handler ) { var arr = attrs.split("|"), - i = attrs.length; + i = arr.length; while ( i-- ) { Expr.attrHandle[ arr[i] ] = handler; @@ -1362,8 +918,7 @@ function addHandle( attrs, handler ) { function siblingCheck( a, b ) { var cur = b && a, diff = cur && a.nodeType === 1 && b.nodeType === 1 && - ( ~b.sourceIndex || MAX_NEGATIVE ) - - ( ~a.sourceIndex || MAX_NEGATIVE ); + a.sourceIndex - b.sourceIndex; // Use IE sourceIndex if available on both nodes if ( diff ) { @@ -1404,6 +959,62 @@ function createButtonPseudo( type ) { }; } +/** + * Returns a function to use in pseudos for :enabled/:disabled + * @param {Boolean} disabled true for :disabled; false for :enabled + */ +function createDisabledPseudo( disabled ) { + + // Known :disabled false positives: fieldset[disabled] > legend:nth-of-type(n+2) :can-disable + return function( elem ) { + + // Only certain elements can match :enabled or :disabled + // https://html.spec.whatwg.org/multipage/scripting.html#selector-enabled + // https://html.spec.whatwg.org/multipage/scripting.html#selector-disabled + if ( "form" in elem ) { + + // Check for inherited disabledness on relevant non-disabled elements: + // * listed form-associated elements in a disabled fieldset + // https://html.spec.whatwg.org/multipage/forms.html#category-listed + // https://html.spec.whatwg.org/multipage/forms.html#concept-fe-disabled + // * option elements in a disabled optgroup + // https://html.spec.whatwg.org/multipage/forms.html#concept-option-disabled + // All such elements have a "form" property. + if ( elem.parentNode && elem.disabled === false ) { + + // Option elements defer to a parent optgroup if present + if ( "label" in elem ) { + if ( "label" in elem.parentNode ) { + return elem.parentNode.disabled === disabled; + } else { + return elem.disabled === disabled; + } + } + + // Support: IE 6 - 11 + // Use the isDisabled shortcut property to check for disabled fieldset ancestors + return elem.isDisabled === disabled || + + // Where there is no isDisabled, check manually + /* jshint -W018 */ + elem.isDisabled !== !disabled && + disabledAncestor( elem ) === disabled; + } + + return elem.disabled === disabled; + + // Try to winnow out elements that can't be disabled before trusting the disabled property. + // Some victims get caught in our net (label, legend, menu, track), but it shouldn't + // even exist on them, let alone have a boolean value. + } else if ( "label" in elem ) { + return elem.disabled === disabled; + } + + // Remaining elements are neither :enabled nor :disabled + return false; + }; +} + /** * Returns a function to use in pseudos for positionals * @param {Function} fn @@ -1427,8 +1038,21 @@ function createPositionalPseudo( fn ) { } /** - * Detect xml + * Checks a node for validity as a Sizzle context + * @param {Element|Object=} context + * @returns {Element|Object|Boolean} The input node if acceptable, otherwise a falsy value + */ +function testContext( context ) { + return context && typeof context.getElementsByTagName !== "undefined" && context; +} + +// Expose support vars for convenience +support = Sizzle.support = {}; + +/** + * Detects XML nodes * @param {Element|Object} elem An element or a document + * @returns {Boolean} True iff elem is a non-HTML XML node */ isXML = Sizzle.isXML = function( elem ) { // documentElement is verified for cases where it doesn't yet exist @@ -1437,121 +1061,144 @@ isXML = Sizzle.isXML = function( elem ) { return documentElement ? documentElement.nodeName !== "HTML" : false; }; -// Expose support vars for convenience -support = Sizzle.support = {}; - /** * Sets document-related variables once based on the current document * @param {Element|Object} [doc] An element or document object to use to set the document * @returns {Object} Returns the current document */ setDocument = Sizzle.setDocument = function( node ) { - var doc = node ? node.ownerDocument || node : preferredDoc, - parent = doc.defaultView; + var hasCompare, subWindow, + doc = node ? node.ownerDocument || node : preferredDoc; - // If no document and documentElement is available, return + // Return early if doc is invalid or already selected if ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) { return document; } - // Set our document + // Update global variables document = doc; - docElem = doc.documentElement; + docElem = document.documentElement; + documentIsHTML = !isXML( document ); - // Support tests - documentIsHTML = !isXML( doc ); + // Support: IE 9-11, Edge + // Accessing iframe documents after unload throws "permission denied" errors (jQuery #13936) + if ( preferredDoc !== document && + (subWindow = document.defaultView) && subWindow.top !== subWindow ) { - // Support: IE>8 - // If iframe document is assigned to "document" variable and if iframe has been reloaded, - // IE will throw "permission denied" error when accessing "document" variable, see jQuery #13936 - // IE6-8 do not support the defaultView property so parent will be undefined - if ( parent && parent.attachEvent && parent !== parent.top ) { - parent.attachEvent( "onbeforeunload", function() { - setDocument(); - }); + // Support: IE 11, Edge + if ( subWindow.addEventListener ) { + subWindow.addEventListener( "unload", unloadHandler, false ); + + // Support: IE 9 - 10 only + } else if ( subWindow.attachEvent ) { + subWindow.attachEvent( "onunload", unloadHandler ); + } } /* Attributes ---------------------------------------------------------------------- */ // Support: IE<8 - // Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans) - support.attributes = assert(function( div ) { - div.className = "i"; - return !div.getAttribute("className"); + // Verify that getAttribute really returns attributes and not properties + // (excepting IE8 booleans) + support.attributes = assert(function( el ) { + el.className = "i"; + return !el.getAttribute("className"); }); /* getElement(s)By* ---------------------------------------------------------------------- */ // Check if getElementsByTagName("*") returns only elements - support.getElementsByTagName = assert(function( div ) { - div.appendChild( doc.createComment("") ); - return !div.getElementsByTagName("*").length; + support.getElementsByTagName = assert(function( el ) { + el.appendChild( document.createComment("") ); + return !el.getElementsByTagName("*").length; }); - // Check if getElementsByClassName can be trusted - support.getElementsByClassName = assert(function( div ) { - div.innerHTML = "
"; - - // Support: Safari<4 - // Catch class over-caching - div.firstChild.className = "i"; - // Support: Opera<10 - // Catch gEBCN failure to find non-leading classes - return div.getElementsByClassName("i").length === 2; - }); + // Support: IE<9 + support.getElementsByClassName = rnative.test( document.getElementsByClassName ); // Support: IE<10 // Check if getElementById returns elements by name // The broken getElementById methods don't pick up programmatically-set names, // so use a roundabout getElementsByName test - support.getById = assert(function( div ) { - docElem.appendChild( div ).id = expando; - return !doc.getElementsByName || !doc.getElementsByName( expando ).length; + support.getById = assert(function( el ) { + docElem.appendChild( el ).id = expando; + return !document.getElementsByName || !document.getElementsByName( expando ).length; }); - // ID find and filter + // ID filter and find if ( support.getById ) { - Expr.find["ID"] = function( id, context ) { - if ( typeof context.getElementById !== strundefined && documentIsHTML ) { - var m = context.getElementById( id ); - // Check parentNode to catch when Blackberry 4.6 returns - // nodes that are no longer in the document #6963 - return m && m.parentNode ? [m] : []; - } - }; Expr.filter["ID"] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { return elem.getAttribute("id") === attrId; }; }; + Expr.find["ID"] = function( id, context ) { + if ( typeof context.getElementById !== "undefined" && documentIsHTML ) { + var elem = context.getElementById( id ); + return elem ? [ elem ] : []; + } + }; } else { - // Support: IE6/7 - // getElementById is not reliable as a find shortcut - delete Expr.find["ID"]; - Expr.filter["ID"] = function( id ) { var attrId = id.replace( runescape, funescape ); return function( elem ) { - var node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode("id"); + var node = typeof elem.getAttributeNode !== "undefined" && + elem.getAttributeNode("id"); return node && node.value === attrId; }; }; + + // Support: IE 6 - 7 only + // getElementById is not reliable as a find shortcut + Expr.find["ID"] = function( id, context ) { + if ( typeof context.getElementById !== "undefined" && documentIsHTML ) { + var node, i, elems, + elem = context.getElementById( id ); + + if ( elem ) { + + // Verify the id attribute + node = elem.getAttributeNode("id"); + if ( node && node.value === id ) { + return [ elem ]; + } + + // Fall back on getElementsByName + elems = context.getElementsByName( id ); + i = 0; + while ( (elem = elems[i++]) ) { + node = elem.getAttributeNode("id"); + if ( node && node.value === id ) { + return [ elem ]; + } + } + } + + return []; + } + }; } // Tag Expr.find["TAG"] = support.getElementsByTagName ? function( tag, context ) { - if ( typeof context.getElementsByTagName !== strundefined ) { + if ( typeof context.getElementsByTagName !== "undefined" ) { return context.getElementsByTagName( tag ); + + // DocumentFragment nodes don't have gEBTN + } else if ( support.qsa ) { + return context.querySelectorAll( tag ); } } : + function( tag, context ) { var elem, tmp = [], i = 0, + // By happy coincidence, a (broken) gEBTN appears on DocumentFragment nodes too results = context.getElementsByTagName( tag ); // Filter out possible comments @@ -1569,7 +1216,7 @@ setDocument = Sizzle.setDocument = function( node ) { // Class Expr.find["CLASS"] = support.getElementsByClassName && function( className, context ) { - if ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) { + if ( typeof context.getElementsByClassName !== "undefined" && documentIsHTML ) { return context.getElementsByClassName( className ); } }; @@ -1586,74 +1233,105 @@ setDocument = Sizzle.setDocument = function( node ) { // We allow this because of a bug in IE8/9 that throws an error // whenever `document.activeElement` is accessed on an iframe // So, we allow :focus to pass through QSA all the time to avoid the IE error - // See http://bugs.jquery.com/ticket/13378 + // See https://bugs.jquery.com/ticket/13378 rbuggyQSA = []; - if ( (support.qsa = rnative.test( doc.querySelectorAll )) ) { + if ( (support.qsa = rnative.test( document.querySelectorAll )) ) { // Build QSA regex // Regex strategy adopted from Diego Perini - assert(function( div ) { + assert(function( el ) { // Select is set to empty string on purpose // This is to test IE's treatment of not explicitly // setting a boolean content attribute, // since its presence should be enough - // http://bugs.jquery.com/ticket/12359 - div.innerHTML = ""; + // https://bugs.jquery.com/ticket/12359 + docElem.appendChild( el ).innerHTML = "" + + ""; + + // Support: IE8, Opera 11-12.16 + // Nothing should be selected when empty strings follow ^= or $= or *= + // The test attribute must be unknown in Opera but "safe" for WinRT + // https://msdn.microsoft.com/en-us/library/ie/hh465388.aspx#attribute_section + if ( el.querySelectorAll("[msallowcapture^='']").length ) { + rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); + } // Support: IE8 // Boolean attributes and "value" are not treated correctly - if ( !div.querySelectorAll("[selected]").length ) { + if ( !el.querySelectorAll("[selected]").length ) { rbuggyQSA.push( "\\[" + whitespace + "*(?:value|" + booleans + ")" ); } + // Support: Chrome<29, Android<4.4, Safari<7.0+, iOS<7.0+, PhantomJS<1.9.8+ + if ( !el.querySelectorAll( "[id~=" + expando + "-]" ).length ) { + rbuggyQSA.push("~="); + } + // Webkit/Opera - :checked should return selected option elements // http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":checked").length ) { + if ( !el.querySelectorAll(":checked").length ) { rbuggyQSA.push(":checked"); } + + // Support: Safari 8+, iOS 8+ + // https://bugs.webkit.org/show_bug.cgi?id=136851 + // In-page `selector#id sibling-combinator selector` fails + if ( !el.querySelectorAll( "a#" + expando + "+*" ).length ) { + rbuggyQSA.push(".#.+[+~]"); + } }); - assert(function( div ) { + assert(function( el ) { + el.innerHTML = "" + + ""; - // Support: Opera 10-12/IE8 - // ^= $= *= and empty values - // Should not select anything // Support: Windows 8 Native Apps - // The type attribute is restricted during .innerHTML assignment - var input = doc.createElement("input"); + // The type and name attributes are restricted during .innerHTML assignment + var input = document.createElement("input"); input.setAttribute( "type", "hidden" ); - div.appendChild( input ).setAttribute( "t", "" ); + el.appendChild( input ).setAttribute( "name", "D" ); - if ( div.querySelectorAll("[t^='']").length ) { - rbuggyQSA.push( "[*^$]=" + whitespace + "*(?:''|\"\")" ); + // Support: IE8 + // Enforce case-sensitivity of name attribute + if ( el.querySelectorAll("[name=d]").length ) { + rbuggyQSA.push( "name" + whitespace + "*[*^$|!~]?=" ); } // FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled) // IE8 throws error here and will not see later tests - if ( !div.querySelectorAll(":enabled").length ) { + if ( el.querySelectorAll(":enabled").length !== 2 ) { + rbuggyQSA.push( ":enabled", ":disabled" ); + } + + // Support: IE9-11+ + // IE's :disabled selector does not pick up the children of disabled fieldsets + docElem.appendChild( el ).disabled = true; + if ( el.querySelectorAll(":disabled").length !== 2 ) { rbuggyQSA.push( ":enabled", ":disabled" ); } // Opera 10-11 does not throw on post-comma invalid pseudos - div.querySelectorAll("*,:x"); + el.querySelectorAll("*,:x"); rbuggyQSA.push(",.*:"); }); } - if ( (support.matchesSelector = rnative.test( (matches = docElem.webkitMatchesSelector || + if ( (support.matchesSelector = rnative.test( (matches = docElem.matches || + docElem.webkitMatchesSelector || docElem.mozMatchesSelector || docElem.oMatchesSelector || docElem.msMatchesSelector) )) ) { - assert(function( div ) { + assert(function( el ) { // Check to see if it's possible to do matchesSelector // on a disconnected node (IE 9) - support.disconnectedMatch = matches.call( div, "div" ); + support.disconnectedMatch = matches.call( el, "*" ); // This should fail with an exception // Gecko does not error, returns false instead - matches.call( div, "[s!='']:x" ); + matches.call( el, "[s!='']:x" ); rbuggyMatches.push( "!=", pseudos ); }); } @@ -1663,11 +1341,12 @@ setDocument = Sizzle.setDocument = function( node ) { /* Contains ---------------------------------------------------------------------- */ + hasCompare = rnative.test( docElem.compareDocumentPosition ); // Element contains another - // Purposefully does not implement inclusive descendent + // Purposefully self-exclusive // As in, an element does not contain itself - contains = rnative.test( docElem.contains ) || docElem.compareDocumentPosition ? + contains = hasCompare || rnative.test( docElem.contains ) ? function( a, b ) { var adown = a.nodeType === 9 ? a.documentElement : a, bup = b && b.parentNode; @@ -1692,7 +1371,7 @@ setDocument = Sizzle.setDocument = function( node ) { ---------------------------------------------------------------------- */ // Document order sorting - sortOrder = docElem.compareDocumentPosition ? + sortOrder = hasCompare ? function( a, b ) { // Flag for duplicate removal @@ -1701,34 +1380,46 @@ setDocument = Sizzle.setDocument = function( node ) { return 0; } - var compare = b.compareDocumentPosition && a.compareDocumentPosition && a.compareDocumentPosition( b ); - + // Sort on method existence if only one input has compareDocumentPosition + var compare = !a.compareDocumentPosition - !b.compareDocumentPosition; if ( compare ) { - // Disconnected nodes - if ( compare & 1 || - (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { - - // Choose the first element that is related to our preferred document - if ( a === doc || contains(preferredDoc, a) ) { - return -1; - } - if ( b === doc || contains(preferredDoc, b) ) { - return 1; - } - - // Maintain original order - return sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : - 0; - } - - return compare & 4 ? -1 : 1; + return compare; } - // Not directly comparable, sort on existence of method - return a.compareDocumentPosition ? -1 : 1; + // Calculate position if both inputs belong to the same document + compare = ( a.ownerDocument || a ) === ( b.ownerDocument || b ) ? + a.compareDocumentPosition( b ) : + + // Otherwise we know they are disconnected + 1; + + // Disconnected nodes + if ( compare & 1 || + (!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) { + + // Choose the first element that is related to our preferred document + if ( a === document || a.ownerDocument === preferredDoc && contains(preferredDoc, a) ) { + return -1; + } + if ( b === document || b.ownerDocument === preferredDoc && contains(preferredDoc, b) ) { + return 1; + } + + // Maintain original order + return sortInput ? + ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : + 0; + } + + return compare & 4 ? -1 : 1; } : function( a, b ) { + // Exit early if the nodes are identical + if ( a === b ) { + hasDuplicate = true; + return 0; + } + var cur, i = 0, aup = a.parentNode, @@ -1736,19 +1427,14 @@ setDocument = Sizzle.setDocument = function( node ) { ap = [ a ], bp = [ b ]; - // Exit early if the nodes are identical - if ( a === b ) { - hasDuplicate = true; - return 0; - // Parentless nodes are either documents or disconnected - } else if ( !aup || !bup ) { - return a === doc ? -1 : - b === doc ? 1 : + if ( !aup || !bup ) { + return a === document ? -1 : + b === document ? 1 : aup ? -1 : bup ? 1 : sortInput ? - ( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) : + ( indexOf( sortInput, a ) - indexOf( sortInput, b ) ) : 0; // If the nodes are siblings, we can do a quick check @@ -1781,7 +1467,7 @@ setDocument = Sizzle.setDocument = function( node ) { 0; }; - return doc; + return document; }; Sizzle.matches = function( expr, elements ) { @@ -1798,6 +1484,7 @@ Sizzle.matchesSelector = function( elem, expr ) { expr = expr.replace( rattributeQuotes, "='$1']" ); if ( support.matchesSelector && documentIsHTML && + !compilerCache[ expr + " " ] && ( !rbuggyMatches || !rbuggyMatches.test( expr ) ) && ( !rbuggyQSA || !rbuggyQSA.test( expr ) ) ) { @@ -1811,10 +1498,10 @@ Sizzle.matchesSelector = function( elem, expr ) { elem.document && elem.document.nodeType !== 11 ) { return ret; } - } catch(e) {} + } catch (e) {} } - return Sizzle( expr, document, null, [elem] ).length > 0; + return Sizzle( expr, document, null, [ elem ] ).length > 0; }; Sizzle.contains = function( context, elem ) { @@ -1837,13 +1524,17 @@ Sizzle.attr = function( elem, name ) { fn( elem, name, !documentIsHTML ) : undefined; - return val === undefined ? + return val !== undefined ? + val : support.attributes || !documentIsHTML ? elem.getAttribute( name ) : (val = elem.getAttributeNode(name)) && val.specified ? val.value : - null : - val; + null; +}; + +Sizzle.escape = function( sel ) { + return (sel + "").replace( rcssescape, fcssescape ); }; Sizzle.error = function( msg ) { @@ -1876,6 +1567,10 @@ Sizzle.uniqueSort = function( results ) { } } + // Clear input after sorting to release objects + // See https://github.com/jquery/sizzle/pull/225 + sortInput = null; + return results; }; @@ -1891,13 +1586,13 @@ getText = Sizzle.getText = function( elem ) { if ( !nodeType ) { // If no nodeType, this is expected to be an array - for ( ; (node = elem[i]); i++ ) { + while ( (node = elem[i++]) ) { // Do not traverse comment nodes ret += getText( node ); } } else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) { // Use textContent for elements - // innerText usage removed for consistency of new lines (see #11153) + // innerText usage removed for consistency of new lines (jQuery #11153) if ( typeof elem.textContent === "string" ) { return elem.textContent; } else { @@ -1939,7 +1634,7 @@ Expr = Sizzle.selectors = { match[1] = match[1].replace( runescape, funescape ); // Move the given value to match[3] whether quoted or unquoted - match[3] = ( match[4] || match[5] || "" ).replace( runescape, funescape ); + match[3] = ( match[3] || match[4] || match[5] || "" ).replace( runescape, funescape ); if ( match[2] === "~=" ) { match[3] = " " + match[3] + " "; @@ -1982,15 +1677,15 @@ Expr = Sizzle.selectors = { "PSEUDO": function( match ) { var excess, - unquoted = !match[5] && match[2]; + unquoted = !match[6] && match[2]; if ( matchExpr["CHILD"].test( match[0] ) ) { return null; } // Accept quoted arguments as-is - if ( match[3] && match[4] !== undefined ) { - match[2] = match[4]; + if ( match[3] ) { + match[2] = match[4] || match[5] || ""; // Strip excess characters from unquoted arguments } else if ( unquoted && rpseudo.test( unquoted ) && @@ -2026,7 +1721,7 @@ Expr = Sizzle.selectors = { return pattern || (pattern = new RegExp( "(^|" + whitespace + ")" + className + "(" + whitespace + "|$)" )) && classCache( className, function( elem ) { - return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute("class") || "" ); + return pattern.test( typeof elem.className === "string" && elem.className || typeof elem.getAttribute !== "undefined" && elem.getAttribute("class") || "" ); }); }, @@ -2048,7 +1743,7 @@ Expr = Sizzle.selectors = { operator === "^=" ? check && result.indexOf( check ) === 0 : operator === "*=" ? check && result.indexOf( check ) > -1 : operator === "$=" ? check && result.slice( -check.length ) === check : - operator === "~=" ? ( " " + result + " " ).indexOf( check ) > -1 : + operator === "~=" ? ( " " + result.replace( rwhitespace, " " ) + " " ).indexOf( check ) > -1 : operator === "|=" ? result === check || result.slice( 0, check.length + 1 ) === check + "-" : false; }; @@ -2067,11 +1762,12 @@ Expr = Sizzle.selectors = { } : function( elem, context, xml ) { - var cache, outerCache, node, diff, nodeIndex, start, + var cache, uniqueCache, outerCache, node, nodeIndex, start, dir = simple !== forward ? "nextSibling" : "previousSibling", parent = elem.parentNode, name = ofType && elem.nodeName.toLowerCase(), - useCache = !xml && !ofType; + useCache = !xml && !ofType, + diff = false; if ( parent ) { @@ -2080,7 +1776,10 @@ Expr = Sizzle.selectors = { while ( dir ) { node = elem; while ( (node = node[ dir ]) ) { - if ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) { + if ( ofType ? + node.nodeName.toLowerCase() === name : + node.nodeType === 1 ) { + return false; } } @@ -2094,11 +1793,21 @@ Expr = Sizzle.selectors = { // non-xml :nth-child(...) stores cache data on `parent` if ( forward && useCache ) { + // Seek `elem` from a previously-cached index - outerCache = parent[ expando ] || (parent[ expando ] = {}); - cache = outerCache[ type ] || []; - nodeIndex = cache[0] === dirruns && cache[1]; - diff = cache[0] === dirruns && cache[2]; + + // ...in a gzip-friendly way + node = parent; + outerCache = node[ expando ] || (node[ expando ] = {}); + + // Support: IE <9 only + // Defend against cloned attroperties (jQuery gh-1709) + uniqueCache = outerCache[ node.uniqueID ] || + (outerCache[ node.uniqueID ] = {}); + + cache = uniqueCache[ type ] || []; + nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; + diff = nodeIndex && cache[ 2 ]; node = nodeIndex && parent.childNodes[ nodeIndex ]; while ( (node = ++nodeIndex && node && node[ dir ] || @@ -2108,29 +1817,55 @@ Expr = Sizzle.selectors = { // When found, cache indexes on `parent` and break if ( node.nodeType === 1 && ++diff && node === elem ) { - outerCache[ type ] = [ dirruns, nodeIndex, diff ]; + uniqueCache[ type ] = [ dirruns, nodeIndex, diff ]; break; } } - // Use previously-cached element index if available - } else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) { - diff = cache[1]; - - // xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...) } else { - // Use the same loop as above to seek `elem` from the start - while ( (node = ++nodeIndex && node && node[ dir ] || - (diff = nodeIndex = 0) || start.pop()) ) { + // Use previously-cached element index if available + if ( useCache ) { + // ...in a gzip-friendly way + node = elem; + outerCache = node[ expando ] || (node[ expando ] = {}); - if ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) { - // Cache the index of each encountered element - if ( useCache ) { - (node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ]; - } + // Support: IE <9 only + // Defend against cloned attroperties (jQuery gh-1709) + uniqueCache = outerCache[ node.uniqueID ] || + (outerCache[ node.uniqueID ] = {}); - if ( node === elem ) { - break; + cache = uniqueCache[ type ] || []; + nodeIndex = cache[ 0 ] === dirruns && cache[ 1 ]; + diff = nodeIndex; + } + + // xml :nth-child(...) + // or :nth-last-child(...) or :nth(-last)?-of-type(...) + if ( diff === false ) { + // Use the same loop as above to seek `elem` from the start + while ( (node = ++nodeIndex && node && node[ dir ] || + (diff = nodeIndex = 0) || start.pop()) ) { + + if ( ( ofType ? + node.nodeName.toLowerCase() === name : + node.nodeType === 1 ) && + ++diff ) { + + // Cache the index of each encountered element + if ( useCache ) { + outerCache = node[ expando ] || (node[ expando ] = {}); + + // Support: IE <9 only + // Defend against cloned attroperties (jQuery gh-1709) + uniqueCache = outerCache[ node.uniqueID ] || + (outerCache[ node.uniqueID ] = {}); + + uniqueCache[ type ] = [ dirruns, diff ]; + } + + if ( node === elem ) { + break; + } } } } @@ -2168,7 +1903,7 @@ Expr = Sizzle.selectors = { matched = fn( seed, argument ), i = matched.length; while ( i-- ) { - idx = indexOf.call( seed, matched[i] ); + idx = indexOf( seed, matched[i] ); seed[ idx ] = !( matches[ idx ] = matched[i] ); } }) : @@ -2207,6 +1942,8 @@ Expr = Sizzle.selectors = { function( elem, context, xml ) { input[0] = elem; matcher( input, null, xml, results ); + // Don't keep the element (issue #299) + input[0] = null; return !results.pop(); }; }), @@ -2218,6 +1955,7 @@ Expr = Sizzle.selectors = { }), "contains": markFunction(function( text ) { + text = text.replace( runescape, funescape ); return function( elem ) { return ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1; }; @@ -2266,13 +2004,8 @@ Expr = Sizzle.selectors = { }, // Boolean properties - "enabled": function( elem ) { - return elem.disabled === false; - }, - - "disabled": function( elem ) { - return elem.disabled === true; - }, + "enabled": createDisabledPseudo( false ), + "disabled": createDisabledPseudo( true ), "checked": function( elem ) { // In CSS3, :checked should return both checked and selected elements @@ -2294,12 +2027,11 @@ Expr = Sizzle.selectors = { // Contents "empty": function( elem ) { // http://www.w3.org/TR/selectors/#empty-pseudo - // :empty is only affected by element nodes and content nodes(including text(3), cdata(4)), - // not comment, processing instructions, or others - // Thanks to Diego Perini for the nodeName shortcut - // Greater than "@" means alpha characters (specifically not starting with "#" or "?") + // :empty is negated by element (1) or content nodes (text: 3; cdata: 4; entity ref: 5), + // but not by others (comment: 8; processing instruction: 7; etc.) + // nodeType < 6 works because attributes (2) do not appear as children for ( elem = elem.firstChild; elem; elem = elem.nextSibling ) { - if ( elem.nodeName > "@" || elem.nodeType === 3 || elem.nodeType === 4 ) { + if ( elem.nodeType < 6 ) { return false; } } @@ -2326,11 +2058,12 @@ Expr = Sizzle.selectors = { "text": function( elem ) { var attr; - // IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc) - // use getAttribute instead to test this case return elem.nodeName.toLowerCase() === "input" && elem.type === "text" && - ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === elem.type ); + + // Support: IE<8 + // New HTML5 attribute values (e.g., "search") appear with elem.type === "text" + ( (attr = elem.getAttribute("type")) == null || attr.toLowerCase() === "text" ); }, // Position-in-collection @@ -2395,7 +2128,7 @@ function setFilters() {} setFilters.prototype = Expr.filters = Expr.pseudos; Expr.setFilters = new setFilters(); -function tokenize( selector, parseOnly ) { +tokenize = Sizzle.tokenize = function( selector, parseOnly ) { var matched, match, tokens, type, soFar, groups, preFilters, cached = tokenCache[ selector + " " ]; @@ -2416,7 +2149,7 @@ function tokenize( selector, parseOnly ) { // Don't consume trailing commas as valid soFar = soFar.slice( match[0].length ) || soFar; } - groups.push( tokens = [] ); + groups.push( (tokens = []) ); } matched = false; @@ -2460,7 +2193,7 @@ function tokenize( selector, parseOnly ) { Sizzle.error( selector ) : // Cache the tokens tokenCache( selector, groups ).slice( 0 ); -} +}; function toSelector( tokens ) { var i = 0, @@ -2474,7 +2207,9 @@ function toSelector( tokens ) { function addCombinator( matcher, combinator, base ) { var dir = combinator.dir, - checkNonElements = base && dir === "parentNode", + skip = combinator.next, + key = skip || dir, + checkNonElements = base && key === "parentNode", doneName = done++; return combinator.first ? @@ -2485,14 +2220,15 @@ function addCombinator( matcher, combinator, base ) { return matcher( elem, context, xml ); } } + return false; } : // Check against all ancestor/preceding elements function( elem, context, xml ) { - var data, cache, outerCache, - dirkey = dirruns + " " + doneName; + var oldCache, uniqueCache, outerCache, + newCache = [ dirruns, doneName ]; - // We can't set arbitrary data on XML nodes, so they don't benefit from dir caching + // We can't set arbitrary data on XML nodes, so they don't benefit from combinator caching if ( xml ) { while ( (elem = elem[ dir ]) ) { if ( elem.nodeType === 1 || checkNonElements ) { @@ -2505,20 +2241,31 @@ function addCombinator( matcher, combinator, base ) { while ( (elem = elem[ dir ]) ) { if ( elem.nodeType === 1 || checkNonElements ) { outerCache = elem[ expando ] || (elem[ expando ] = {}); - if ( (cache = outerCache[ dir ]) && cache[0] === dirkey ) { - if ( (data = cache[1]) === true || data === cachedruns ) { - return data === true; - } + + // Support: IE <9 only + // Defend against cloned attroperties (jQuery gh-1709) + uniqueCache = outerCache[ elem.uniqueID ] || (outerCache[ elem.uniqueID ] = {}); + + if ( skip && skip === elem.nodeName.toLowerCase() ) { + elem = elem[ dir ] || elem; + } else if ( (oldCache = uniqueCache[ key ]) && + oldCache[ 0 ] === dirruns && oldCache[ 1 ] === doneName ) { + + // Assign to newCache so results back-propagate to previous elements + return (newCache[ 2 ] = oldCache[ 2 ]); } else { - cache = outerCache[ dir ] = [ dirkey ]; - cache[1] = matcher( elem, context, xml ) || cachedruns; - if ( cache[1] === true ) { + // Reuse newcache so results back-propagate to previous elements + uniqueCache[ key ] = newCache; + + // A match means we're done; a fail means we have to keep checking + if ( (newCache[ 2 ] = matcher( elem, context, xml )) ) { return true; } } } } } + return false; }; } @@ -2536,6 +2283,15 @@ function elementMatcher( matchers ) { matchers[0]; } +function multipleContexts( selector, contexts, results ) { + var i = 0, + len = contexts.length; + for ( ; i < len; i++ ) { + Sizzle( selector, contexts[i], results ); + } + return results; +} + function condense( unmatched, map, filter, context, xml ) { var elem, newUnmatched = [], @@ -2627,7 +2383,7 @@ function setMatcher( preFilter, selector, matcher, postFilter, postFinder, postS i = matcherOut.length; while ( i-- ) { if ( (elem = matcherOut[i]) && - (temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) { + (temp = postFinder ? indexOf( seed, elem ) : preMap[i]) > -1 ) { seed[temp] = !(results[temp] = elem); } @@ -2662,13 +2418,16 @@ function matcherFromTokens( tokens ) { return elem === checkContext; }, implicitRelative, true ), matchAnyContext = addCombinator( function( elem ) { - return indexOf.call( checkContext, elem ) > -1; + return indexOf( checkContext, elem ) > -1; }, implicitRelative, true ), matchers = [ function( elem, context, xml ) { - return ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( + var ret = ( !leadingRelative && ( xml || context !== outermostContext ) ) || ( (checkContext = context).nodeType ? matchContext( elem, context, xml ) : matchAnyContext( elem, context, xml ) ); + // Avoid hanging onto element (issue #299) + checkContext = null; + return ret; } ]; for ( ; i < len; i++ ) { @@ -2706,42 +2465,43 @@ function matcherFromTokens( tokens ) { } function matcherFromGroupMatchers( elementMatchers, setMatchers ) { - // A counter to specify which element is currently being matched - var matcherCachedRuns = 0, - bySet = setMatchers.length > 0, + var bySet = setMatchers.length > 0, byElement = elementMatchers.length > 0, - superMatcher = function( seed, context, xml, results, expandContext ) { + superMatcher = function( seed, context, xml, results, outermost ) { var elem, j, matcher, - setMatched = [], matchedCount = 0, i = "0", unmatched = seed && [], - outermost = expandContext != null, + setMatched = [], contextBackup = outermostContext, - // We must always have either seed elements or context - elems = seed || byElement && Expr.find["TAG"]( "*", expandContext && context.parentNode || context ), + // We must always have either seed elements or outermost context + elems = seed || byElement && Expr.find["TAG"]( "*", outermost ), // Use integer dirruns iff this is the outermost matcher - dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1); + dirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1), + len = elems.length; if ( outermost ) { - outermostContext = context !== document && context; - cachedruns = matcherCachedRuns; + outermostContext = context === document || context || outermost; } // Add elements passing elementMatchers directly to results - // Keep `i` a string if there are no elements so `matchedCount` will be "00" below - for ( ; (elem = elems[i]) != null; i++ ) { + // Support: IE<9, Safari + // Tolerate NodeList properties (IE: "length"; Safari: ) matching elements by id + for ( ; i !== len && (elem = elems[i]) != null; i++ ) { if ( byElement && elem ) { j = 0; + if ( !context && elem.ownerDocument !== document ) { + setDocument( elem ); + xml = !documentIsHTML; + } while ( (matcher = elementMatchers[j++]) ) { - if ( matcher( elem, context, xml ) ) { + if ( matcher( elem, context || document, xml) ) { results.push( elem ); break; } } if ( outermost ) { dirruns = dirrunsUnique; - cachedruns = ++matcherCachedRuns; } } @@ -2759,8 +2519,17 @@ function matcherFromGroupMatchers( elementMatchers, setMatchers ) { } } - // Apply set filters to unmatched elements + // `i` is now the count of elements visited above, and adding it to `matchedCount` + // makes the latter nonnegative. matchedCount += i; + + // Apply set filters to unmatched elements + // NOTE: This can be skipped if there are no unmatched elements (i.e., `matchedCount` + // equals `i`), unless we didn't visit _any_ elements in the above loop because we have + // no element matchers and no seed. + // Incrementing an initially-string "0" `i` allows `i` to remain a string only in that + // case, which will result in a "00" `matchedCount` that differs from `i` but is also + // numerically zero. if ( bySet && i !== matchedCount ) { j = 0; while ( (matcher = setMatchers[j++]) ) { @@ -2806,7 +2575,7 @@ function matcherFromGroupMatchers( elementMatchers, setMatchers ) { superMatcher; } -compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { +compile = Sizzle.compile = function( selector, match /* Internal Use Only */ ) { var i, setMatchers = [], elementMatchers = [], @@ -2814,12 +2583,12 @@ compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { if ( !cached ) { // Generate a function of recursive functions that can be used to check each element - if ( !group ) { - group = tokenize( selector ); + if ( !match ) { + match = tokenize( selector ); } - i = group.length; + i = match.length; while ( i-- ) { - cached = matcherFromTokens( group[i] ); + cached = matcherFromTokens( match[i] ); if ( cached[ expando ] ) { setMatchers.push( cached ); } else { @@ -2829,108 +2598,117 @@ compile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) { // Cache the compiled function cached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) ); + + // Save selector and tokenization + cached.selector = selector; } return cached; }; -function multipleContexts( selector, contexts, results ) { - var i = 0, - len = contexts.length; - for ( ; i < len; i++ ) { - Sizzle( selector, contexts[i], results ); - } - return results; -} - -function select( selector, context, results, seed ) { +/** + * A low-level selection function that works with Sizzle's compiled + * selector functions + * @param {String|Function} selector A selector or a pre-compiled + * selector function built with Sizzle.compile + * @param {Element} context + * @param {Array} [results] + * @param {Array} [seed] A set of elements to match against + */ +select = Sizzle.select = function( selector, context, results, seed ) { var i, tokens, token, type, find, - match = tokenize( selector ); + compiled = typeof selector === "function" && selector, + match = !seed && tokenize( (selector = compiled.selector || selector) ); - if ( !seed ) { - // Try to minimize operations if there is only one group - if ( match.length === 1 ) { + results = results || []; - // Take a shortcut and set the context if the root selector is an ID - tokens = match[0] = match[0].slice( 0 ); - if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && - support.getById && context.nodeType === 9 && documentIsHTML && - Expr.relative[ tokens[1].type ] ) { + // Try to minimize operations if there is only one selector in the list and no seed + // (the latter of which guarantees us context) + if ( match.length === 1 ) { - context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; - if ( !context ) { - return results; - } - selector = selector.slice( tokens.shift().value.length ); + // Reduce context if the leading compound selector is an ID + tokens = match[0] = match[0].slice( 0 ); + if ( tokens.length > 2 && (token = tokens[0]).type === "ID" && + context.nodeType === 9 && documentIsHTML && Expr.relative[ tokens[1].type ] ) { + + context = ( Expr.find["ID"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0]; + if ( !context ) { + return results; + + // Precompiled matchers will still verify ancestry, so step up a level + } else if ( compiled ) { + context = context.parentNode; } - // Fetch a seed set for right-to-left matching - i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; - while ( i-- ) { - token = tokens[i]; + selector = selector.slice( tokens.shift().value.length ); + } - // Abort if we hit a combinator - if ( Expr.relative[ (type = token.type) ] ) { - break; - } - if ( (find = Expr.find[ type ]) ) { - // Search, expanding context for leading sibling combinators - if ( (seed = find( - token.matches[0].replace( runescape, funescape ), - rsibling.test( tokens[0].type ) && context.parentNode || context - )) ) { + // Fetch a seed set for right-to-left matching + i = matchExpr["needsContext"].test( selector ) ? 0 : tokens.length; + while ( i-- ) { + token = tokens[i]; - // If seed is empty or no tokens remain, we can return early - tokens.splice( i, 1 ); - selector = seed.length && toSelector( tokens ); - if ( !selector ) { - push.apply( results, seed ); - return results; - } + // Abort if we hit a combinator + if ( Expr.relative[ (type = token.type) ] ) { + break; + } + if ( (find = Expr.find[ type ]) ) { + // Search, expanding context for leading sibling combinators + if ( (seed = find( + token.matches[0].replace( runescape, funescape ), + rsibling.test( tokens[0].type ) && testContext( context.parentNode ) || context + )) ) { - break; + // If seed is empty or no tokens remain, we can return early + tokens.splice( i, 1 ); + selector = seed.length && toSelector( tokens ); + if ( !selector ) { + push.apply( results, seed ); + return results; } + + break; } } } } - // Compile and execute a filtering function + // Compile and execute a filtering function if one is not provided // Provide `match` to avoid retokenization if we modified the selector above - compile( selector, match )( + ( compiled || compile( selector, match ) )( seed, context, !documentIsHTML, results, - rsibling.test( selector ) + !context || rsibling.test( selector ) && testContext( context.parentNode ) || context ); return results; -} +}; // One-time assignments // Sort stability support.sortStable = expando.split("").sort( sortOrder ).join("") === expando; -// Support: Chrome<14 +// Support: Chrome 14-35+ // Always assume duplicates if they aren't passed to the comparison function -support.detectDuplicates = hasDuplicate; +support.detectDuplicates = !!hasDuplicate; // Initialize against the default document setDocument(); // Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27) // Detached nodes confoundingly follow *each other* -support.sortDetached = assert(function( div1 ) { +support.sortDetached = assert(function( el ) { // Should return 1, but returns 4 (following) - return div1.compareDocumentPosition( document.createElement("div") ) & 1; + return el.compareDocumentPosition( document.createElement("fieldset") ) & 1; }); // Support: IE<8 // Prevent attribute/property "interpolation" -// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx -if ( !assert(function( div ) { - div.innerHTML = ""; - return div.firstChild.getAttribute("href") === "#" ; +// https://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx +if ( !assert(function( el ) { + el.innerHTML = ""; + return el.firstChild.getAttribute("href") === "#" ; }) ) { addHandle( "type|href|height|width", function( elem, name, isXML ) { if ( !isXML ) { @@ -2941,10 +2719,10 @@ if ( !assert(function( div ) { // Support: IE<9 // Use defaultValue in place of getAttribute("value") -if ( !support.attributes || !assert(function( div ) { - div.innerHTML = ""; - div.firstChild.setAttribute( "value", "" ); - return div.firstChild.getAttribute( "value" ) === ""; +if ( !support.attributes || !assert(function( el ) { + el.innerHTML = ""; + el.firstChild.setAttribute( "value", "" ); + return el.firstChild.getAttribute( "value" ) === ""; }) ) { addHandle( "value", function( elem, name, isXML ) { if ( !isXML && elem.nodeName.toLowerCase() === "input" ) { @@ -2955,38 +2733,470 @@ if ( !support.attributes || !assert(function( div ) { // Support: IE<9 // Use getAttributeNode to fetch booleans when getAttribute lies -if ( !assert(function( div ) { - return div.getAttribute("disabled") == null; +if ( !assert(function( el ) { + return el.getAttribute("disabled") == null; }) ) { addHandle( booleans, function( elem, name, isXML ) { var val; if ( !isXML ) { - return (val = elem.getAttributeNode( name )) && val.specified ? - val.value : - elem[ name ] === true ? name.toLowerCase() : null; + return elem[ name ] === true ? name.toLowerCase() : + (val = elem.getAttributeNode( name )) && val.specified ? + val.value : + null; } }); } +return Sizzle; + +})( window ); + + + jQuery.find = Sizzle; jQuery.expr = Sizzle.selectors; -jQuery.expr[":"] = jQuery.expr.pseudos; -jQuery.unique = Sizzle.uniqueSort; + +// Deprecated +jQuery.expr[ ":" ] = jQuery.expr.pseudos; +jQuery.uniqueSort = jQuery.unique = Sizzle.uniqueSort; jQuery.text = Sizzle.getText; jQuery.isXMLDoc = Sizzle.isXML; jQuery.contains = Sizzle.contains; +jQuery.escapeSelector = Sizzle.escape; -})( window ); -// String to Object options format cache -var optionsCache = {}; -// Convert String-formatted options into Object-formatted ones and store in cache + +var dir = function( elem, dir, until ) { + var matched = [], + truncate = until !== undefined; + + while ( ( elem = elem[ dir ] ) && elem.nodeType !== 9 ) { + if ( elem.nodeType === 1 ) { + if ( truncate && jQuery( elem ).is( until ) ) { + break; + } + matched.push( elem ); + } + } + return matched; +}; + + +var siblings = function( n, elem ) { + var matched = []; + + for ( ; n; n = n.nextSibling ) { + if ( n.nodeType === 1 && n !== elem ) { + matched.push( n ); + } + } + + return matched; +}; + + +var rneedsContext = jQuery.expr.match.needsContext; + + + +function nodeName( elem, name ) { + + return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); + +}; +var rsingleTag = ( /^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i ); + + + +// Implement the identical functionality for filter and not +function winnow( elements, qualifier, not ) { + if ( isFunction( qualifier ) ) { + return jQuery.grep( elements, function( elem, i ) { + return !!qualifier.call( elem, i, elem ) !== not; + } ); + } + + // Single element + if ( qualifier.nodeType ) { + return jQuery.grep( elements, function( elem ) { + return ( elem === qualifier ) !== not; + } ); + } + + // Arraylike of elements (jQuery, arguments, Array) + if ( typeof qualifier !== "string" ) { + return jQuery.grep( elements, function( elem ) { + return ( indexOf.call( qualifier, elem ) > -1 ) !== not; + } ); + } + + // Filtered directly for both simple and complex selectors + return jQuery.filter( qualifier, elements, not ); +} + +jQuery.filter = function( expr, elems, not ) { + var elem = elems[ 0 ]; + + if ( not ) { + expr = ":not(" + expr + ")"; + } + + if ( elems.length === 1 && elem.nodeType === 1 ) { + return jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : []; + } + + return jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { + return elem.nodeType === 1; + } ) ); +}; + +jQuery.fn.extend( { + find: function( selector ) { + var i, ret, + len = this.length, + self = this; + + if ( typeof selector !== "string" ) { + return this.pushStack( jQuery( selector ).filter( function() { + for ( i = 0; i < len; i++ ) { + if ( jQuery.contains( self[ i ], this ) ) { + return true; + } + } + } ) ); + } + + ret = this.pushStack( [] ); + + for ( i = 0; i < len; i++ ) { + jQuery.find( selector, self[ i ], ret ); + } + + return len > 1 ? jQuery.uniqueSort( ret ) : ret; + }, + filter: function( selector ) { + return this.pushStack( winnow( this, selector || [], false ) ); + }, + not: function( selector ) { + return this.pushStack( winnow( this, selector || [], true ) ); + }, + is: function( selector ) { + return !!winnow( + this, + + // If this is a positional/relative selector, check membership in the returned set + // so $("p:first").is("p:last") won't return true for a doc with two "p". + typeof selector === "string" && rneedsContext.test( selector ) ? + jQuery( selector ) : + selector || [], + false + ).length; + } +} ); + + +// Initialize a jQuery object + + +// A central reference to the root jQuery(document) +var rootjQuery, + + // A simple way to check for HTML strings + // Prioritize #id over to avoid XSS via location.hash (#9521) + // Strict HTML recognition (#11290: must start with <) + // Shortcut simple #id case for speed + rquickExpr = /^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/, + + init = jQuery.fn.init = function( selector, context, root ) { + var match, elem; + + // HANDLE: $(""), $(null), $(undefined), $(false) + if ( !selector ) { + return this; + } + + // Method init() accepts an alternate rootjQuery + // so migrate can support jQuery.sub (gh-2101) + root = root || rootjQuery; + + // Handle HTML strings + if ( typeof selector === "string" ) { + if ( selector[ 0 ] === "<" && + selector[ selector.length - 1 ] === ">" && + selector.length >= 3 ) { + + // Assume that strings that start and end with <> are HTML and skip the regex check + match = [ null, selector, null ]; + + } else { + match = rquickExpr.exec( selector ); + } + + // Match html or make sure no context is specified for #id + if ( match && ( match[ 1 ] || !context ) ) { + + // HANDLE: $(html) -> $(array) + if ( match[ 1 ] ) { + context = context instanceof jQuery ? context[ 0 ] : context; + + // Option to run scripts is true for back-compat + // Intentionally let the error be thrown if parseHTML is not present + jQuery.merge( this, jQuery.parseHTML( + match[ 1 ], + context && context.nodeType ? context.ownerDocument || context : document, + true + ) ); + + // HANDLE: $(html, props) + if ( rsingleTag.test( match[ 1 ] ) && jQuery.isPlainObject( context ) ) { + for ( match in context ) { + + // Properties of context are called as methods if possible + if ( isFunction( this[ match ] ) ) { + this[ match ]( context[ match ] ); + + // ...and otherwise set as attributes + } else { + this.attr( match, context[ match ] ); + } + } + } + + return this; + + // HANDLE: $(#id) + } else { + elem = document.getElementById( match[ 2 ] ); + + if ( elem ) { + + // Inject the element directly into the jQuery object + this[ 0 ] = elem; + this.length = 1; + } + return this; + } + + // HANDLE: $(expr, $(...)) + } else if ( !context || context.jquery ) { + return ( context || root ).find( selector ); + + // HANDLE: $(expr, context) + // (which is just equivalent to: $(context).find(expr) + } else { + return this.constructor( context ).find( selector ); + } + + // HANDLE: $(DOMElement) + } else if ( selector.nodeType ) { + this[ 0 ] = selector; + this.length = 1; + return this; + + // HANDLE: $(function) + // Shortcut for document ready + } else if ( isFunction( selector ) ) { + return root.ready !== undefined ? + root.ready( selector ) : + + // Execute immediately if ready is not present + selector( jQuery ); + } + + return jQuery.makeArray( selector, this ); + }; + +// Give the init function the jQuery prototype for later instantiation +init.prototype = jQuery.fn; + +// Initialize central reference +rootjQuery = jQuery( document ); + + +var rparentsprev = /^(?:parents|prev(?:Until|All))/, + + // Methods guaranteed to produce a unique set when starting from a unique set + guaranteedUnique = { + children: true, + contents: true, + next: true, + prev: true + }; + +jQuery.fn.extend( { + has: function( target ) { + var targets = jQuery( target, this ), + l = targets.length; + + return this.filter( function() { + var i = 0; + for ( ; i < l; i++ ) { + if ( jQuery.contains( this, targets[ i ] ) ) { + return true; + } + } + } ); + }, + + closest: function( selectors, context ) { + var cur, + i = 0, + l = this.length, + matched = [], + targets = typeof selectors !== "string" && jQuery( selectors ); + + // Positional selectors never match, since there's no _selection_ context + if ( !rneedsContext.test( selectors ) ) { + for ( ; i < l; i++ ) { + for ( cur = this[ i ]; cur && cur !== context; cur = cur.parentNode ) { + + // Always skip document fragments + if ( cur.nodeType < 11 && ( targets ? + targets.index( cur ) > -1 : + + // Don't pass non-elements to Sizzle + cur.nodeType === 1 && + jQuery.find.matchesSelector( cur, selectors ) ) ) { + + matched.push( cur ); + break; + } + } + } + } + + return this.pushStack( matched.length > 1 ? jQuery.uniqueSort( matched ) : matched ); + }, + + // Determine the position of an element within the set + index: function( elem ) { + + // No argument, return index in parent + if ( !elem ) { + return ( this[ 0 ] && this[ 0 ].parentNode ) ? this.first().prevAll().length : -1; + } + + // Index in selector + if ( typeof elem === "string" ) { + return indexOf.call( jQuery( elem ), this[ 0 ] ); + } + + // Locate the position of the desired element + return indexOf.call( this, + + // If it receives a jQuery object, the first element is used + elem.jquery ? elem[ 0 ] : elem + ); + }, + + add: function( selector, context ) { + return this.pushStack( + jQuery.uniqueSort( + jQuery.merge( this.get(), jQuery( selector, context ) ) + ) + ); + }, + + addBack: function( selector ) { + return this.add( selector == null ? + this.prevObject : this.prevObject.filter( selector ) + ); + } +} ); + +function sibling( cur, dir ) { + while ( ( cur = cur[ dir ] ) && cur.nodeType !== 1 ) {} + return cur; +} + +jQuery.each( { + parent: function( elem ) { + var parent = elem.parentNode; + return parent && parent.nodeType !== 11 ? parent : null; + }, + parents: function( elem ) { + return dir( elem, "parentNode" ); + }, + parentsUntil: function( elem, i, until ) { + return dir( elem, "parentNode", until ); + }, + next: function( elem ) { + return sibling( elem, "nextSibling" ); + }, + prev: function( elem ) { + return sibling( elem, "previousSibling" ); + }, + nextAll: function( elem ) { + return dir( elem, "nextSibling" ); + }, + prevAll: function( elem ) { + return dir( elem, "previousSibling" ); + }, + nextUntil: function( elem, i, until ) { + return dir( elem, "nextSibling", until ); + }, + prevUntil: function( elem, i, until ) { + return dir( elem, "previousSibling", until ); + }, + siblings: function( elem ) { + return siblings( ( elem.parentNode || {} ).firstChild, elem ); + }, + children: function( elem ) { + return siblings( elem.firstChild ); + }, + contents: function( elem ) { + if ( nodeName( elem, "iframe" ) ) { + return elem.contentDocument; + } + + // Support: IE 9 - 11 only, iOS 7 only, Android Browser <=4.3 only + // Treat the template element as a regular one in browsers that + // don't support it. + if ( nodeName( elem, "template" ) ) { + elem = elem.content || elem; + } + + return jQuery.merge( [], elem.childNodes ); + } +}, function( name, fn ) { + jQuery.fn[ name ] = function( until, selector ) { + var matched = jQuery.map( this, fn, until ); + + if ( name.slice( -5 ) !== "Until" ) { + selector = until; + } + + if ( selector && typeof selector === "string" ) { + matched = jQuery.filter( selector, matched ); + } + + if ( this.length > 1 ) { + + // Remove duplicates + if ( !guaranteedUnique[ name ] ) { + jQuery.uniqueSort( matched ); + } + + // Reverse order for parents* and prev-derivatives + if ( rparentsprev.test( name ) ) { + matched.reverse(); + } + } + + return this.pushStack( matched ); + }; +} ); +var rnothtmlwhite = ( /[^\x20\t\r\n\f]+/g ); + + + +// Convert String-formatted options into Object-formatted ones function createOptions( options ) { - var object = optionsCache[ options ] = {}; - jQuery.each( options.match( core_rnotwhite ) || [], function( _, flag ) { + var object = {}; + jQuery.each( options.match( rnothtmlwhite ) || [], function( _, flag ) { object[ flag ] = true; - }); + } ); return object; } @@ -3017,156 +3227,186 @@ jQuery.Callbacks = function( options ) { // Convert options from String-formatted to Object-formatted if needed // (we check in cache first) options = typeof options === "string" ? - ( optionsCache[ options ] || createOptions( options ) ) : + createOptions( options ) : jQuery.extend( {}, options ); var // Flag to know if list is currently firing firing, - // Last fire value (for non-forgettable lists) + + // Last fire value for non-forgettable lists memory, + // Flag to know if list was already fired fired, - // End of the loop when firing - firingLength, - // Index of currently firing callback (modified by remove if needed) - firingIndex, - // First callback to fire (used internally by add and fireWith) - firingStart, + + // Flag to prevent firing + locked, + // Actual callback list list = [], - // Stack of fire calls for repeatable lists - stack = !options.once && [], + + // Queue of execution data for repeatable lists + queue = [], + + // Index of currently firing callback (modified by add/remove as needed) + firingIndex = -1, + // Fire callbacks - fire = function( data ) { - memory = options.memory && data; - fired = true; - firingIndex = firingStart || 0; - firingStart = 0; - firingLength = list.length; - firing = true; - for ( ; list && firingIndex < firingLength; firingIndex++ ) { - if ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) { - memory = false; // To prevent further calls using add - break; + fire = function() { + + // Enforce single-firing + locked = locked || options.once; + + // Execute callbacks for all pending executions, + // respecting firingIndex overrides and runtime changes + fired = firing = true; + for ( ; queue.length; firingIndex = -1 ) { + memory = queue.shift(); + while ( ++firingIndex < list.length ) { + + // Run callback and check for early termination + if ( list[ firingIndex ].apply( memory[ 0 ], memory[ 1 ] ) === false && + options.stopOnFalse ) { + + // Jump to end and forget the data so .add doesn't re-fire + firingIndex = list.length; + memory = false; + } } } + + // Forget the data if we're done with it + if ( !options.memory ) { + memory = false; + } + firing = false; - if ( list ) { - if ( stack ) { - if ( stack.length ) { - fire( stack.shift() ); - } - } else if ( memory ) { + + // Clean up if we're done firing for good + if ( locked ) { + + // Keep an empty list if we have data for future add calls + if ( memory ) { list = []; + + // Otherwise, this object is spent } else { - self.disable(); + list = ""; } } }, + // Actual Callbacks object self = { + // Add a callback or a collection of callbacks to the list add: function() { if ( list ) { - // First, we save the current length - var start = list.length; - (function add( args ) { + + // If we have memory from a past run, we should fire after adding + if ( memory && !firing ) { + firingIndex = list.length - 1; + queue.push( memory ); + } + + ( function add( args ) { jQuery.each( args, function( _, arg ) { - var type = jQuery.type( arg ); - if ( type === "function" ) { + if ( isFunction( arg ) ) { if ( !options.unique || !self.has( arg ) ) { list.push( arg ); } - } else if ( arg && arg.length && type !== "string" ) { + } else if ( arg && arg.length && toType( arg ) !== "string" ) { + // Inspect recursively add( arg ); } - }); - })( arguments ); - // Do we need to add the callbacks to the - // current firing batch? - if ( firing ) { - firingLength = list.length; - // With memory, if we're not firing then - // we should call right away - } else if ( memory ) { - firingStart = start; - fire( memory ); + } ); + } )( arguments ); + + if ( memory && !firing ) { + fire(); } } return this; }, + // Remove a callback from the list remove: function() { - if ( list ) { - jQuery.each( arguments, function( _, arg ) { - var index; - while( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { - list.splice( index, 1 ); - // Handle firing indexes - if ( firing ) { - if ( index <= firingLength ) { - firingLength--; - } - if ( index <= firingIndex ) { - firingIndex--; - } - } + jQuery.each( arguments, function( _, arg ) { + var index; + while ( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) { + list.splice( index, 1 ); + + // Handle firing indexes + if ( index <= firingIndex ) { + firingIndex--; } - }); - } + } + } ); return this; }, + // Check if a given callback is in the list. // If no argument is given, return whether or not list has callbacks attached. has: function( fn ) { - return fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length ); + return fn ? + jQuery.inArray( fn, list ) > -1 : + list.length > 0; }, + // Remove all callbacks from the list empty: function() { - list = []; - firingLength = 0; - return this; - }, - // Have the list do nothing anymore - disable: function() { - list = stack = memory = undefined; - return this; - }, - // Is it disabled? - disabled: function() { - return !list; - }, - // Lock the list in its current state - lock: function() { - stack = undefined; - if ( !memory ) { - self.disable(); + if ( list ) { + list = []; } return this; }, - // Is it locked? - locked: function() { - return !stack; + + // Disable .fire and .add + // Abort any current/pending executions + // Clear all callbacks and values + disable: function() { + locked = queue = []; + list = memory = ""; + return this; }, + disabled: function() { + return !list; + }, + + // Disable .fire + // Also disable .add unless we have memory (since it would have no effect) + // Abort any pending executions + lock: function() { + locked = queue = []; + if ( !memory && !firing ) { + list = memory = ""; + } + return this; + }, + locked: function() { + return !!locked; + }, + // Call all callbacks with the given context and arguments fireWith: function( context, args ) { - if ( list && ( !fired || stack ) ) { + if ( !locked ) { args = args || []; args = [ context, args.slice ? args.slice() : args ]; - if ( firing ) { - stack.push( args ); - } else { - fire( args ); + queue.push( args ); + if ( !firing ) { + fire(); } } return this; }, + // Call all the callbacks with the given arguments fire: function() { self.fireWith( this, arguments ); return this; }, + // To know if the callbacks have already been called at least once fired: function() { return !!fired; @@ -3175,14 +3415,61 @@ jQuery.Callbacks = function( options ) { return self; }; -jQuery.extend({ + + +function Identity( v ) { + return v; +} +function Thrower( ex ) { + throw ex; +} + +function adoptValue( value, resolve, reject, noValue ) { + var method; + + try { + + // Check for promise aspect first to privilege synchronous behavior + if ( value && isFunction( ( method = value.promise ) ) ) { + method.call( value ).done( resolve ).fail( reject ); + + // Other thenables + } else if ( value && isFunction( ( method = value.then ) ) ) { + method.call( value, resolve, reject ); + + // Other non-thenables + } else { + + // Control `resolve` arguments by letting Array#slice cast boolean `noValue` to integer: + // * false: [ value ].slice( 0 ) => resolve( value ) + // * true: [ value ].slice( 1 ) => resolve() + resolve.apply( undefined, [ value ].slice( noValue ) ); + } + + // For Promises/A+, convert exceptions into rejections + // Since jQuery.when doesn't unwrap thenables, we can skip the extra checks appearing in + // Deferred#then to conditionally suppress rejection. + } catch ( value ) { + + // Support: Android 4.0 only + // Strict mode functions invoked without .call/.apply get global-object context + reject.apply( undefined, [ value ] ); + } +} + +jQuery.extend( { Deferred: function( func ) { var tuples = [ - // action, add listener, listener list, final state - [ "resolve", "done", jQuery.Callbacks("once memory"), "resolved" ], - [ "reject", "fail", jQuery.Callbacks("once memory"), "rejected" ], - [ "notify", "progress", jQuery.Callbacks("memory") ] + + // action, add listener, callbacks, + // ... .then handlers, argument index, [final state] + [ "notify", "progress", jQuery.Callbacks( "memory" ), + jQuery.Callbacks( "memory" ), 2 ], + [ "resolve", "done", jQuery.Callbacks( "once memory" ), + jQuery.Callbacks( "once memory" ), 0, "resolved" ], + [ "reject", "fail", jQuery.Callbacks( "once memory" ), + jQuery.Callbacks( "once memory" ), 1, "rejected" ] ], state = "pending", promise = { @@ -3193,28 +3480,206 @@ jQuery.extend({ deferred.done( arguments ).fail( arguments ); return this; }, - then: function( /* fnDone, fnFail, fnProgress */ ) { - var fns = arguments; - return jQuery.Deferred(function( newDefer ) { - jQuery.each( tuples, function( i, tuple ) { - var action = tuple[ 0 ], - fn = jQuery.isFunction( fns[ i ] ) && fns[ i ]; - // deferred[ done | fail | progress ] for forwarding actions to newDefer - deferred[ tuple[1] ](function() { - var returned = fn && fn.apply( this, arguments ); - if ( returned && jQuery.isFunction( returned.promise ) ) { - returned.promise() - .done( newDefer.resolve ) - .fail( newDefer.reject ) - .progress( newDefer.notify ); - } else { - newDefer[ action + "With" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments ); - } - }); - }); - fns = null; - }).promise(); + "catch": function( fn ) { + return promise.then( null, fn ); }, + + // Keep pipe for back-compat + pipe: function( /* fnDone, fnFail, fnProgress */ ) { + var fns = arguments; + + return jQuery.Deferred( function( newDefer ) { + jQuery.each( tuples, function( i, tuple ) { + + // Map tuples (progress, done, fail) to arguments (done, fail, progress) + var fn = isFunction( fns[ tuple[ 4 ] ] ) && fns[ tuple[ 4 ] ]; + + // deferred.progress(function() { bind to newDefer or newDefer.notify }) + // deferred.done(function() { bind to newDefer or newDefer.resolve }) + // deferred.fail(function() { bind to newDefer or newDefer.reject }) + deferred[ tuple[ 1 ] ]( function() { + var returned = fn && fn.apply( this, arguments ); + if ( returned && isFunction( returned.promise ) ) { + returned.promise() + .progress( newDefer.notify ) + .done( newDefer.resolve ) + .fail( newDefer.reject ); + } else { + newDefer[ tuple[ 0 ] + "With" ]( + this, + fn ? [ returned ] : arguments + ); + } + } ); + } ); + fns = null; + } ).promise(); + }, + then: function( onFulfilled, onRejected, onProgress ) { + var maxDepth = 0; + function resolve( depth, deferred, handler, special ) { + return function() { + var that = this, + args = arguments, + mightThrow = function() { + var returned, then; + + // Support: Promises/A+ section 2.3.3.3.3 + // https://promisesaplus.com/#point-59 + // Ignore double-resolution attempts + if ( depth < maxDepth ) { + return; + } + + returned = handler.apply( that, args ); + + // Support: Promises/A+ section 2.3.1 + // https://promisesaplus.com/#point-48 + if ( returned === deferred.promise() ) { + throw new TypeError( "Thenable self-resolution" ); + } + + // Support: Promises/A+ sections 2.3.3.1, 3.5 + // https://promisesaplus.com/#point-54 + // https://promisesaplus.com/#point-75 + // Retrieve `then` only once + then = returned && + + // Support: Promises/A+ section 2.3.4 + // https://promisesaplus.com/#point-64 + // Only check objects and functions for thenability + ( typeof returned === "object" || + typeof returned === "function" ) && + returned.then; + + // Handle a returned thenable + if ( isFunction( then ) ) { + + // Special processors (notify) just wait for resolution + if ( special ) { + then.call( + returned, + resolve( maxDepth, deferred, Identity, special ), + resolve( maxDepth, deferred, Thrower, special ) + ); + + // Normal processors (resolve) also hook into progress + } else { + + // ...and disregard older resolution values + maxDepth++; + + then.call( + returned, + resolve( maxDepth, deferred, Identity, special ), + resolve( maxDepth, deferred, Thrower, special ), + resolve( maxDepth, deferred, Identity, + deferred.notifyWith ) + ); + } + + // Handle all other returned values + } else { + + // Only substitute handlers pass on context + // and multiple values (non-spec behavior) + if ( handler !== Identity ) { + that = undefined; + args = [ returned ]; + } + + // Process the value(s) + // Default process is resolve + ( special || deferred.resolveWith )( that, args ); + } + }, + + // Only normal processors (resolve) catch and reject exceptions + process = special ? + mightThrow : + function() { + try { + mightThrow(); + } catch ( e ) { + + if ( jQuery.Deferred.exceptionHook ) { + jQuery.Deferred.exceptionHook( e, + process.stackTrace ); + } + + // Support: Promises/A+ section 2.3.3.3.4.1 + // https://promisesaplus.com/#point-61 + // Ignore post-resolution exceptions + if ( depth + 1 >= maxDepth ) { + + // Only substitute handlers pass on context + // and multiple values (non-spec behavior) + if ( handler !== Thrower ) { + that = undefined; + args = [ e ]; + } + + deferred.rejectWith( that, args ); + } + } + }; + + // Support: Promises/A+ section 2.3.3.3.1 + // https://promisesaplus.com/#point-57 + // Re-resolve promises immediately to dodge false rejection from + // subsequent errors + if ( depth ) { + process(); + } else { + + // Call an optional hook to record the stack, in case of exception + // since it's otherwise lost when execution goes async + if ( jQuery.Deferred.getStackHook ) { + process.stackTrace = jQuery.Deferred.getStackHook(); + } + window.setTimeout( process ); + } + }; + } + + return jQuery.Deferred( function( newDefer ) { + + // progress_handlers.add( ... ) + tuples[ 0 ][ 3 ].add( + resolve( + 0, + newDefer, + isFunction( onProgress ) ? + onProgress : + Identity, + newDefer.notifyWith + ) + ); + + // fulfilled_handlers.add( ... ) + tuples[ 1 ][ 3 ].add( + resolve( + 0, + newDefer, + isFunction( onFulfilled ) ? + onFulfilled : + Identity + ) + ); + + // rejected_handlers.add( ... ) + tuples[ 2 ][ 3 ].add( + resolve( + 0, + newDefer, + isFunction( onRejected ) ? + onRejected : + Thrower + ) + ); + } ).promise(); + }, + // Get a promise for this deferred // If obj is provided, the promise aspect is added to the object promise: function( obj ) { @@ -3223,34 +3688,60 @@ jQuery.extend({ }, deferred = {}; - // Keep pipe for back-compat - promise.pipe = promise.then; - // Add list-specific methods jQuery.each( tuples, function( i, tuple ) { var list = tuple[ 2 ], - stateString = tuple[ 3 ]; + stateString = tuple[ 5 ]; - // promise[ done | fail | progress ] = list.add - promise[ tuple[1] ] = list.add; + // promise.progress = list.add + // promise.done = list.add + // promise.fail = list.add + promise[ tuple[ 1 ] ] = list.add; // Handle state if ( stateString ) { - list.add(function() { - // state = [ resolved | rejected ] - state = stateString; + list.add( + function() { - // [ reject_list | resolve_list ].disable; progress_list.lock - }, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock ); + // state = "resolved" (i.e., fulfilled) + // state = "rejected" + state = stateString; + }, + + // rejected_callbacks.disable + // fulfilled_callbacks.disable + tuples[ 3 - i ][ 2 ].disable, + + // rejected_handlers.disable + // fulfilled_handlers.disable + tuples[ 3 - i ][ 3 ].disable, + + // progress_callbacks.lock + tuples[ 0 ][ 2 ].lock, + + // progress_handlers.lock + tuples[ 0 ][ 3 ].lock + ); } - // deferred[ resolve | reject | notify ] - deferred[ tuple[0] ] = function() { - deferred[ tuple[0] + "With" ]( this === deferred ? promise : this, arguments ); + // progress_handlers.fire + // fulfilled_handlers.fire + // rejected_handlers.fire + list.add( tuple[ 3 ].fire ); + + // deferred.notify = function() { deferred.notifyWith(...) } + // deferred.resolve = function() { deferred.resolveWith(...) } + // deferred.reject = function() { deferred.rejectWith(...) } + deferred[ tuple[ 0 ] ] = function() { + deferred[ tuple[ 0 ] + "With" ]( this === deferred ? undefined : this, arguments ); return this; }; - deferred[ tuple[0] + "With" ] = list.fireWith; - }); + + // deferred.notifyWith = list.fireWith + // deferred.resolveWith = list.fireWith + // deferred.rejectWith = list.fireWith + deferred[ tuple[ 0 ] + "With" ] = list.fireWith; + } ); // Make the deferred a promise promise.promise( deferred ); @@ -3265,560 +3756,518 @@ jQuery.extend({ }, // Deferred helper - when: function( subordinate /* , ..., subordinateN */ ) { - var i = 0, - resolveValues = core_slice.call( arguments ), - length = resolveValues.length, + when: function( singleValue ) { + var - // the count of uncompleted subordinates - remaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0, + // count of uncompleted subordinates + remaining = arguments.length, - // the master Deferred. If resolveValues consist of only a single Deferred, just use that. - deferred = remaining === 1 ? subordinate : jQuery.Deferred(), + // count of unprocessed arguments + i = remaining, - // Update function for both resolve and progress values - updateFunc = function( i, contexts, values ) { + // subordinate fulfillment data + resolveContexts = Array( i ), + resolveValues = slice.call( arguments ), + + // the master Deferred + master = jQuery.Deferred(), + + // subordinate callback factory + updateFunc = function( i ) { return function( value ) { - contexts[ i ] = this; - values[ i ] = arguments.length > 1 ? core_slice.call( arguments ) : value; - if( values === progressValues ) { - deferred.notifyWith( contexts, values ); - } else if ( !( --remaining ) ) { - deferred.resolveWith( contexts, values ); + resolveContexts[ i ] = this; + resolveValues[ i ] = arguments.length > 1 ? slice.call( arguments ) : value; + if ( !( --remaining ) ) { + master.resolveWith( resolveContexts, resolveValues ); } }; - }, + }; - progressValues, progressContexts, resolveContexts; + // Single- and empty arguments are adopted like Promise.resolve + if ( remaining <= 1 ) { + adoptValue( singleValue, master.done( updateFunc( i ) ).resolve, master.reject, + !remaining ); - // add listeners to Deferred subordinates; treat others as resolved - if ( length > 1 ) { - progressValues = new Array( length ); - progressContexts = new Array( length ); - resolveContexts = new Array( length ); - for ( ; i < length; i++ ) { - if ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) { - resolveValues[ i ].promise() - .done( updateFunc( i, resolveContexts, resolveValues ) ) - .fail( deferred.reject ) - .progress( updateFunc( i, progressContexts, progressValues ) ); - } else { - --remaining; - } + // Use .then() to unwrap secondary thenables (cf. gh-3000) + if ( master.state() === "pending" || + isFunction( resolveValues[ i ] && resolveValues[ i ].then ) ) { + + return master.then(); } } - // if we're not waiting on anything, resolve the master - if ( !remaining ) { - deferred.resolveWith( resolveContexts, resolveValues ); + // Multiple arguments are aggregated like Promise.all array elements + while ( i-- ) { + adoptValue( resolveValues[ i ], updateFunc( i ), master.reject ); } - return deferred.promise(); + return master.promise(); } -}); -jQuery.support = (function( support ) { +} ); - var all, a, input, select, fragment, opt, eventName, isSupported, i, - div = document.createElement("div"); - // Setup - div.setAttribute( "className", "t" ); - div.innerHTML = "
a"; +// These usually indicate a programmer mistake during development, +// warn about them ASAP rather than swallowing them by default. +var rerrorNames = /^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/; - // Finish early in limited (non-browser) environments - all = div.getElementsByTagName("*") || []; - a = div.getElementsByTagName("a")[ 0 ]; - if ( !a || !a.style || !all.length ) { - return support; +jQuery.Deferred.exceptionHook = function( error, stack ) { + + // Support: IE 8 - 9 only + // Console exists when dev tools are open, which can happen at any time + if ( window.console && window.console.warn && error && rerrorNames.test( error.name ) ) { + window.console.warn( "jQuery.Deferred exception: " + error.message, error.stack, stack ); } +}; - // First batch of tests - select = document.createElement("select"); - opt = select.appendChild( document.createElement("option") ); - input = div.getElementsByTagName("input")[ 0 ]; - a.style.cssText = "top:1px;float:left;opacity:.5"; - // Test setAttribute on camelCase class. If it works, we need attrFixes when doing get/setAttribute (ie6/7) - support.getSetAttribute = div.className !== "t"; - // IE strips leading whitespace when .innerHTML is used - support.leadingWhitespace = div.firstChild.nodeType === 3; +jQuery.readyException = function( error ) { + window.setTimeout( function() { + throw error; + } ); +}; - // Make sure that tbody elements aren't automatically inserted - // IE will insert them into empty tables - support.tbody = !div.getElementsByTagName("tbody").length; - // Make sure that link elements get serialized correctly by innerHTML - // This requires a wrapper element in IE - support.htmlSerialize = !!div.getElementsByTagName("link").length; - // Get the style information from getAttribute - // (IE uses .cssText instead) - support.style = /top/.test( a.getAttribute("style") ); - // Make sure that URLs aren't manipulated - // (IE normalizes it by default) - support.hrefNormalized = a.getAttribute("href") === "/a"; +// The deferred used on DOM ready +var readyList = jQuery.Deferred(); - // Make sure that element opacity exists - // (IE uses filter instead) - // Use a regex to work around a WebKit issue. See #5145 - support.opacity = /^0.5/.test( a.style.opacity ); +jQuery.fn.ready = function( fn ) { - // Verify style float existence - // (IE uses styleFloat instead of cssFloat) - support.cssFloat = !!a.style.cssFloat; + readyList + .then( fn ) - // Check the default checkbox/radio value ("" on WebKit; "on" elsewhere) - support.checkOn = !!input.value; + // Wrap jQuery.readyException in a function so that the lookup + // happens at the time of error handling instead of callback + // registration. + .catch( function( error ) { + jQuery.readyException( error ); + } ); - // Make sure that a selected-by-default option has a working selected property. - // (WebKit defaults to false instead of true, IE too, if it's in an optgroup) - support.optSelected = opt.selected; + return this; +}; - // Tests for enctype support on a form (#6743) - support.enctype = !!document.createElement("form").enctype; +jQuery.extend( { - // Makes sure cloning an html5 element does not cause problems - // Where outerHTML is undefined, this still works - support.html5Clone = document.createElement("nav").cloneNode( true ).outerHTML !== "<:nav>"; + // Is the DOM ready to be used? Set to true once it occurs. + isReady: false, - // Will be defined later - support.inlineBlockNeedsLayout = false; - support.shrinkWrapBlocks = false; - support.pixelPosition = false; - support.deleteExpando = true; - support.noCloneEvent = true; - support.reliableMarginRight = true; - support.boxSizingReliable = true; + // A counter to track how many items to wait for before + // the ready event fires. See #6781 + readyWait: 1, - // Make sure checked status is properly cloned - input.checked = true; - support.noCloneChecked = input.cloneNode( true ).checked; + // Handle when the DOM is ready + ready: function( wait ) { - // Make sure that the options inside disabled selects aren't marked as disabled - // (WebKit marks them as disabled) - select.disabled = true; - support.optDisabled = !opt.disabled; - - // Support: IE<9 - try { - delete div.test; - } catch( e ) { - support.deleteExpando = false; - } - - // Check if we can trust getAttribute("value") - input = document.createElement("input"); - input.setAttribute( "value", "" ); - support.input = input.getAttribute( "value" ) === ""; - - // Check if an input maintains its value after becoming a radio - input.value = "t"; - input.setAttribute( "type", "radio" ); - support.radioValue = input.value === "t"; - - // #11217 - WebKit loses check when the name is after the checked attribute - input.setAttribute( "checked", "t" ); - input.setAttribute( "name", "t" ); - - fragment = document.createDocumentFragment(); - fragment.appendChild( input ); - - // Check if a disconnected checkbox will retain its checked - // value of true after appended to the DOM (IE6/7) - support.appendChecked = input.checked; - - // WebKit doesn't clone checked state correctly in fragments - support.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked; - - // Support: IE<9 - // Opera does not clone events (and typeof div.attachEvent === undefined). - // IE9-10 clones events bound via attachEvent, but they don't trigger with .click() - if ( div.attachEvent ) { - div.attachEvent( "onclick", function() { - support.noCloneEvent = false; - }); - - div.cloneNode( true ).click(); - } - - // Support: IE<9 (lack submit/change bubble), Firefox 17+ (lack focusin event) - // Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP) - for ( i in { submit: true, change: true, focusin: true }) { - div.setAttribute( eventName = "on" + i, "t" ); - - support[ i + "Bubbles" ] = eventName in window || div.attributes[ eventName ].expando === false; - } - - div.style.backgroundClip = "content-box"; - div.cloneNode( true ).style.backgroundClip = ""; - support.clearCloneStyle = div.style.backgroundClip === "content-box"; - - // Support: IE<9 - // Iteration over object's inherited properties before its own. - for ( i in jQuery( support ) ) { - break; - } - support.ownLast = i !== "0"; - - // Run tests that need a body at doc ready - jQuery(function() { - var container, marginDiv, tds, - divReset = "padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;", - body = document.getElementsByTagName("body")[0]; - - if ( !body ) { - // Return for frameset docs that don't have a body + // Abort if there are pending holds or we're already ready + if ( wait === true ? --jQuery.readyWait : jQuery.isReady ) { return; } - container = document.createElement("div"); - container.style.cssText = "border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px"; + // Remember that the DOM is ready + jQuery.isReady = true; - body.appendChild( container ).appendChild( div ); - - // Support: IE8 - // Check if table cells still have offsetWidth/Height when they are set - // to display:none and there are still other visible table cells in a - // table row; if so, offsetWidth/Height are not reliable for use when - // determining if an element has been hidden directly using - // display:none (it is still safe to use offsets if a parent element is - // hidden; don safety goggles and see bug #4512 for more information). - div.innerHTML = "
t
"; - tds = div.getElementsByTagName("td"); - tds[ 0 ].style.cssText = "padding:0;margin:0;border:0;display:none"; - isSupported = ( tds[ 0 ].offsetHeight === 0 ); - - tds[ 0 ].style.display = ""; - tds[ 1 ].style.display = "none"; - - // Support: IE8 - // Check if empty table cells still have offsetWidth/Height - support.reliableHiddenOffsets = isSupported && ( tds[ 0 ].offsetHeight === 0 ); - - // Check box-sizing and margin behavior. - div.innerHTML = ""; - div.style.cssText = "box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;"; - - // Workaround failing boxSizing test due to offsetWidth returning wrong value - // with some non-1 values of body zoom, ticket #13543 - jQuery.swap( body, body.style.zoom != null ? { zoom: 1 } : {}, function() { - support.boxSizing = div.offsetWidth === 4; - }); - - // Use window.getComputedStyle because jsdom on node.js will break without it. - if ( window.getComputedStyle ) { - support.pixelPosition = ( window.getComputedStyle( div, null ) || {} ).top !== "1%"; - support.boxSizingReliable = ( window.getComputedStyle( div, null ) || { width: "4px" } ).width === "4px"; - - // Check if div with explicit width and no margin-right incorrectly - // gets computed margin-right based on width of container. (#3333) - // Fails in WebKit before Feb 2011 nightlies - // WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right - marginDiv = div.appendChild( document.createElement("div") ); - marginDiv.style.cssText = div.style.cssText = divReset; - marginDiv.style.marginRight = marginDiv.style.width = "0"; - div.style.width = "1px"; - - support.reliableMarginRight = - !parseFloat( ( window.getComputedStyle( marginDiv, null ) || {} ).marginRight ); + // If a normal DOM Ready event fired, decrement, and wait if need be + if ( wait !== true && --jQuery.readyWait > 0 ) { + return; } - if ( typeof div.style.zoom !== core_strundefined ) { - // Support: IE<8 - // Check if natively block-level elements act like inline-block - // elements when setting their display to 'inline' and giving - // them layout - div.innerHTML = ""; - div.style.cssText = divReset + "width:1px;padding:1px;display:inline;zoom:1"; - support.inlineBlockNeedsLayout = ( div.offsetWidth === 3 ); - - // Support: IE6 - // Check if elements with layout shrink-wrap their children - div.style.display = "block"; - div.innerHTML = "
"; - div.firstChild.style.width = "5px"; - support.shrinkWrapBlocks = ( div.offsetWidth !== 3 ); - - if ( support.inlineBlockNeedsLayout ) { - // Prevent IE 6 from affecting layout for positioned elements #11048 - // Prevent IE from shrinking the body in IE 7 mode #12869 - // Support: IE<8 - body.style.zoom = 1; - } - } - - body.removeChild( container ); - - // Null elements to avoid leaks in IE - container = div = tds = marginDiv = null; - }); - - // Null elements to avoid leaks in IE - all = select = fragment = opt = a = input = null; - - return support; -})({}); - -var rbrace = /(?:\{[\s\S]*\}|\[[\s\S]*\])$/, - rmultiDash = /([A-Z])/g; - -function internalData( elem, name, data, pvt /* Internal Use Only */ ){ - if ( !jQuery.acceptData( elem ) ) { - return; + // If there are functions bound, to execute + readyList.resolveWith( document, [ jQuery ] ); } +} ); - var ret, thisCache, - internalKey = jQuery.expando, +jQuery.ready.then = readyList.then; - // We have to handle DOM nodes and JS objects differently because IE6-7 - // can't GC object references properly across the DOM-JS boundary - isNode = elem.nodeType, - - // Only DOM nodes need the global jQuery cache; JS object data is - // attached directly to the object so GC can occur automatically - cache = isNode ? jQuery.cache : elem, - - // Only defining an ID for JS objects if its cache already exists allows - // the code to shortcut on the same path as a DOM node with no cache - id = isNode ? elem[ internalKey ] : elem[ internalKey ] && internalKey; - - // Avoid doing any more work than we need to when trying to get data on an - // object that has no data at all - if ( (!id || !cache[id] || (!pvt && !cache[id].data)) && data === undefined && typeof name === "string" ) { - return; - } - - if ( !id ) { - // Only DOM nodes need a new unique ID for each element since their data - // ends up in the global cache - if ( isNode ) { - id = elem[ internalKey ] = core_deletedIds.pop() || jQuery.guid++; - } else { - id = internalKey; - } - } - - if ( !cache[ id ] ) { - // Avoid exposing jQuery metadata on plain JS objects when the object - // is serialized using JSON.stringify - cache[ id ] = isNode ? {} : { toJSON: jQuery.noop }; - } - - // An object can be passed to jQuery.data instead of a key/value pair; this gets - // shallow copied over onto the existing cache - if ( typeof name === "object" || typeof name === "function" ) { - if ( pvt ) { - cache[ id ] = jQuery.extend( cache[ id ], name ); - } else { - cache[ id ].data = jQuery.extend( cache[ id ].data, name ); - } - } - - thisCache = cache[ id ]; - - // jQuery data() is stored in a separate object inside the object's internal data - // cache in order to avoid key collisions between internal data and user-defined - // data. - if ( !pvt ) { - if ( !thisCache.data ) { - thisCache.data = {}; - } - - thisCache = thisCache.data; - } - - if ( data !== undefined ) { - thisCache[ jQuery.camelCase( name ) ] = data; - } - - // Check for both converted-to-camel and non-converted data property names - // If a data property was specified - if ( typeof name === "string" ) { - - // First Try to find as-is property data - ret = thisCache[ name ]; - - // Test for null|undefined property data - if ( ret == null ) { - - // Try to find the camelCased property - ret = thisCache[ jQuery.camelCase( name ) ]; - } - } else { - ret = thisCache; - } - - return ret; +// The ready event handler and self cleanup method +function completed() { + document.removeEventListener( "DOMContentLoaded", completed ); + window.removeEventListener( "load", completed ); + jQuery.ready(); } -function internalRemoveData( elem, name, pvt ) { - if ( !jQuery.acceptData( elem ) ) { - return; - } +// Catch cases where $(document).ready() is called +// after the browser event has already occurred. +// Support: IE <=9 - 10 only +// Older IE sometimes signals "interactive" too soon +if ( document.readyState === "complete" || + ( document.readyState !== "loading" && !document.documentElement.doScroll ) ) { - var thisCache, i, - isNode = elem.nodeType, + // Handle it asynchronously to allow scripts the opportunity to delay ready + window.setTimeout( jQuery.ready ); - // See jQuery.data for more information - cache = isNode ? jQuery.cache : elem, - id = isNode ? elem[ jQuery.expando ] : jQuery.expando; +} else { - // If there is already no cache entry for this object, there is no - // purpose in continuing - if ( !cache[ id ] ) { - return; - } + // Use the handy event callback + document.addEventListener( "DOMContentLoaded", completed ); - if ( name ) { + // A fallback to window.onload, that will always work + window.addEventListener( "load", completed ); +} - thisCache = pvt ? cache[ id ] : cache[ id ].data; - if ( thisCache ) { - // Support array or space separated string names for data keys - if ( !jQuery.isArray( name ) ) { - // try the string as a key before any manipulation - if ( name in thisCache ) { - name = [ name ]; - } else { +// Multifunctional method to get and set values of a collection +// The value/s can optionally be executed if it's a function +var access = function( elems, fn, key, value, chainable, emptyGet, raw ) { + var i = 0, + len = elems.length, + bulk = key == null; - // split the camel cased version by spaces unless a key with the spaces exists - name = jQuery.camelCase( name ); - if ( name in thisCache ) { - name = [ name ]; - } else { - name = name.split(" "); - } - } + // Sets many values + if ( toType( key ) === "object" ) { + chainable = true; + for ( i in key ) { + access( elems, fn, i, key[ i ], true, emptyGet, raw ); + } + + // Sets one value + } else if ( value !== undefined ) { + chainable = true; + + if ( !isFunction( value ) ) { + raw = true; + } + + if ( bulk ) { + + // Bulk operations run against the entire set + if ( raw ) { + fn.call( elems, value ); + fn = null; + + // ...except when executing function values } else { - // If "name" is an array of keys... - // When data is initially created, via ("key", "val") signature, - // keys will be converted to camelCase. - // Since there is no way to tell _how_ a key was added, remove - // both plain key and camelCase key. #12786 - // This will only penalize the array argument path. - name = name.concat( jQuery.map( name, jQuery.camelCase ) ); + bulk = fn; + fn = function( elem, key, value ) { + return bulk.call( jQuery( elem ), value ); + }; } + } - i = name.length; - while ( i-- ) { - delete thisCache[ name[i] ]; - } - - // If there is no data left in the cache, we want to continue - // and let the cache object itself get destroyed - if ( pvt ? !isEmptyDataObject(thisCache) : !jQuery.isEmptyObject(thisCache) ) { - return; + if ( fn ) { + for ( ; i < len; i++ ) { + fn( + elems[ i ], key, raw ? + value : + value.call( elems[ i ], i, fn( elems[ i ], key ) ) + ); } } } - // See jQuery.data for more information - if ( !pvt ) { - delete cache[ id ].data; - - // Don't destroy the parent cache unless the internal data object - // had been the only thing left in it - if ( !isEmptyDataObject( cache[ id ] ) ) { - return; - } + if ( chainable ) { + return elems; } - // Destroy the cache - if ( isNode ) { - jQuery.cleanData( [ elem ], true ); - - // Use delete when supported for expandos or `cache` is not a window per isWindow (#10080) - /* jshint eqeqeq: false */ - } else if ( jQuery.support.deleteExpando || cache != cache.window ) { - /* jshint eqeqeq: true */ - delete cache[ id ]; - - // When all else fails, null - } else { - cache[ id ] = null; + // Gets + if ( bulk ) { + return fn.call( elems ); } + + return len ? fn( elems[ 0 ], key ) : emptyGet; +}; + + +// Matches dashed string for camelizing +var rmsPrefix = /^-ms-/, + rdashAlpha = /-([a-z])/g; + +// Used by camelCase as callback to replace() +function fcamelCase( all, letter ) { + return letter.toUpperCase(); } -jQuery.extend({ - cache: {}, +// Convert dashed to camelCase; used by the css and data modules +// Support: IE <=9 - 11, Edge 12 - 15 +// Microsoft forgot to hump their vendor prefix (#9572) +function camelCase( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); +} +var acceptData = function( owner ) { - // The following elements throw uncatchable exceptions if you - // attempt to add expando properties to them. - noData: { - "applet": true, - "embed": true, - // Ban all objects except for Flash (which handle expandos) - "object": "clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" + // Accepts only: + // - Node + // - Node.ELEMENT_NODE + // - Node.DOCUMENT_NODE + // - Object + // - Any + return owner.nodeType === 1 || owner.nodeType === 9 || !( +owner.nodeType ); +}; + + + + +function Data() { + this.expando = jQuery.expando + Data.uid++; +} + +Data.uid = 1; + +Data.prototype = { + + cache: function( owner ) { + + // Check if the owner object already has a cache + var value = owner[ this.expando ]; + + // If not, create one + if ( !value ) { + value = {}; + + // We can accept data for non-element nodes in modern browsers, + // but we should not, see #8335. + // Always return an empty object. + if ( acceptData( owner ) ) { + + // If it is a node unlikely to be stringify-ed or looped over + // use plain assignment + if ( owner.nodeType ) { + owner[ this.expando ] = value; + + // Otherwise secure it in a non-enumerable property + // configurable must be true to allow the property to be + // deleted when data is removed + } else { + Object.defineProperty( owner, this.expando, { + value: value, + configurable: true + } ); + } + } + } + + return value; }, + set: function( owner, data, value ) { + var prop, + cache = this.cache( owner ); + // Handle: [ owner, key, value ] args + // Always use camelCase key (gh-2257) + if ( typeof data === "string" ) { + cache[ camelCase( data ) ] = value; + + // Handle: [ owner, { properties } ] args + } else { + + // Copy the properties one-by-one to the cache object + for ( prop in data ) { + cache[ camelCase( prop ) ] = data[ prop ]; + } + } + return cache; + }, + get: function( owner, key ) { + return key === undefined ? + this.cache( owner ) : + + // Always use camelCase key (gh-2257) + owner[ this.expando ] && owner[ this.expando ][ camelCase( key ) ]; + }, + access: function( owner, key, value ) { + + // In cases where either: + // + // 1. No key was specified + // 2. A string key was specified, but no value provided + // + // Take the "read" path and allow the get method to determine + // which value to return, respectively either: + // + // 1. The entire cache object + // 2. The data stored at the key + // + if ( key === undefined || + ( ( key && typeof key === "string" ) && value === undefined ) ) { + + return this.get( owner, key ); + } + + // When the key is not a string, or both a key and value + // are specified, set or extend (existing objects) with either: + // + // 1. An object of properties + // 2. A key and value + // + this.set( owner, key, value ); + + // Since the "set" path can have two possible entry points + // return the expected data based on which path was taken[*] + return value !== undefined ? value : key; + }, + remove: function( owner, key ) { + var i, + cache = owner[ this.expando ]; + + if ( cache === undefined ) { + return; + } + + if ( key !== undefined ) { + + // Support array or space separated string of keys + if ( Array.isArray( key ) ) { + + // If key is an array of keys... + // We always set camelCase keys, so remove that. + key = key.map( camelCase ); + } else { + key = camelCase( key ); + + // If a key with the spaces exists, use it. + // Otherwise, create an array by matching non-whitespace + key = key in cache ? + [ key ] : + ( key.match( rnothtmlwhite ) || [] ); + } + + i = key.length; + + while ( i-- ) { + delete cache[ key[ i ] ]; + } + } + + // Remove the expando if there's no more data + if ( key === undefined || jQuery.isEmptyObject( cache ) ) { + + // Support: Chrome <=35 - 45 + // Webkit & Blink performance suffers when deleting properties + // from DOM nodes, so set to undefined instead + // https://bugs.chromium.org/p/chromium/issues/detail?id=378607 (bug restricted) + if ( owner.nodeType ) { + owner[ this.expando ] = undefined; + } else { + delete owner[ this.expando ]; + } + } + }, + hasData: function( owner ) { + var cache = owner[ this.expando ]; + return cache !== undefined && !jQuery.isEmptyObject( cache ); + } +}; +var dataPriv = new Data(); + +var dataUser = new Data(); + + + +// Implementation Summary +// +// 1. Enforce API surface and semantic compatibility with 1.9.x branch +// 2. Improve the module's maintainability by reducing the storage +// paths to a single mechanism. +// 3. Use the same single mechanism to support "private" and "user" data. +// 4. _Never_ expose "private" data to user code (TODO: Drop _data, _removeData) +// 5. Avoid exposing implementation details on user objects (eg. expando properties) +// 6. Provide a clear path for implementation upgrade to WeakMap in 2014 + +var rbrace = /^(?:\{[\w\W]*\}|\[[\w\W]*\])$/, + rmultiDash = /[A-Z]/g; + +function getData( data ) { + if ( data === "true" ) { + return true; + } + + if ( data === "false" ) { + return false; + } + + if ( data === "null" ) { + return null; + } + + // Only convert to a number if it doesn't change the string + if ( data === +data + "" ) { + return +data; + } + + if ( rbrace.test( data ) ) { + return JSON.parse( data ); + } + + return data; +} + +function dataAttr( elem, key, data ) { + var name; + + // If nothing was found internally, try to fetch any + // data from the HTML5 data-* attribute + if ( data === undefined && elem.nodeType === 1 ) { + name = "data-" + key.replace( rmultiDash, "-$&" ).toLowerCase(); + data = elem.getAttribute( name ); + + if ( typeof data === "string" ) { + try { + data = getData( data ); + } catch ( e ) {} + + // Make sure we set the data so it isn't changed later + dataUser.set( elem, key, data ); + } else { + data = undefined; + } + } + return data; +} + +jQuery.extend( { hasData: function( elem ) { - elem = elem.nodeType ? jQuery.cache[ elem[jQuery.expando] ] : elem[ jQuery.expando ]; - return !!elem && !isEmptyDataObject( elem ); + return dataUser.hasData( elem ) || dataPriv.hasData( elem ); }, data: function( elem, name, data ) { - return internalData( elem, name, data ); + return dataUser.access( elem, name, data ); }, removeData: function( elem, name ) { - return internalRemoveData( elem, name ); + dataUser.remove( elem, name ); }, - // For internal use only. + // TODO: Now that all calls to _data and _removeData have been replaced + // with direct calls to dataPriv methods, these can be deprecated. _data: function( elem, name, data ) { - return internalData( elem, name, data, true ); + return dataPriv.access( elem, name, data ); }, _removeData: function( elem, name ) { - return internalRemoveData( elem, name, true ); - }, - - // A method for determining if a DOM node can handle the data expando - acceptData: function( elem ) { - // Do not set data on non-element because it will not be cleared (#8335). - if ( elem.nodeType && elem.nodeType !== 1 && elem.nodeType !== 9 ) { - return false; - } - - var noData = elem.nodeName && jQuery.noData[ elem.nodeName.toLowerCase() ]; - - // nodes accept data unless otherwise specified; rejection can be conditional - return !noData || noData !== true && elem.getAttribute("classid") === noData; + dataPriv.remove( elem, name ); } -}); +} ); -jQuery.fn.extend({ +jQuery.fn.extend( { data: function( key, value ) { - var attrs, name, - data = null, - i = 0, - elem = this[0]; - - // Special expections of .data basically thwart jQuery.access, - // so implement the relevant behavior ourselves + var i, name, data, + elem = this[ 0 ], + attrs = elem && elem.attributes; // Gets all values if ( key === undefined ) { if ( this.length ) { - data = jQuery.data( elem ); + data = dataUser.get( elem ); - if ( elem.nodeType === 1 && !jQuery._data( elem, "parsedAttrs" ) ) { - attrs = elem.attributes; - for ( ; i < attrs.length; i++ ) { - name = attrs[i].name; + if ( elem.nodeType === 1 && !dataPriv.get( elem, "hasDataAttrs" ) ) { + i = attrs.length; + while ( i-- ) { - if ( name.indexOf("data-") === 0 ) { - name = jQuery.camelCase( name.slice(5) ); - - dataAttr( elem, name, data[ name ] ); + // Support: IE 11 only + // The attrs elements can be null (#14894) + if ( attrs[ i ] ) { + name = attrs[ i ].name; + if ( name.indexOf( "data-" ) === 0 ) { + name = camelCase( name.slice( 5 ) ); + dataAttr( elem, name, data[ name ] ); + } } } - jQuery._data( elem, "parsedAttrs", true ); + dataPriv.set( elem, "hasDataAttrs", true ); } } @@ -3827,89 +4276,68 @@ jQuery.fn.extend({ // Sets multiple values if ( typeof key === "object" ) { - return this.each(function() { - jQuery.data( this, key ); - }); + return this.each( function() { + dataUser.set( this, key ); + } ); } - return arguments.length > 1 ? + return access( this, function( value ) { + var data; - // Sets one value - this.each(function() { - jQuery.data( this, key, value ); - }) : + // The calling jQuery object (element matches) is not empty + // (and therefore has an element appears at this[ 0 ]) and the + // `value` parameter was not undefined. An empty jQuery object + // will result in `undefined` for elem = this[ 0 ] which will + // throw an exception if an attempt to read a data cache is made. + if ( elem && value === undefined ) { - // Gets one value - // Try to fetch any internally stored data first - elem ? dataAttr( elem, key, jQuery.data( elem, key ) ) : null; + // Attempt to get data from the cache + // The key will always be camelCased in Data + data = dataUser.get( elem, key ); + if ( data !== undefined ) { + return data; + } + + // Attempt to "discover" the data in + // HTML5 custom data-* attrs + data = dataAttr( elem, key ); + if ( data !== undefined ) { + return data; + } + + // We tried really hard, but the data doesn't exist. + return; + } + + // Set the data... + this.each( function() { + + // We always store the camelCased key + dataUser.set( this, key, value ); + } ); + }, null, value, arguments.length > 1, null, true ); }, removeData: function( key ) { - return this.each(function() { - jQuery.removeData( this, key ); - }); + return this.each( function() { + dataUser.remove( this, key ); + } ); } -}); +} ); -function dataAttr( elem, key, data ) { - // If nothing was found internally, try to fetch any - // data from the HTML5 data-* attribute - if ( data === undefined && elem.nodeType === 1 ) { - var name = "data-" + key.replace( rmultiDash, "-$1" ).toLowerCase(); - - data = elem.getAttribute( name ); - - if ( typeof data === "string" ) { - try { - data = data === "true" ? true : - data === "false" ? false : - data === "null" ? null : - // Only convert to a number if it doesn't change the string - +data + "" === data ? +data : - rbrace.test( data ) ? jQuery.parseJSON( data ) : - data; - } catch( e ) {} - - // Make sure we set the data so it isn't changed later - jQuery.data( elem, key, data ); - - } else { - data = undefined; - } - } - - return data; -} - -// checks a cache object for emptiness -function isEmptyDataObject( obj ) { - var name; - for ( name in obj ) { - - // if the public data object is empty, the private is still empty - if ( name === "data" && jQuery.isEmptyObject( obj[name] ) ) { - continue; - } - if ( name !== "toJSON" ) { - return false; - } - } - - return true; -} -jQuery.extend({ +jQuery.extend( { queue: function( elem, type, data ) { var queue; if ( elem ) { type = ( type || "fx" ) + "queue"; - queue = jQuery._data( elem, type ); + queue = dataPriv.get( elem, type ); // Speed up dequeue by getting out quickly if this is just a lookup if ( data ) { - if ( !queue || jQuery.isArray(data) ) { - queue = jQuery._data( elem, type, jQuery.makeArray(data) ); + if ( !queue || Array.isArray( data ) ) { + queue = dataPriv.access( elem, type, jQuery.makeArray( data ) ); } else { queue.push( data ); } @@ -3943,7 +4371,7 @@ jQuery.extend({ queue.unshift( "inprogress" ); } - // clear up the last queue stop function + // Clear up the last queue stop function delete hooks.stop; fn.call( elem, next, hooks ); } @@ -3953,19 +4381,18 @@ jQuery.extend({ } }, - // not intended for public consumption - generates a queueHooks object, or returns the current one + // Not public - generate a queueHooks object, or return the current one _queueHooks: function( elem, type ) { var key = type + "queueHooks"; - return jQuery._data( elem, key ) || jQuery._data( elem, key, { - empty: jQuery.Callbacks("once memory").add(function() { - jQuery._removeData( elem, type + "queue" ); - jQuery._removeData( elem, key ); - }) - }); + return dataPriv.get( elem, key ) || dataPriv.access( elem, key, { + empty: jQuery.Callbacks( "once memory" ).add( function() { + dataPriv.remove( elem, [ type + "queue", key ] ); + } ) + } ); } -}); +} ); -jQuery.fn.extend({ +jQuery.fn.extend( { queue: function( type, data ) { var setter = 2; @@ -3976,43 +4403,31 @@ jQuery.fn.extend({ } if ( arguments.length < setter ) { - return jQuery.queue( this[0], type ); + return jQuery.queue( this[ 0 ], type ); } return data === undefined ? this : - this.each(function() { + this.each( function() { var queue = jQuery.queue( this, type, data ); - // ensure a hooks for this queue + // Ensure a hooks for this queue jQuery._queueHooks( this, type ); - if ( type === "fx" && queue[0] !== "inprogress" ) { + if ( type === "fx" && queue[ 0 ] !== "inprogress" ) { jQuery.dequeue( this, type ); } - }); + } ); }, dequeue: function( type ) { - return this.each(function() { + return this.each( function() { jQuery.dequeue( this, type ); - }); - }, - // Based off of the plugin by Clint Helfers, with permission. - // http://blindsignals.com/index.php/2009/07/jquery-delay/ - delay: function( time, type ) { - time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; - type = type || "fx"; - - return this.queue( type, function( next, hooks ) { - var timeout = setTimeout( next, time ); - hooks.stop = function() { - clearTimeout( timeout ); - }; - }); + } ); }, clearQueue: function( type ) { return this.queue( type || "fx", [] ); }, + // Get a promise resolved when queues of a certain type // are emptied (fx is the type by default) promise: function( type, obj ) { @@ -4033,8 +4448,8 @@ jQuery.fn.extend({ } type = type || "fx"; - while( i-- ) { - tmp = jQuery._data( elements[ i ], type + "queueHooks" ); + while ( i-- ) { + tmp = dataPriv.get( elements[ i ], type + "queueHooks" ); if ( tmp && tmp.empty ) { count++; tmp.empty.add( resolve ); @@ -4043,74 +4458,3380 @@ jQuery.fn.extend({ resolve(); return defer.promise( obj ); } -}); -var nodeHook, boolHook, - rclass = /[\t\r\n\f]/g, - rreturn = /\r/g, - rfocusable = /^(?:input|select|textarea|button|object)$/i, - rclickable = /^(?:a|area)$/i, - ruseDefault = /^(?:checked|selected)$/i, - getSetAttribute = jQuery.support.getSetAttribute, - getSetInput = jQuery.support.input; +} ); +var pnum = ( /[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/ ).source; -jQuery.fn.extend({ +var rcssNum = new RegExp( "^(?:([+-])=|)(" + pnum + ")([a-z%]*)$", "i" ); + + +var cssExpand = [ "Top", "Right", "Bottom", "Left" ]; + +var isHiddenWithinTree = function( elem, el ) { + + // isHiddenWithinTree might be called from jQuery#filter function; + // in that case, element will be second argument + elem = el || elem; + + // Inline style trumps all + return elem.style.display === "none" || + elem.style.display === "" && + + // Otherwise, check computed style + // Support: Firefox <=43 - 45 + // Disconnected elements can have computed display: none, so first confirm that elem is + // in the document. + jQuery.contains( elem.ownerDocument, elem ) && + + jQuery.css( elem, "display" ) === "none"; + }; + +var swap = function( elem, options, callback, args ) { + var ret, name, + old = {}; + + // Remember the old values, and insert the new ones + for ( name in options ) { + old[ name ] = elem.style[ name ]; + elem.style[ name ] = options[ name ]; + } + + ret = callback.apply( elem, args || [] ); + + // Revert the old values + for ( name in options ) { + elem.style[ name ] = old[ name ]; + } + + return ret; +}; + + + + +function adjustCSS( elem, prop, valueParts, tween ) { + var adjusted, scale, + maxIterations = 20, + currentValue = tween ? + function() { + return tween.cur(); + } : + function() { + return jQuery.css( elem, prop, "" ); + }, + initial = currentValue(), + unit = valueParts && valueParts[ 3 ] || ( jQuery.cssNumber[ prop ] ? "" : "px" ), + + // Starting value computation is required for potential unit mismatches + initialInUnit = ( jQuery.cssNumber[ prop ] || unit !== "px" && +initial ) && + rcssNum.exec( jQuery.css( elem, prop ) ); + + if ( initialInUnit && initialInUnit[ 3 ] !== unit ) { + + // Support: Firefox <=54 + // Halve the iteration target value to prevent interference from CSS upper bounds (gh-2144) + initial = initial / 2; + + // Trust units reported by jQuery.css + unit = unit || initialInUnit[ 3 ]; + + // Iteratively approximate from a nonzero starting point + initialInUnit = +initial || 1; + + while ( maxIterations-- ) { + + // Evaluate and update our best guess (doubling guesses that zero out). + // Finish if the scale equals or crosses 1 (making the old*new product non-positive). + jQuery.style( elem, prop, initialInUnit + unit ); + if ( ( 1 - scale ) * ( 1 - ( scale = currentValue() / initial || 0.5 ) ) <= 0 ) { + maxIterations = 0; + } + initialInUnit = initialInUnit / scale; + + } + + initialInUnit = initialInUnit * 2; + jQuery.style( elem, prop, initialInUnit + unit ); + + // Make sure we update the tween properties later on + valueParts = valueParts || []; + } + + if ( valueParts ) { + initialInUnit = +initialInUnit || +initial || 0; + + // Apply relative offset (+=/-=) if specified + adjusted = valueParts[ 1 ] ? + initialInUnit + ( valueParts[ 1 ] + 1 ) * valueParts[ 2 ] : + +valueParts[ 2 ]; + if ( tween ) { + tween.unit = unit; + tween.start = initialInUnit; + tween.end = adjusted; + } + } + return adjusted; +} + + +var defaultDisplayMap = {}; + +function getDefaultDisplay( elem ) { + var temp, + doc = elem.ownerDocument, + nodeName = elem.nodeName, + display = defaultDisplayMap[ nodeName ]; + + if ( display ) { + return display; + } + + temp = doc.body.appendChild( doc.createElement( nodeName ) ); + display = jQuery.css( temp, "display" ); + + temp.parentNode.removeChild( temp ); + + if ( display === "none" ) { + display = "block"; + } + defaultDisplayMap[ nodeName ] = display; + + return display; +} + +function showHide( elements, show ) { + var display, elem, + values = [], + index = 0, + length = elements.length; + + // Determine new display value for elements that need to change + for ( ; index < length; index++ ) { + elem = elements[ index ]; + if ( !elem.style ) { + continue; + } + + display = elem.style.display; + if ( show ) { + + // Since we force visibility upon cascade-hidden elements, an immediate (and slow) + // check is required in this first loop unless we have a nonempty display value (either + // inline or about-to-be-restored) + if ( display === "none" ) { + values[ index ] = dataPriv.get( elem, "display" ) || null; + if ( !values[ index ] ) { + elem.style.display = ""; + } + } + if ( elem.style.display === "" && isHiddenWithinTree( elem ) ) { + values[ index ] = getDefaultDisplay( elem ); + } + } else { + if ( display !== "none" ) { + values[ index ] = "none"; + + // Remember what we're overwriting + dataPriv.set( elem, "display", display ); + } + } + } + + // Set the display of the elements in a second loop to avoid constant reflow + for ( index = 0; index < length; index++ ) { + if ( values[ index ] != null ) { + elements[ index ].style.display = values[ index ]; + } + } + + return elements; +} + +jQuery.fn.extend( { + show: function() { + return showHide( this, true ); + }, + hide: function() { + return showHide( this ); + }, + toggle: function( state ) { + if ( typeof state === "boolean" ) { + return state ? this.show() : this.hide(); + } + + return this.each( function() { + if ( isHiddenWithinTree( this ) ) { + jQuery( this ).show(); + } else { + jQuery( this ).hide(); + } + } ); + } +} ); +var rcheckableType = ( /^(?:checkbox|radio)$/i ); + +var rtagName = ( /<([a-z][^\/\0>\x20\t\r\n\f]+)/i ); + +var rscriptType = ( /^$|^module$|\/(?:java|ecma)script/i ); + + + +// We have to close these tags to support XHTML (#13200) +var wrapMap = { + + // Support: IE <=9 only + option: [ 1, "" ], + + // XHTML parsers do not magically insert elements in the + // same way that tag soup parsers do. So we cannot shorten + // this by omitting or other required elements. + thead: [ 1, "", "
" ], + col: [ 2, "", "
" ], + tr: [ 2, "", "
" ], + td: [ 3, "", "
" ], + + _default: [ 0, "", "" ] +}; + +// Support: IE <=9 only +wrapMap.optgroup = wrapMap.option; + +wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; +wrapMap.th = wrapMap.td; + + +function getAll( context, tag ) { + + // Support: IE <=9 - 11 only + // Use typeof to avoid zero-argument method invocation on host objects (#15151) + var ret; + + if ( typeof context.getElementsByTagName !== "undefined" ) { + ret = context.getElementsByTagName( tag || "*" ); + + } else if ( typeof context.querySelectorAll !== "undefined" ) { + ret = context.querySelectorAll( tag || "*" ); + + } else { + ret = []; + } + + if ( tag === undefined || tag && nodeName( context, tag ) ) { + return jQuery.merge( [ context ], ret ); + } + + return ret; +} + + +// Mark scripts as having already been evaluated +function setGlobalEval( elems, refElements ) { + var i = 0, + l = elems.length; + + for ( ; i < l; i++ ) { + dataPriv.set( + elems[ i ], + "globalEval", + !refElements || dataPriv.get( refElements[ i ], "globalEval" ) + ); + } +} + + +var rhtml = /<|&#?\w+;/; + +function buildFragment( elems, context, scripts, selection, ignored ) { + var elem, tmp, tag, wrap, contains, j, + fragment = context.createDocumentFragment(), + nodes = [], + i = 0, + l = elems.length; + + for ( ; i < l; i++ ) { + elem = elems[ i ]; + + if ( elem || elem === 0 ) { + + // Add nodes directly + if ( toType( elem ) === "object" ) { + + // Support: Android <=4.0 only, PhantomJS 1 only + // push.apply(_, arraylike) throws on ancient WebKit + jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); + + // Convert non-html into a text node + } else if ( !rhtml.test( elem ) ) { + nodes.push( context.createTextNode( elem ) ); + + // Convert html into DOM nodes + } else { + tmp = tmp || fragment.appendChild( context.createElement( "div" ) ); + + // Deserialize a standard representation + tag = ( rtagName.exec( elem ) || [ "", "" ] )[ 1 ].toLowerCase(); + wrap = wrapMap[ tag ] || wrapMap._default; + tmp.innerHTML = wrap[ 1 ] + jQuery.htmlPrefilter( elem ) + wrap[ 2 ]; + + // Descend through wrappers to the right content + j = wrap[ 0 ]; + while ( j-- ) { + tmp = tmp.lastChild; + } + + // Support: Android <=4.0 only, PhantomJS 1 only + // push.apply(_, arraylike) throws on ancient WebKit + jQuery.merge( nodes, tmp.childNodes ); + + // Remember the top-level container + tmp = fragment.firstChild; + + // Ensure the created nodes are orphaned (#12392) + tmp.textContent = ""; + } + } + } + + // Remove wrapper from fragment + fragment.textContent = ""; + + i = 0; + while ( ( elem = nodes[ i++ ] ) ) { + + // Skip elements already in the context collection (trac-4087) + if ( selection && jQuery.inArray( elem, selection ) > -1 ) { + if ( ignored ) { + ignored.push( elem ); + } + continue; + } + + contains = jQuery.contains( elem.ownerDocument, elem ); + + // Append to fragment + tmp = getAll( fragment.appendChild( elem ), "script" ); + + // Preserve script evaluation history + if ( contains ) { + setGlobalEval( tmp ); + } + + // Capture executables + if ( scripts ) { + j = 0; + while ( ( elem = tmp[ j++ ] ) ) { + if ( rscriptType.test( elem.type || "" ) ) { + scripts.push( elem ); + } + } + } + } + + return fragment; +} + + +( function() { + var fragment = document.createDocumentFragment(), + div = fragment.appendChild( document.createElement( "div" ) ), + input = document.createElement( "input" ); + + // Support: Android 4.0 - 4.3 only + // Check state lost if the name is set (#11217) + // Support: Windows Web Apps (WWA) + // `name` and `type` must use .setAttribute for WWA (#14901) + input.setAttribute( "type", "radio" ); + input.setAttribute( "checked", "checked" ); + input.setAttribute( "name", "t" ); + + div.appendChild( input ); + + // Support: Android <=4.1 only + // Older WebKit doesn't clone checked state correctly in fragments + support.checkClone = div.cloneNode( true ).cloneNode( true ).lastChild.checked; + + // Support: IE <=11 only + // Make sure textarea (and checkbox) defaultValue is properly cloned + div.innerHTML = ""; + support.noCloneChecked = !!div.cloneNode( true ).lastChild.defaultValue; +} )(); +var documentElement = document.documentElement; + + + +var + rkeyEvent = /^key/, + rmouseEvent = /^(?:mouse|pointer|contextmenu|drag|drop)|click/, + rtypenamespace = /^([^.]*)(?:\.(.+)|)/; + +function returnTrue() { + return true; +} + +function returnFalse() { + return false; +} + +// Support: IE <=9 only +// See #13393 for more info +function safeActiveElement() { + try { + return document.activeElement; + } catch ( err ) { } +} + +function on( elem, types, selector, data, fn, one ) { + var origFn, type; + + // Types can be a map of types/handlers + if ( typeof types === "object" ) { + + // ( types-Object, selector, data ) + if ( typeof selector !== "string" ) { + + // ( types-Object, data ) + data = data || selector; + selector = undefined; + } + for ( type in types ) { + on( elem, type, selector, data, types[ type ], one ); + } + return elem; + } + + if ( data == null && fn == null ) { + + // ( types, fn ) + fn = selector; + data = selector = undefined; + } else if ( fn == null ) { + if ( typeof selector === "string" ) { + + // ( types, selector, fn ) + fn = data; + data = undefined; + } else { + + // ( types, data, fn ) + fn = data; + data = selector; + selector = undefined; + } + } + if ( fn === false ) { + fn = returnFalse; + } else if ( !fn ) { + return elem; + } + + if ( one === 1 ) { + origFn = fn; + fn = function( event ) { + + // Can use an empty set, since event contains the info + jQuery().off( event ); + return origFn.apply( this, arguments ); + }; + + // Use same guid so caller can remove using origFn + fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); + } + return elem.each( function() { + jQuery.event.add( this, types, fn, data, selector ); + } ); +} + +/* + * Helper functions for managing events -- not part of the public interface. + * Props to Dean Edwards' addEvent library for many of the ideas. + */ +jQuery.event = { + + global: {}, + + add: function( elem, types, handler, data, selector ) { + + var handleObjIn, eventHandle, tmp, + events, t, handleObj, + special, handlers, type, namespaces, origType, + elemData = dataPriv.get( elem ); + + // Don't attach events to noData or text/comment nodes (but allow plain objects) + if ( !elemData ) { + return; + } + + // Caller can pass in an object of custom data in lieu of the handler + if ( handler.handler ) { + handleObjIn = handler; + handler = handleObjIn.handler; + selector = handleObjIn.selector; + } + + // Ensure that invalid selectors throw exceptions at attach time + // Evaluate against documentElement in case elem is a non-element node (e.g., document) + if ( selector ) { + jQuery.find.matchesSelector( documentElement, selector ); + } + + // Make sure that the handler has a unique ID, used to find/remove it later + if ( !handler.guid ) { + handler.guid = jQuery.guid++; + } + + // Init the element's event structure and main handler, if this is the first + if ( !( events = elemData.events ) ) { + events = elemData.events = {}; + } + if ( !( eventHandle = elemData.handle ) ) { + eventHandle = elemData.handle = function( e ) { + + // Discard the second event of a jQuery.event.trigger() and + // when an event is called after a page has unloaded + return typeof jQuery !== "undefined" && jQuery.event.triggered !== e.type ? + jQuery.event.dispatch.apply( elem, arguments ) : undefined; + }; + } + + // Handle multiple events separated by a space + types = ( types || "" ).match( rnothtmlwhite ) || [ "" ]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[ t ] ) || []; + type = origType = tmp[ 1 ]; + namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); + + // There *must* be a type, no attaching namespace-only handlers + if ( !type ) { + continue; + } + + // If event changes its type, use the special event handlers for the changed type + special = jQuery.event.special[ type ] || {}; + + // If selector defined, determine special event api type, otherwise given type + type = ( selector ? special.delegateType : special.bindType ) || type; + + // Update special based on newly reset type + special = jQuery.event.special[ type ] || {}; + + // handleObj is passed to all event handlers + handleObj = jQuery.extend( { + type: type, + origType: origType, + data: data, + handler: handler, + guid: handler.guid, + selector: selector, + needsContext: selector && jQuery.expr.match.needsContext.test( selector ), + namespace: namespaces.join( "." ) + }, handleObjIn ); + + // Init the event handler queue if we're the first + if ( !( handlers = events[ type ] ) ) { + handlers = events[ type ] = []; + handlers.delegateCount = 0; + + // Only use addEventListener if the special events handler returns false + if ( !special.setup || + special.setup.call( elem, data, namespaces, eventHandle ) === false ) { + + if ( elem.addEventListener ) { + elem.addEventListener( type, eventHandle ); + } + } + } + + if ( special.add ) { + special.add.call( elem, handleObj ); + + if ( !handleObj.handler.guid ) { + handleObj.handler.guid = handler.guid; + } + } + + // Add to the element's handler list, delegates in front + if ( selector ) { + handlers.splice( handlers.delegateCount++, 0, handleObj ); + } else { + handlers.push( handleObj ); + } + + // Keep track of which events have ever been used, for event optimization + jQuery.event.global[ type ] = true; + } + + }, + + // Detach an event or set of events from an element + remove: function( elem, types, handler, selector, mappedTypes ) { + + var j, origCount, tmp, + events, t, handleObj, + special, handlers, type, namespaces, origType, + elemData = dataPriv.hasData( elem ) && dataPriv.get( elem ); + + if ( !elemData || !( events = elemData.events ) ) { + return; + } + + // Once for each type.namespace in types; type may be omitted + types = ( types || "" ).match( rnothtmlwhite ) || [ "" ]; + t = types.length; + while ( t-- ) { + tmp = rtypenamespace.exec( types[ t ] ) || []; + type = origType = tmp[ 1 ]; + namespaces = ( tmp[ 2 ] || "" ).split( "." ).sort(); + + // Unbind all events (on this namespace, if provided) for the element + if ( !type ) { + for ( type in events ) { + jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); + } + continue; + } + + special = jQuery.event.special[ type ] || {}; + type = ( selector ? special.delegateType : special.bindType ) || type; + handlers = events[ type ] || []; + tmp = tmp[ 2 ] && + new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ); + + // Remove matching events + origCount = j = handlers.length; + while ( j-- ) { + handleObj = handlers[ j ]; + + if ( ( mappedTypes || origType === handleObj.origType ) && + ( !handler || handler.guid === handleObj.guid ) && + ( !tmp || tmp.test( handleObj.namespace ) ) && + ( !selector || selector === handleObj.selector || + selector === "**" && handleObj.selector ) ) { + handlers.splice( j, 1 ); + + if ( handleObj.selector ) { + handlers.delegateCount--; + } + if ( special.remove ) { + special.remove.call( elem, handleObj ); + } + } + } + + // Remove generic event handler if we removed something and no more handlers exist + // (avoids potential for endless recursion during removal of special event handlers) + if ( origCount && !handlers.length ) { + if ( !special.teardown || + special.teardown.call( elem, namespaces, elemData.handle ) === false ) { + + jQuery.removeEvent( elem, type, elemData.handle ); + } + + delete events[ type ]; + } + } + + // Remove data and the expando if it's no longer used + if ( jQuery.isEmptyObject( events ) ) { + dataPriv.remove( elem, "handle events" ); + } + }, + + dispatch: function( nativeEvent ) { + + // Make a writable jQuery.Event from the native event object + var event = jQuery.event.fix( nativeEvent ); + + var i, j, ret, matched, handleObj, handlerQueue, + args = new Array( arguments.length ), + handlers = ( dataPriv.get( this, "events" ) || {} )[ event.type ] || [], + special = jQuery.event.special[ event.type ] || {}; + + // Use the fix-ed jQuery.Event rather than the (read-only) native event + args[ 0 ] = event; + + for ( i = 1; i < arguments.length; i++ ) { + args[ i ] = arguments[ i ]; + } + + event.delegateTarget = this; + + // Call the preDispatch hook for the mapped type, and let it bail if desired + if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { + return; + } + + // Determine handlers + handlerQueue = jQuery.event.handlers.call( this, event, handlers ); + + // Run delegates first; they may want to stop propagation beneath us + i = 0; + while ( ( matched = handlerQueue[ i++ ] ) && !event.isPropagationStopped() ) { + event.currentTarget = matched.elem; + + j = 0; + while ( ( handleObj = matched.handlers[ j++ ] ) && + !event.isImmediatePropagationStopped() ) { + + // Triggered event must either 1) have no namespace, or 2) have namespace(s) + // a subset or equal to those in the bound event (both can have no namespace). + if ( !event.rnamespace || event.rnamespace.test( handleObj.namespace ) ) { + + event.handleObj = handleObj; + event.data = handleObj.data; + + ret = ( ( jQuery.event.special[ handleObj.origType ] || {} ).handle || + handleObj.handler ).apply( matched.elem, args ); + + if ( ret !== undefined ) { + if ( ( event.result = ret ) === false ) { + event.preventDefault(); + event.stopPropagation(); + } + } + } + } + } + + // Call the postDispatch hook for the mapped type + if ( special.postDispatch ) { + special.postDispatch.call( this, event ); + } + + return event.result; + }, + + handlers: function( event, handlers ) { + var i, handleObj, sel, matchedHandlers, matchedSelectors, + handlerQueue = [], + delegateCount = handlers.delegateCount, + cur = event.target; + + // Find delegate handlers + if ( delegateCount && + + // Support: IE <=9 + // Black-hole SVG instance trees (trac-13180) + cur.nodeType && + + // Support: Firefox <=42 + // Suppress spec-violating clicks indicating a non-primary pointer button (trac-3861) + // https://www.w3.org/TR/DOM-Level-3-Events/#event-type-click + // Support: IE 11 only + // ...but not arrow key "clicks" of radio inputs, which can have `button` -1 (gh-2343) + !( event.type === "click" && event.button >= 1 ) ) { + + for ( ; cur !== this; cur = cur.parentNode || this ) { + + // Don't check non-elements (#13208) + // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) + if ( cur.nodeType === 1 && !( event.type === "click" && cur.disabled === true ) ) { + matchedHandlers = []; + matchedSelectors = {}; + for ( i = 0; i < delegateCount; i++ ) { + handleObj = handlers[ i ]; + + // Don't conflict with Object.prototype properties (#13203) + sel = handleObj.selector + " "; + + if ( matchedSelectors[ sel ] === undefined ) { + matchedSelectors[ sel ] = handleObj.needsContext ? + jQuery( sel, this ).index( cur ) > -1 : + jQuery.find( sel, this, null, [ cur ] ).length; + } + if ( matchedSelectors[ sel ] ) { + matchedHandlers.push( handleObj ); + } + } + if ( matchedHandlers.length ) { + handlerQueue.push( { elem: cur, handlers: matchedHandlers } ); + } + } + } + } + + // Add the remaining (directly-bound) handlers + cur = this; + if ( delegateCount < handlers.length ) { + handlerQueue.push( { elem: cur, handlers: handlers.slice( delegateCount ) } ); + } + + return handlerQueue; + }, + + addProp: function( name, hook ) { + Object.defineProperty( jQuery.Event.prototype, name, { + enumerable: true, + configurable: true, + + get: isFunction( hook ) ? + function() { + if ( this.originalEvent ) { + return hook( this.originalEvent ); + } + } : + function() { + if ( this.originalEvent ) { + return this.originalEvent[ name ]; + } + }, + + set: function( value ) { + Object.defineProperty( this, name, { + enumerable: true, + configurable: true, + writable: true, + value: value + } ); + } + } ); + }, + + fix: function( originalEvent ) { + return originalEvent[ jQuery.expando ] ? + originalEvent : + new jQuery.Event( originalEvent ); + }, + + special: { + load: { + + // Prevent triggered image.load events from bubbling to window.load + noBubble: true + }, + focus: { + + // Fire native event if possible so blur/focus sequence is correct + trigger: function() { + if ( this !== safeActiveElement() && this.focus ) { + this.focus(); + return false; + } + }, + delegateType: "focusin" + }, + blur: { + trigger: function() { + if ( this === safeActiveElement() && this.blur ) { + this.blur(); + return false; + } + }, + delegateType: "focusout" + }, + click: { + + // For checkbox, fire native event so checked state will be right + trigger: function() { + if ( this.type === "checkbox" && this.click && nodeName( this, "input" ) ) { + this.click(); + return false; + } + }, + + // For cross-browser consistency, don't fire native .click() on links + _default: function( event ) { + return nodeName( event.target, "a" ); + } + }, + + beforeunload: { + postDispatch: function( event ) { + + // Support: Firefox 20+ + // Firefox doesn't alert if the returnValue field is not set. + if ( event.result !== undefined && event.originalEvent ) { + event.originalEvent.returnValue = event.result; + } + } + } + } +}; + +jQuery.removeEvent = function( elem, type, handle ) { + + // This "if" is needed for plain objects + if ( elem.removeEventListener ) { + elem.removeEventListener( type, handle ); + } +}; + +jQuery.Event = function( src, props ) { + + // Allow instantiation without the 'new' keyword + if ( !( this instanceof jQuery.Event ) ) { + return new jQuery.Event( src, props ); + } + + // Event object + if ( src && src.type ) { + this.originalEvent = src; + this.type = src.type; + + // Events bubbling up the document may have been marked as prevented + // by a handler lower down the tree; reflect the correct value. + this.isDefaultPrevented = src.defaultPrevented || + src.defaultPrevented === undefined && + + // Support: Android <=2.3 only + src.returnValue === false ? + returnTrue : + returnFalse; + + // Create target properties + // Support: Safari <=6 - 7 only + // Target should not be a text node (#504, #13143) + this.target = ( src.target && src.target.nodeType === 3 ) ? + src.target.parentNode : + src.target; + + this.currentTarget = src.currentTarget; + this.relatedTarget = src.relatedTarget; + + // Event type + } else { + this.type = src; + } + + // Put explicitly provided properties onto the event object + if ( props ) { + jQuery.extend( this, props ); + } + + // Create a timestamp if incoming event doesn't have one + this.timeStamp = src && src.timeStamp || Date.now(); + + // Mark it as fixed + this[ jQuery.expando ] = true; +}; + +// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding +// https://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html +jQuery.Event.prototype = { + constructor: jQuery.Event, + isDefaultPrevented: returnFalse, + isPropagationStopped: returnFalse, + isImmediatePropagationStopped: returnFalse, + isSimulated: false, + + preventDefault: function() { + var e = this.originalEvent; + + this.isDefaultPrevented = returnTrue; + + if ( e && !this.isSimulated ) { + e.preventDefault(); + } + }, + stopPropagation: function() { + var e = this.originalEvent; + + this.isPropagationStopped = returnTrue; + + if ( e && !this.isSimulated ) { + e.stopPropagation(); + } + }, + stopImmediatePropagation: function() { + var e = this.originalEvent; + + this.isImmediatePropagationStopped = returnTrue; + + if ( e && !this.isSimulated ) { + e.stopImmediatePropagation(); + } + + this.stopPropagation(); + } +}; + +// Includes all common event props including KeyEvent and MouseEvent specific props +jQuery.each( { + altKey: true, + bubbles: true, + cancelable: true, + changedTouches: true, + ctrlKey: true, + detail: true, + eventPhase: true, + metaKey: true, + pageX: true, + pageY: true, + shiftKey: true, + view: true, + "char": true, + charCode: true, + key: true, + keyCode: true, + button: true, + buttons: true, + clientX: true, + clientY: true, + offsetX: true, + offsetY: true, + pointerId: true, + pointerType: true, + screenX: true, + screenY: true, + targetTouches: true, + toElement: true, + touches: true, + + which: function( event ) { + var button = event.button; + + // Add which for key events + if ( event.which == null && rkeyEvent.test( event.type ) ) { + return event.charCode != null ? event.charCode : event.keyCode; + } + + // Add which for click: 1 === left; 2 === middle; 3 === right + if ( !event.which && button !== undefined && rmouseEvent.test( event.type ) ) { + if ( button & 1 ) { + return 1; + } + + if ( button & 2 ) { + return 3; + } + + if ( button & 4 ) { + return 2; + } + + return 0; + } + + return event.which; + } +}, jQuery.event.addProp ); + +// Create mouseenter/leave events using mouseover/out and event-time checks +// so that event delegation works in jQuery. +// Do the same for pointerenter/pointerleave and pointerover/pointerout +// +// Support: Safari 7 only +// Safari sends mouseenter too often; see: +// https://bugs.chromium.org/p/chromium/issues/detail?id=470258 +// for the description of the bug (it existed in older Chrome versions as well). +jQuery.each( { + mouseenter: "mouseover", + mouseleave: "mouseout", + pointerenter: "pointerover", + pointerleave: "pointerout" +}, function( orig, fix ) { + jQuery.event.special[ orig ] = { + delegateType: fix, + bindType: fix, + + handle: function( event ) { + var ret, + target = this, + related = event.relatedTarget, + handleObj = event.handleObj; + + // For mouseenter/leave call the handler if related is outside the target. + // NB: No relatedTarget if the mouse left/entered the browser window + if ( !related || ( related !== target && !jQuery.contains( target, related ) ) ) { + event.type = handleObj.origType; + ret = handleObj.handler.apply( this, arguments ); + event.type = fix; + } + return ret; + } + }; +} ); + +jQuery.fn.extend( { + + on: function( types, selector, data, fn ) { + return on( this, types, selector, data, fn ); + }, + one: function( types, selector, data, fn ) { + return on( this, types, selector, data, fn, 1 ); + }, + off: function( types, selector, fn ) { + var handleObj, type; + if ( types && types.preventDefault && types.handleObj ) { + + // ( event ) dispatched jQuery.Event + handleObj = types.handleObj; + jQuery( types.delegateTarget ).off( + handleObj.namespace ? + handleObj.origType + "." + handleObj.namespace : + handleObj.origType, + handleObj.selector, + handleObj.handler + ); + return this; + } + if ( typeof types === "object" ) { + + // ( types-object [, selector] ) + for ( type in types ) { + this.off( type, selector, types[ type ] ); + } + return this; + } + if ( selector === false || typeof selector === "function" ) { + + // ( types [, fn] ) + fn = selector; + selector = undefined; + } + if ( fn === false ) { + fn = returnFalse; + } + return this.each( function() { + jQuery.event.remove( this, types, fn, selector ); + } ); + } +} ); + + +var + + /* eslint-disable max-len */ + + // See https://github.com/eslint/eslint/issues/3229 + rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([a-z][^\/\0>\x20\t\r\n\f]*)[^>]*)\/>/gi, + + /* eslint-enable */ + + // Support: IE <=10 - 11, Edge 12 - 13 only + // In IE/Edge using regex groups here causes severe slowdowns. + // See https://connect.microsoft.com/IE/feedback/details/1736512/ + rnoInnerhtml = /\s*$/g; + +// Prefer a tbody over its parent table for containing new rows +function manipulationTarget( elem, content ) { + if ( nodeName( elem, "table" ) && + nodeName( content.nodeType !== 11 ? content : content.firstChild, "tr" ) ) { + + return jQuery( elem ).children( "tbody" )[ 0 ] || elem; + } + + return elem; +} + +// Replace/restore the type attribute of script elements for safe DOM manipulation +function disableScript( elem ) { + elem.type = ( elem.getAttribute( "type" ) !== null ) + "/" + elem.type; + return elem; +} +function restoreScript( elem ) { + if ( ( elem.type || "" ).slice( 0, 5 ) === "true/" ) { + elem.type = elem.type.slice( 5 ); + } else { + elem.removeAttribute( "type" ); + } + + return elem; +} + +function cloneCopyEvent( src, dest ) { + var i, l, type, pdataOld, pdataCur, udataOld, udataCur, events; + + if ( dest.nodeType !== 1 ) { + return; + } + + // 1. Copy private data: events, handlers, etc. + if ( dataPriv.hasData( src ) ) { + pdataOld = dataPriv.access( src ); + pdataCur = dataPriv.set( dest, pdataOld ); + events = pdataOld.events; + + if ( events ) { + delete pdataCur.handle; + pdataCur.events = {}; + + for ( type in events ) { + for ( i = 0, l = events[ type ].length; i < l; i++ ) { + jQuery.event.add( dest, type, events[ type ][ i ] ); + } + } + } + } + + // 2. Copy user data + if ( dataUser.hasData( src ) ) { + udataOld = dataUser.access( src ); + udataCur = jQuery.extend( {}, udataOld ); + + dataUser.set( dest, udataCur ); + } +} + +// Fix IE bugs, see support tests +function fixInput( src, dest ) { + var nodeName = dest.nodeName.toLowerCase(); + + // Fails to persist the checked state of a cloned checkbox or radio button. + if ( nodeName === "input" && rcheckableType.test( src.type ) ) { + dest.checked = src.checked; + + // Fails to return the selected option to the default selected state when cloning options + } else if ( nodeName === "input" || nodeName === "textarea" ) { + dest.defaultValue = src.defaultValue; + } +} + +function domManip( collection, args, callback, ignored ) { + + // Flatten any nested arrays + args = concat.apply( [], args ); + + var fragment, first, scripts, hasScripts, node, doc, + i = 0, + l = collection.length, + iNoClone = l - 1, + value = args[ 0 ], + valueIsFunction = isFunction( value ); + + // We can't cloneNode fragments that contain checked, in WebKit + if ( valueIsFunction || + ( l > 1 && typeof value === "string" && + !support.checkClone && rchecked.test( value ) ) ) { + return collection.each( function( index ) { + var self = collection.eq( index ); + if ( valueIsFunction ) { + args[ 0 ] = value.call( this, index, self.html() ); + } + domManip( self, args, callback, ignored ); + } ); + } + + if ( l ) { + fragment = buildFragment( args, collection[ 0 ].ownerDocument, false, collection, ignored ); + first = fragment.firstChild; + + if ( fragment.childNodes.length === 1 ) { + fragment = first; + } + + // Require either new content or an interest in ignored elements to invoke the callback + if ( first || ignored ) { + scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); + hasScripts = scripts.length; + + // Use the original fragment for the last item + // instead of the first because it can end up + // being emptied incorrectly in certain situations (#8070). + for ( ; i < l; i++ ) { + node = fragment; + + if ( i !== iNoClone ) { + node = jQuery.clone( node, true, true ); + + // Keep references to cloned scripts for later restoration + if ( hasScripts ) { + + // Support: Android <=4.0 only, PhantomJS 1 only + // push.apply(_, arraylike) throws on ancient WebKit + jQuery.merge( scripts, getAll( node, "script" ) ); + } + } + + callback.call( collection[ i ], node, i ); + } + + if ( hasScripts ) { + doc = scripts[ scripts.length - 1 ].ownerDocument; + + // Reenable scripts + jQuery.map( scripts, restoreScript ); + + // Evaluate executable scripts on first document insertion + for ( i = 0; i < hasScripts; i++ ) { + node = scripts[ i ]; + if ( rscriptType.test( node.type || "" ) && + !dataPriv.access( node, "globalEval" ) && + jQuery.contains( doc, node ) ) { + + if ( node.src && ( node.type || "" ).toLowerCase() !== "module" ) { + + // Optional AJAX dependency, but won't run scripts if not present + if ( jQuery._evalUrl ) { + jQuery._evalUrl( node.src ); + } + } else { + DOMEval( node.textContent.replace( rcleanScript, "" ), doc, node ); + } + } + } + } + } + } + + return collection; +} + +function remove( elem, selector, keepData ) { + var node, + nodes = selector ? jQuery.filter( selector, elem ) : elem, + i = 0; + + for ( ; ( node = nodes[ i ] ) != null; i++ ) { + if ( !keepData && node.nodeType === 1 ) { + jQuery.cleanData( getAll( node ) ); + } + + if ( node.parentNode ) { + if ( keepData && jQuery.contains( node.ownerDocument, node ) ) { + setGlobalEval( getAll( node, "script" ) ); + } + node.parentNode.removeChild( node ); + } + } + + return elem; +} + +jQuery.extend( { + htmlPrefilter: function( html ) { + return html.replace( rxhtmlTag, "<$1>" ); + }, + + clone: function( elem, dataAndEvents, deepDataAndEvents ) { + var i, l, srcElements, destElements, + clone = elem.cloneNode( true ), + inPage = jQuery.contains( elem.ownerDocument, elem ); + + // Fix IE cloning issues + if ( !support.noCloneChecked && ( elem.nodeType === 1 || elem.nodeType === 11 ) && + !jQuery.isXMLDoc( elem ) ) { + + // We eschew Sizzle here for performance reasons: https://jsperf.com/getall-vs-sizzle/2 + destElements = getAll( clone ); + srcElements = getAll( elem ); + + for ( i = 0, l = srcElements.length; i < l; i++ ) { + fixInput( srcElements[ i ], destElements[ i ] ); + } + } + + // Copy the events from the original to the clone + if ( dataAndEvents ) { + if ( deepDataAndEvents ) { + srcElements = srcElements || getAll( elem ); + destElements = destElements || getAll( clone ); + + for ( i = 0, l = srcElements.length; i < l; i++ ) { + cloneCopyEvent( srcElements[ i ], destElements[ i ] ); + } + } else { + cloneCopyEvent( elem, clone ); + } + } + + // Preserve script evaluation history + destElements = getAll( clone, "script" ); + if ( destElements.length > 0 ) { + setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); + } + + // Return the cloned set + return clone; + }, + + cleanData: function( elems ) { + var data, elem, type, + special = jQuery.event.special, + i = 0; + + for ( ; ( elem = elems[ i ] ) !== undefined; i++ ) { + if ( acceptData( elem ) ) { + if ( ( data = elem[ dataPriv.expando ] ) ) { + if ( data.events ) { + for ( type in data.events ) { + if ( special[ type ] ) { + jQuery.event.remove( elem, type ); + + // This is a shortcut to avoid jQuery.event.remove's overhead + } else { + jQuery.removeEvent( elem, type, data.handle ); + } + } + } + + // Support: Chrome <=35 - 45+ + // Assign undefined instead of using delete, see Data#remove + elem[ dataPriv.expando ] = undefined; + } + if ( elem[ dataUser.expando ] ) { + + // Support: Chrome <=35 - 45+ + // Assign undefined instead of using delete, see Data#remove + elem[ dataUser.expando ] = undefined; + } + } + } + } +} ); + +jQuery.fn.extend( { + detach: function( selector ) { + return remove( this, selector, true ); + }, + + remove: function( selector ) { + return remove( this, selector ); + }, + + text: function( value ) { + return access( this, function( value ) { + return value === undefined ? + jQuery.text( this ) : + this.empty().each( function() { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + this.textContent = value; + } + } ); + }, null, value, arguments.length ); + }, + + append: function() { + return domManip( this, arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.appendChild( elem ); + } + } ); + }, + + prepend: function() { + return domManip( this, arguments, function( elem ) { + if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { + var target = manipulationTarget( this, elem ); + target.insertBefore( elem, target.firstChild ); + } + } ); + }, + + before: function() { + return domManip( this, arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this ); + } + } ); + }, + + after: function() { + return domManip( this, arguments, function( elem ) { + if ( this.parentNode ) { + this.parentNode.insertBefore( elem, this.nextSibling ); + } + } ); + }, + + empty: function() { + var elem, + i = 0; + + for ( ; ( elem = this[ i ] ) != null; i++ ) { + if ( elem.nodeType === 1 ) { + + // Prevent memory leaks + jQuery.cleanData( getAll( elem, false ) ); + + // Remove any remaining nodes + elem.textContent = ""; + } + } + + return this; + }, + + clone: function( dataAndEvents, deepDataAndEvents ) { + dataAndEvents = dataAndEvents == null ? false : dataAndEvents; + deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; + + return this.map( function() { + return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); + } ); + }, + + html: function( value ) { + return access( this, function( value ) { + var elem = this[ 0 ] || {}, + i = 0, + l = this.length; + + if ( value === undefined && elem.nodeType === 1 ) { + return elem.innerHTML; + } + + // See if we can take a shortcut and just use innerHTML + if ( typeof value === "string" && !rnoInnerhtml.test( value ) && + !wrapMap[ ( rtagName.exec( value ) || [ "", "" ] )[ 1 ].toLowerCase() ] ) { + + value = jQuery.htmlPrefilter( value ); + + try { + for ( ; i < l; i++ ) { + elem = this[ i ] || {}; + + // Remove element nodes and prevent memory leaks + if ( elem.nodeType === 1 ) { + jQuery.cleanData( getAll( elem, false ) ); + elem.innerHTML = value; + } + } + + elem = 0; + + // If using innerHTML throws an exception, use the fallback method + } catch ( e ) {} + } + + if ( elem ) { + this.empty().append( value ); + } + }, null, value, arguments.length ); + }, + + replaceWith: function() { + var ignored = []; + + // Make the changes, replacing each non-ignored context element with the new content + return domManip( this, arguments, function( elem ) { + var parent = this.parentNode; + + if ( jQuery.inArray( this, ignored ) < 0 ) { + jQuery.cleanData( getAll( this ) ); + if ( parent ) { + parent.replaceChild( elem, this ); + } + } + + // Force callback invocation + }, ignored ); + } +} ); + +jQuery.each( { + appendTo: "append", + prependTo: "prepend", + insertBefore: "before", + insertAfter: "after", + replaceAll: "replaceWith" +}, function( name, original ) { + jQuery.fn[ name ] = function( selector ) { + var elems, + ret = [], + insert = jQuery( selector ), + last = insert.length - 1, + i = 0; + + for ( ; i <= last; i++ ) { + elems = i === last ? this : this.clone( true ); + jQuery( insert[ i ] )[ original ]( elems ); + + // Support: Android <=4.0 only, PhantomJS 1 only + // .get() because push.apply(_, arraylike) throws on ancient WebKit + push.apply( ret, elems.get() ); + } + + return this.pushStack( ret ); + }; +} ); +var rnumnonpx = new RegExp( "^(" + pnum + ")(?!px)[a-z%]+$", "i" ); + +var getStyles = function( elem ) { + + // Support: IE <=11 only, Firefox <=30 (#15098, #14150) + // IE throws on elements created in popups + // FF meanwhile throws on frame elements through "defaultView.getComputedStyle" + var view = elem.ownerDocument.defaultView; + + if ( !view || !view.opener ) { + view = window; + } + + return view.getComputedStyle( elem ); + }; + +var rboxStyle = new RegExp( cssExpand.join( "|" ), "i" ); + + + +( function() { + + // Executing both pixelPosition & boxSizingReliable tests require only one layout + // so they're executed at the same time to save the second computation. + function computeStyleTests() { + + // This is a singleton, we need to execute it only once + if ( !div ) { + return; + } + + container.style.cssText = "position:absolute;left:-11111px;width:60px;" + + "margin-top:1px;padding:0;border:0"; + div.style.cssText = + "position:relative;display:block;box-sizing:border-box;overflow:scroll;" + + "margin:auto;border:1px;padding:1px;" + + "width:60%;top:1%"; + documentElement.appendChild( container ).appendChild( div ); + + var divStyle = window.getComputedStyle( div ); + pixelPositionVal = divStyle.top !== "1%"; + + // Support: Android 4.0 - 4.3 only, Firefox <=3 - 44 + reliableMarginLeftVal = roundPixelMeasures( divStyle.marginLeft ) === 12; + + // Support: Android 4.0 - 4.3 only, Safari <=9.1 - 10.1, iOS <=7.0 - 9.3 + // Some styles come back with percentage values, even though they shouldn't + div.style.right = "60%"; + pixelBoxStylesVal = roundPixelMeasures( divStyle.right ) === 36; + + // Support: IE 9 - 11 only + // Detect misreporting of content dimensions for box-sizing:border-box elements + boxSizingReliableVal = roundPixelMeasures( divStyle.width ) === 36; + + // Support: IE 9 only + // Detect overflow:scroll screwiness (gh-3699) + div.style.position = "absolute"; + scrollboxSizeVal = div.offsetWidth === 36 || "absolute"; + + documentElement.removeChild( container ); + + // Nullify the div so it wouldn't be stored in the memory and + // it will also be a sign that checks already performed + div = null; + } + + function roundPixelMeasures( measure ) { + return Math.round( parseFloat( measure ) ); + } + + var pixelPositionVal, boxSizingReliableVal, scrollboxSizeVal, pixelBoxStylesVal, + reliableMarginLeftVal, + container = document.createElement( "div" ), + div = document.createElement( "div" ); + + // Finish early in limited (non-browser) environments + if ( !div.style ) { + return; + } + + // Support: IE <=9 - 11 only + // Style of cloned element affects source element cloned (#8908) + div.style.backgroundClip = "content-box"; + div.cloneNode( true ).style.backgroundClip = ""; + support.clearCloneStyle = div.style.backgroundClip === "content-box"; + + jQuery.extend( support, { + boxSizingReliable: function() { + computeStyleTests(); + return boxSizingReliableVal; + }, + pixelBoxStyles: function() { + computeStyleTests(); + return pixelBoxStylesVal; + }, + pixelPosition: function() { + computeStyleTests(); + return pixelPositionVal; + }, + reliableMarginLeft: function() { + computeStyleTests(); + return reliableMarginLeftVal; + }, + scrollboxSize: function() { + computeStyleTests(); + return scrollboxSizeVal; + } + } ); +} )(); + + +function curCSS( elem, name, computed ) { + var width, minWidth, maxWidth, ret, + + // Support: Firefox 51+ + // Retrieving style before computed somehow + // fixes an issue with getting wrong values + // on detached elements + style = elem.style; + + computed = computed || getStyles( elem ); + + // getPropertyValue is needed for: + // .css('filter') (IE 9 only, #12537) + // .css('--customProperty) (#3144) + if ( computed ) { + ret = computed.getPropertyValue( name ) || computed[ name ]; + + if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) { + ret = jQuery.style( elem, name ); + } + + // A tribute to the "awesome hack by Dean Edwards" + // Android Browser returns percentage for some values, + // but width seems to be reliably pixels. + // This is against the CSSOM draft spec: + // https://drafts.csswg.org/cssom/#resolved-values + if ( !support.pixelBoxStyles() && rnumnonpx.test( ret ) && rboxStyle.test( name ) ) { + + // Remember the original values + width = style.width; + minWidth = style.minWidth; + maxWidth = style.maxWidth; + + // Put in the new values to get a computed value out + style.minWidth = style.maxWidth = style.width = ret; + ret = computed.width; + + // Revert the changed values + style.width = width; + style.minWidth = minWidth; + style.maxWidth = maxWidth; + } + } + + return ret !== undefined ? + + // Support: IE <=9 - 11 only + // IE returns zIndex value as an integer. + ret + "" : + ret; +} + + +function addGetHookIf( conditionFn, hookFn ) { + + // Define the hook, we'll check on the first run if it's really needed. + return { + get: function() { + if ( conditionFn() ) { + + // Hook not needed (or it's not possible to use it due + // to missing dependency), remove it. + delete this.get; + return; + } + + // Hook needed; redefine it so that the support test is not executed again. + return ( this.get = hookFn ).apply( this, arguments ); + } + }; +} + + +var + + // Swappable if display is none or starts with table + // except "table", "table-cell", or "table-caption" + // See here for display values: https://developer.mozilla.org/en-US/docs/CSS/display + rdisplayswap = /^(none|table(?!-c[ea]).+)/, + rcustomProp = /^--/, + cssShow = { position: "absolute", visibility: "hidden", display: "block" }, + cssNormalTransform = { + letterSpacing: "0", + fontWeight: "400" + }, + + cssPrefixes = [ "Webkit", "Moz", "ms" ], + emptyStyle = document.createElement( "div" ).style; + +// Return a css property mapped to a potentially vendor prefixed property +function vendorPropName( name ) { + + // Shortcut for names that are not vendor prefixed + if ( name in emptyStyle ) { + return name; + } + + // Check for vendor prefixed names + var capName = name[ 0 ].toUpperCase() + name.slice( 1 ), + i = cssPrefixes.length; + + while ( i-- ) { + name = cssPrefixes[ i ] + capName; + if ( name in emptyStyle ) { + return name; + } + } +} + +// Return a property mapped along what jQuery.cssProps suggests or to +// a vendor prefixed property. +function finalPropName( name ) { + var ret = jQuery.cssProps[ name ]; + if ( !ret ) { + ret = jQuery.cssProps[ name ] = vendorPropName( name ) || name; + } + return ret; +} + +function setPositiveNumber( elem, value, subtract ) { + + // Any relative (+/-) values have already been + // normalized at this point + var matches = rcssNum.exec( value ); + return matches ? + + // Guard against undefined "subtract", e.g., when used as in cssHooks + Math.max( 0, matches[ 2 ] - ( subtract || 0 ) ) + ( matches[ 3 ] || "px" ) : + value; +} + +function boxModelAdjustment( elem, dimension, box, isBorderBox, styles, computedVal ) { + var i = dimension === "width" ? 1 : 0, + extra = 0, + delta = 0; + + // Adjustment may not be necessary + if ( box === ( isBorderBox ? "border" : "content" ) ) { + return 0; + } + + for ( ; i < 4; i += 2 ) { + + // Both box models exclude margin + if ( box === "margin" ) { + delta += jQuery.css( elem, box + cssExpand[ i ], true, styles ); + } + + // If we get here with a content-box, we're seeking "padding" or "border" or "margin" + if ( !isBorderBox ) { + + // Add padding + delta += jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); + + // For "border" or "margin", add border + if ( box !== "padding" ) { + delta += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); + + // But still keep track of it otherwise + } else { + extra += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); + } + + // If we get here with a border-box (content + padding + border), we're seeking "content" or + // "padding" or "margin" + } else { + + // For "content", subtract padding + if ( box === "content" ) { + delta -= jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); + } + + // For "content" or "padding", subtract border + if ( box !== "margin" ) { + delta -= jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); + } + } + } + + // Account for positive content-box scroll gutter when requested by providing computedVal + if ( !isBorderBox && computedVal >= 0 ) { + + // offsetWidth/offsetHeight is a rounded sum of content, padding, scroll gutter, and border + // Assuming integer scroll gutter, subtract the rest and round down + delta += Math.max( 0, Math.ceil( + elem[ "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ) ] - + computedVal - + delta - + extra - + 0.5 + ) ); + } + + return delta; +} + +function getWidthOrHeight( elem, dimension, extra ) { + + // Start with computed style + var styles = getStyles( elem ), + val = curCSS( elem, dimension, styles ), + isBorderBox = jQuery.css( elem, "boxSizing", false, styles ) === "border-box", + valueIsBorderBox = isBorderBox; + + // Support: Firefox <=54 + // Return a confounding non-pixel value or feign ignorance, as appropriate. + if ( rnumnonpx.test( val ) ) { + if ( !extra ) { + return val; + } + val = "auto"; + } + + // Check for style in case a browser which returns unreliable values + // for getComputedStyle silently falls back to the reliable elem.style + valueIsBorderBox = valueIsBorderBox && + ( support.boxSizingReliable() || val === elem.style[ dimension ] ); + + // Fall back to offsetWidth/offsetHeight when value is "auto" + // This happens for inline elements with no explicit setting (gh-3571) + // Support: Android <=4.1 - 4.3 only + // Also use offsetWidth/offsetHeight for misreported inline dimensions (gh-3602) + if ( val === "auto" || + !parseFloat( val ) && jQuery.css( elem, "display", false, styles ) === "inline" ) { + + val = elem[ "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ) ]; + + // offsetWidth/offsetHeight provide border-box values + valueIsBorderBox = true; + } + + // Normalize "" and auto + val = parseFloat( val ) || 0; + + // Adjust for the element's box model + return ( val + + boxModelAdjustment( + elem, + dimension, + extra || ( isBorderBox ? "border" : "content" ), + valueIsBorderBox, + styles, + + // Provide the current computed size to request scroll gutter calculation (gh-3589) + val + ) + ) + "px"; +} + +jQuery.extend( { + + // Add in style property hooks for overriding the default + // behavior of getting and setting a style property + cssHooks: { + opacity: { + get: function( elem, computed ) { + if ( computed ) { + + // We should always get a number back from opacity + var ret = curCSS( elem, "opacity" ); + return ret === "" ? "1" : ret; + } + } + } + }, + + // Don't automatically add "px" to these possibly-unitless properties + cssNumber: { + "animationIterationCount": true, + "columnCount": true, + "fillOpacity": true, + "flexGrow": true, + "flexShrink": true, + "fontWeight": true, + "lineHeight": true, + "opacity": true, + "order": true, + "orphans": true, + "widows": true, + "zIndex": true, + "zoom": true + }, + + // Add in properties whose names you wish to fix before + // setting or getting the value + cssProps: {}, + + // Get and set the style property on a DOM Node + style: function( elem, name, value, extra ) { + + // Don't set styles on text and comment nodes + if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { + return; + } + + // Make sure that we're working with the right name + var ret, type, hooks, + origName = camelCase( name ), + isCustomProp = rcustomProp.test( name ), + style = elem.style; + + // Make sure that we're working with the right name. We don't + // want to query the value if it is a CSS custom property + // since they are user-defined. + if ( !isCustomProp ) { + name = finalPropName( origName ); + } + + // Gets hook for the prefixed version, then unprefixed version + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // Check if we're setting a value + if ( value !== undefined ) { + type = typeof value; + + // Convert "+=" or "-=" to relative numbers (#7345) + if ( type === "string" && ( ret = rcssNum.exec( value ) ) && ret[ 1 ] ) { + value = adjustCSS( elem, name, ret ); + + // Fixes bug #9237 + type = "number"; + } + + // Make sure that null and NaN values aren't set (#7116) + if ( value == null || value !== value ) { + return; + } + + // If a number was passed in, add the unit (except for certain CSS properties) + if ( type === "number" ) { + value += ret && ret[ 3 ] || ( jQuery.cssNumber[ origName ] ? "" : "px" ); + } + + // background-* props affect original clone's values + if ( !support.clearCloneStyle && value === "" && name.indexOf( "background" ) === 0 ) { + style[ name ] = "inherit"; + } + + // If a hook was provided, use that value, otherwise just set the specified value + if ( !hooks || !( "set" in hooks ) || + ( value = hooks.set( elem, value, extra ) ) !== undefined ) { + + if ( isCustomProp ) { + style.setProperty( name, value ); + } else { + style[ name ] = value; + } + } + + } else { + + // If a hook was provided get the non-computed value from there + if ( hooks && "get" in hooks && + ( ret = hooks.get( elem, false, extra ) ) !== undefined ) { + + return ret; + } + + // Otherwise just get the value from the style object + return style[ name ]; + } + }, + + css: function( elem, name, extra, styles ) { + var val, num, hooks, + origName = camelCase( name ), + isCustomProp = rcustomProp.test( name ); + + // Make sure that we're working with the right name. We don't + // want to modify the value if it is a CSS custom property + // since they are user-defined. + if ( !isCustomProp ) { + name = finalPropName( origName ); + } + + // Try prefixed name followed by the unprefixed name + hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; + + // If a hook was provided get the computed value from there + if ( hooks && "get" in hooks ) { + val = hooks.get( elem, true, extra ); + } + + // Otherwise, if a way to get the computed value exists, use that + if ( val === undefined ) { + val = curCSS( elem, name, styles ); + } + + // Convert "normal" to computed value + if ( val === "normal" && name in cssNormalTransform ) { + val = cssNormalTransform[ name ]; + } + + // Make numeric if forced or a qualifier was provided and val looks numeric + if ( extra === "" || extra ) { + num = parseFloat( val ); + return extra === true || isFinite( num ) ? num || 0 : val; + } + + return val; + } +} ); + +jQuery.each( [ "height", "width" ], function( i, dimension ) { + jQuery.cssHooks[ dimension ] = { + get: function( elem, computed, extra ) { + if ( computed ) { + + // Certain elements can have dimension info if we invisibly show them + // but it must have a current display style that would benefit + return rdisplayswap.test( jQuery.css( elem, "display" ) ) && + + // Support: Safari 8+ + // Table columns in Safari have non-zero offsetWidth & zero + // getBoundingClientRect().width unless display is changed. + // Support: IE <=11 only + // Running getBoundingClientRect on a disconnected node + // in IE throws an error. + ( !elem.getClientRects().length || !elem.getBoundingClientRect().width ) ? + swap( elem, cssShow, function() { + return getWidthOrHeight( elem, dimension, extra ); + } ) : + getWidthOrHeight( elem, dimension, extra ); + } + }, + + set: function( elem, value, extra ) { + var matches, + styles = getStyles( elem ), + isBorderBox = jQuery.css( elem, "boxSizing", false, styles ) === "border-box", + subtract = extra && boxModelAdjustment( + elem, + dimension, + extra, + isBorderBox, + styles + ); + + // Account for unreliable border-box dimensions by comparing offset* to computed and + // faking a content-box to get border and padding (gh-3699) + if ( isBorderBox && support.scrollboxSize() === styles.position ) { + subtract -= Math.ceil( + elem[ "offset" + dimension[ 0 ].toUpperCase() + dimension.slice( 1 ) ] - + parseFloat( styles[ dimension ] ) - + boxModelAdjustment( elem, dimension, "border", false, styles ) - + 0.5 + ); + } + + // Convert to pixels if value adjustment is needed + if ( subtract && ( matches = rcssNum.exec( value ) ) && + ( matches[ 3 ] || "px" ) !== "px" ) { + + elem.style[ dimension ] = value; + value = jQuery.css( elem, dimension ); + } + + return setPositiveNumber( elem, value, subtract ); + } + }; +} ); + +jQuery.cssHooks.marginLeft = addGetHookIf( support.reliableMarginLeft, + function( elem, computed ) { + if ( computed ) { + return ( parseFloat( curCSS( elem, "marginLeft" ) ) || + elem.getBoundingClientRect().left - + swap( elem, { marginLeft: 0 }, function() { + return elem.getBoundingClientRect().left; + } ) + ) + "px"; + } + } +); + +// These hooks are used by animate to expand properties +jQuery.each( { + margin: "", + padding: "", + border: "Width" +}, function( prefix, suffix ) { + jQuery.cssHooks[ prefix + suffix ] = { + expand: function( value ) { + var i = 0, + expanded = {}, + + // Assumes a single number if not a string + parts = typeof value === "string" ? value.split( " " ) : [ value ]; + + for ( ; i < 4; i++ ) { + expanded[ prefix + cssExpand[ i ] + suffix ] = + parts[ i ] || parts[ i - 2 ] || parts[ 0 ]; + } + + return expanded; + } + }; + + if ( prefix !== "margin" ) { + jQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber; + } +} ); + +jQuery.fn.extend( { + css: function( name, value ) { + return access( this, function( elem, name, value ) { + var styles, len, + map = {}, + i = 0; + + if ( Array.isArray( name ) ) { + styles = getStyles( elem ); + len = name.length; + + for ( ; i < len; i++ ) { + map[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles ); + } + + return map; + } + + return value !== undefined ? + jQuery.style( elem, name, value ) : + jQuery.css( elem, name ); + }, name, value, arguments.length > 1 ); + } +} ); + + +function Tween( elem, options, prop, end, easing ) { + return new Tween.prototype.init( elem, options, prop, end, easing ); +} +jQuery.Tween = Tween; + +Tween.prototype = { + constructor: Tween, + init: function( elem, options, prop, end, easing, unit ) { + this.elem = elem; + this.prop = prop; + this.easing = easing || jQuery.easing._default; + this.options = options; + this.start = this.now = this.cur(); + this.end = end; + this.unit = unit || ( jQuery.cssNumber[ prop ] ? "" : "px" ); + }, + cur: function() { + var hooks = Tween.propHooks[ this.prop ]; + + return hooks && hooks.get ? + hooks.get( this ) : + Tween.propHooks._default.get( this ); + }, + run: function( percent ) { + var eased, + hooks = Tween.propHooks[ this.prop ]; + + if ( this.options.duration ) { + this.pos = eased = jQuery.easing[ this.easing ]( + percent, this.options.duration * percent, 0, 1, this.options.duration + ); + } else { + this.pos = eased = percent; + } + this.now = ( this.end - this.start ) * eased + this.start; + + if ( this.options.step ) { + this.options.step.call( this.elem, this.now, this ); + } + + if ( hooks && hooks.set ) { + hooks.set( this ); + } else { + Tween.propHooks._default.set( this ); + } + return this; + } +}; + +Tween.prototype.init.prototype = Tween.prototype; + +Tween.propHooks = { + _default: { + get: function( tween ) { + var result; + + // Use a property on the element directly when it is not a DOM element, + // or when there is no matching style property that exists. + if ( tween.elem.nodeType !== 1 || + tween.elem[ tween.prop ] != null && tween.elem.style[ tween.prop ] == null ) { + return tween.elem[ tween.prop ]; + } + + // Passing an empty string as a 3rd parameter to .css will automatically + // attempt a parseFloat and fallback to a string if the parse fails. + // Simple values such as "10px" are parsed to Float; + // complex values such as "rotate(1rad)" are returned as-is. + result = jQuery.css( tween.elem, tween.prop, "" ); + + // Empty strings, null, undefined and "auto" are converted to 0. + return !result || result === "auto" ? 0 : result; + }, + set: function( tween ) { + + // Use step hook for back compat. + // Use cssHook if its there. + // Use .style if available and use plain properties where available. + if ( jQuery.fx.step[ tween.prop ] ) { + jQuery.fx.step[ tween.prop ]( tween ); + } else if ( tween.elem.nodeType === 1 && + ( tween.elem.style[ jQuery.cssProps[ tween.prop ] ] != null || + jQuery.cssHooks[ tween.prop ] ) ) { + jQuery.style( tween.elem, tween.prop, tween.now + tween.unit ); + } else { + tween.elem[ tween.prop ] = tween.now; + } + } + } +}; + +// Support: IE <=9 only +// Panic based approach to setting things on disconnected nodes +Tween.propHooks.scrollTop = Tween.propHooks.scrollLeft = { + set: function( tween ) { + if ( tween.elem.nodeType && tween.elem.parentNode ) { + tween.elem[ tween.prop ] = tween.now; + } + } +}; + +jQuery.easing = { + linear: function( p ) { + return p; + }, + swing: function( p ) { + return 0.5 - Math.cos( p * Math.PI ) / 2; + }, + _default: "swing" +}; + +jQuery.fx = Tween.prototype.init; + +// Back compat <1.8 extension point +jQuery.fx.step = {}; + + + + +var + fxNow, inProgress, + rfxtypes = /^(?:toggle|show|hide)$/, + rrun = /queueHooks$/; + +function schedule() { + if ( inProgress ) { + if ( document.hidden === false && window.requestAnimationFrame ) { + window.requestAnimationFrame( schedule ); + } else { + window.setTimeout( schedule, jQuery.fx.interval ); + } + + jQuery.fx.tick(); + } +} + +// Animations created synchronously will run synchronously +function createFxNow() { + window.setTimeout( function() { + fxNow = undefined; + } ); + return ( fxNow = Date.now() ); +} + +// Generate parameters to create a standard animation +function genFx( type, includeWidth ) { + var which, + i = 0, + attrs = { height: type }; + + // If we include width, step value is 1 to do all cssExpand values, + // otherwise step value is 2 to skip over Left and Right + includeWidth = includeWidth ? 1 : 0; + for ( ; i < 4; i += 2 - includeWidth ) { + which = cssExpand[ i ]; + attrs[ "margin" + which ] = attrs[ "padding" + which ] = type; + } + + if ( includeWidth ) { + attrs.opacity = attrs.width = type; + } + + return attrs; +} + +function createTween( value, prop, animation ) { + var tween, + collection = ( Animation.tweeners[ prop ] || [] ).concat( Animation.tweeners[ "*" ] ), + index = 0, + length = collection.length; + for ( ; index < length; index++ ) { + if ( ( tween = collection[ index ].call( animation, prop, value ) ) ) { + + // We're done with this property + return tween; + } + } +} + +function defaultPrefilter( elem, props, opts ) { + var prop, value, toggle, hooks, oldfire, propTween, restoreDisplay, display, + isBox = "width" in props || "height" in props, + anim = this, + orig = {}, + style = elem.style, + hidden = elem.nodeType && isHiddenWithinTree( elem ), + dataShow = dataPriv.get( elem, "fxshow" ); + + // Queue-skipping animations hijack the fx hooks + if ( !opts.queue ) { + hooks = jQuery._queueHooks( elem, "fx" ); + if ( hooks.unqueued == null ) { + hooks.unqueued = 0; + oldfire = hooks.empty.fire; + hooks.empty.fire = function() { + if ( !hooks.unqueued ) { + oldfire(); + } + }; + } + hooks.unqueued++; + + anim.always( function() { + + // Ensure the complete handler is called before this completes + anim.always( function() { + hooks.unqueued--; + if ( !jQuery.queue( elem, "fx" ).length ) { + hooks.empty.fire(); + } + } ); + } ); + } + + // Detect show/hide animations + for ( prop in props ) { + value = props[ prop ]; + if ( rfxtypes.test( value ) ) { + delete props[ prop ]; + toggle = toggle || value === "toggle"; + if ( value === ( hidden ? "hide" : "show" ) ) { + + // Pretend to be hidden if this is a "show" and + // there is still data from a stopped show/hide + if ( value === "show" && dataShow && dataShow[ prop ] !== undefined ) { + hidden = true; + + // Ignore all other no-op show/hide data + } else { + continue; + } + } + orig[ prop ] = dataShow && dataShow[ prop ] || jQuery.style( elem, prop ); + } + } + + // Bail out if this is a no-op like .hide().hide() + propTween = !jQuery.isEmptyObject( props ); + if ( !propTween && jQuery.isEmptyObject( orig ) ) { + return; + } + + // Restrict "overflow" and "display" styles during box animations + if ( isBox && elem.nodeType === 1 ) { + + // Support: IE <=9 - 11, Edge 12 - 15 + // Record all 3 overflow attributes because IE does not infer the shorthand + // from identically-valued overflowX and overflowY and Edge just mirrors + // the overflowX value there. + opts.overflow = [ style.overflow, style.overflowX, style.overflowY ]; + + // Identify a display type, preferring old show/hide data over the CSS cascade + restoreDisplay = dataShow && dataShow.display; + if ( restoreDisplay == null ) { + restoreDisplay = dataPriv.get( elem, "display" ); + } + display = jQuery.css( elem, "display" ); + if ( display === "none" ) { + if ( restoreDisplay ) { + display = restoreDisplay; + } else { + + // Get nonempty value(s) by temporarily forcing visibility + showHide( [ elem ], true ); + restoreDisplay = elem.style.display || restoreDisplay; + display = jQuery.css( elem, "display" ); + showHide( [ elem ] ); + } + } + + // Animate inline elements as inline-block + if ( display === "inline" || display === "inline-block" && restoreDisplay != null ) { + if ( jQuery.css( elem, "float" ) === "none" ) { + + // Restore the original display value at the end of pure show/hide animations + if ( !propTween ) { + anim.done( function() { + style.display = restoreDisplay; + } ); + if ( restoreDisplay == null ) { + display = style.display; + restoreDisplay = display === "none" ? "" : display; + } + } + style.display = "inline-block"; + } + } + } + + if ( opts.overflow ) { + style.overflow = "hidden"; + anim.always( function() { + style.overflow = opts.overflow[ 0 ]; + style.overflowX = opts.overflow[ 1 ]; + style.overflowY = opts.overflow[ 2 ]; + } ); + } + + // Implement show/hide animations + propTween = false; + for ( prop in orig ) { + + // General show/hide setup for this element animation + if ( !propTween ) { + if ( dataShow ) { + if ( "hidden" in dataShow ) { + hidden = dataShow.hidden; + } + } else { + dataShow = dataPriv.access( elem, "fxshow", { display: restoreDisplay } ); + } + + // Store hidden/visible for toggle so `.stop().toggle()` "reverses" + if ( toggle ) { + dataShow.hidden = !hidden; + } + + // Show elements before animating them + if ( hidden ) { + showHide( [ elem ], true ); + } + + /* eslint-disable no-loop-func */ + + anim.done( function() { + + /* eslint-enable no-loop-func */ + + // The final step of a "hide" animation is actually hiding the element + if ( !hidden ) { + showHide( [ elem ] ); + } + dataPriv.remove( elem, "fxshow" ); + for ( prop in orig ) { + jQuery.style( elem, prop, orig[ prop ] ); + } + } ); + } + + // Per-property setup + propTween = createTween( hidden ? dataShow[ prop ] : 0, prop, anim ); + if ( !( prop in dataShow ) ) { + dataShow[ prop ] = propTween.start; + if ( hidden ) { + propTween.end = propTween.start; + propTween.start = 0; + } + } + } +} + +function propFilter( props, specialEasing ) { + var index, name, easing, value, hooks; + + // camelCase, specialEasing and expand cssHook pass + for ( index in props ) { + name = camelCase( index ); + easing = specialEasing[ name ]; + value = props[ index ]; + if ( Array.isArray( value ) ) { + easing = value[ 1 ]; + value = props[ index ] = value[ 0 ]; + } + + if ( index !== name ) { + props[ name ] = value; + delete props[ index ]; + } + + hooks = jQuery.cssHooks[ name ]; + if ( hooks && "expand" in hooks ) { + value = hooks.expand( value ); + delete props[ name ]; + + // Not quite $.extend, this won't overwrite existing keys. + // Reusing 'index' because we have the correct "name" + for ( index in value ) { + if ( !( index in props ) ) { + props[ index ] = value[ index ]; + specialEasing[ index ] = easing; + } + } + } else { + specialEasing[ name ] = easing; + } + } +} + +function Animation( elem, properties, options ) { + var result, + stopped, + index = 0, + length = Animation.prefilters.length, + deferred = jQuery.Deferred().always( function() { + + // Don't match elem in the :animated selector + delete tick.elem; + } ), + tick = function() { + if ( stopped ) { + return false; + } + var currentTime = fxNow || createFxNow(), + remaining = Math.max( 0, animation.startTime + animation.duration - currentTime ), + + // Support: Android 2.3 only + // Archaic crash bug won't allow us to use `1 - ( 0.5 || 0 )` (#12497) + temp = remaining / animation.duration || 0, + percent = 1 - temp, + index = 0, + length = animation.tweens.length; + + for ( ; index < length; index++ ) { + animation.tweens[ index ].run( percent ); + } + + deferred.notifyWith( elem, [ animation, percent, remaining ] ); + + // If there's more to do, yield + if ( percent < 1 && length ) { + return remaining; + } + + // If this was an empty animation, synthesize a final progress notification + if ( !length ) { + deferred.notifyWith( elem, [ animation, 1, 0 ] ); + } + + // Resolve the animation and report its conclusion + deferred.resolveWith( elem, [ animation ] ); + return false; + }, + animation = deferred.promise( { + elem: elem, + props: jQuery.extend( {}, properties ), + opts: jQuery.extend( true, { + specialEasing: {}, + easing: jQuery.easing._default + }, options ), + originalProperties: properties, + originalOptions: options, + startTime: fxNow || createFxNow(), + duration: options.duration, + tweens: [], + createTween: function( prop, end ) { + var tween = jQuery.Tween( elem, animation.opts, prop, end, + animation.opts.specialEasing[ prop ] || animation.opts.easing ); + animation.tweens.push( tween ); + return tween; + }, + stop: function( gotoEnd ) { + var index = 0, + + // If we are going to the end, we want to run all the tweens + // otherwise we skip this part + length = gotoEnd ? animation.tweens.length : 0; + if ( stopped ) { + return this; + } + stopped = true; + for ( ; index < length; index++ ) { + animation.tweens[ index ].run( 1 ); + } + + // Resolve when we played the last frame; otherwise, reject + if ( gotoEnd ) { + deferred.notifyWith( elem, [ animation, 1, 0 ] ); + deferred.resolveWith( elem, [ animation, gotoEnd ] ); + } else { + deferred.rejectWith( elem, [ animation, gotoEnd ] ); + } + return this; + } + } ), + props = animation.props; + + propFilter( props, animation.opts.specialEasing ); + + for ( ; index < length; index++ ) { + result = Animation.prefilters[ index ].call( animation, elem, props, animation.opts ); + if ( result ) { + if ( isFunction( result.stop ) ) { + jQuery._queueHooks( animation.elem, animation.opts.queue ).stop = + result.stop.bind( result ); + } + return result; + } + } + + jQuery.map( props, createTween, animation ); + + if ( isFunction( animation.opts.start ) ) { + animation.opts.start.call( elem, animation ); + } + + // Attach callbacks from options + animation + .progress( animation.opts.progress ) + .done( animation.opts.done, animation.opts.complete ) + .fail( animation.opts.fail ) + .always( animation.opts.always ); + + jQuery.fx.timer( + jQuery.extend( tick, { + elem: elem, + anim: animation, + queue: animation.opts.queue + } ) + ); + + return animation; +} + +jQuery.Animation = jQuery.extend( Animation, { + + tweeners: { + "*": [ function( prop, value ) { + var tween = this.createTween( prop, value ); + adjustCSS( tween.elem, prop, rcssNum.exec( value ), tween ); + return tween; + } ] + }, + + tweener: function( props, callback ) { + if ( isFunction( props ) ) { + callback = props; + props = [ "*" ]; + } else { + props = props.match( rnothtmlwhite ); + } + + var prop, + index = 0, + length = props.length; + + for ( ; index < length; index++ ) { + prop = props[ index ]; + Animation.tweeners[ prop ] = Animation.tweeners[ prop ] || []; + Animation.tweeners[ prop ].unshift( callback ); + } + }, + + prefilters: [ defaultPrefilter ], + + prefilter: function( callback, prepend ) { + if ( prepend ) { + Animation.prefilters.unshift( callback ); + } else { + Animation.prefilters.push( callback ); + } + } +} ); + +jQuery.speed = function( speed, easing, fn ) { + var opt = speed && typeof speed === "object" ? jQuery.extend( {}, speed ) : { + complete: fn || !fn && easing || + isFunction( speed ) && speed, + duration: speed, + easing: fn && easing || easing && !isFunction( easing ) && easing + }; + + // Go to the end state if fx are off + if ( jQuery.fx.off ) { + opt.duration = 0; + + } else { + if ( typeof opt.duration !== "number" ) { + if ( opt.duration in jQuery.fx.speeds ) { + opt.duration = jQuery.fx.speeds[ opt.duration ]; + + } else { + opt.duration = jQuery.fx.speeds._default; + } + } + } + + // Normalize opt.queue - true/undefined/null -> "fx" + if ( opt.queue == null || opt.queue === true ) { + opt.queue = "fx"; + } + + // Queueing + opt.old = opt.complete; + + opt.complete = function() { + if ( isFunction( opt.old ) ) { + opt.old.call( this ); + } + + if ( opt.queue ) { + jQuery.dequeue( this, opt.queue ); + } + }; + + return opt; +}; + +jQuery.fn.extend( { + fadeTo: function( speed, to, easing, callback ) { + + // Show any hidden elements after setting opacity to 0 + return this.filter( isHiddenWithinTree ).css( "opacity", 0 ).show() + + // Animate to the value specified + .end().animate( { opacity: to }, speed, easing, callback ); + }, + animate: function( prop, speed, easing, callback ) { + var empty = jQuery.isEmptyObject( prop ), + optall = jQuery.speed( speed, easing, callback ), + doAnimation = function() { + + // Operate on a copy of prop so per-property easing won't be lost + var anim = Animation( this, jQuery.extend( {}, prop ), optall ); + + // Empty animations, or finishing resolves immediately + if ( empty || dataPriv.get( this, "finish" ) ) { + anim.stop( true ); + } + }; + doAnimation.finish = doAnimation; + + return empty || optall.queue === false ? + this.each( doAnimation ) : + this.queue( optall.queue, doAnimation ); + }, + stop: function( type, clearQueue, gotoEnd ) { + var stopQueue = function( hooks ) { + var stop = hooks.stop; + delete hooks.stop; + stop( gotoEnd ); + }; + + if ( typeof type !== "string" ) { + gotoEnd = clearQueue; + clearQueue = type; + type = undefined; + } + if ( clearQueue && type !== false ) { + this.queue( type || "fx", [] ); + } + + return this.each( function() { + var dequeue = true, + index = type != null && type + "queueHooks", + timers = jQuery.timers, + data = dataPriv.get( this ); + + if ( index ) { + if ( data[ index ] && data[ index ].stop ) { + stopQueue( data[ index ] ); + } + } else { + for ( index in data ) { + if ( data[ index ] && data[ index ].stop && rrun.test( index ) ) { + stopQueue( data[ index ] ); + } + } + } + + for ( index = timers.length; index--; ) { + if ( timers[ index ].elem === this && + ( type == null || timers[ index ].queue === type ) ) { + + timers[ index ].anim.stop( gotoEnd ); + dequeue = false; + timers.splice( index, 1 ); + } + } + + // Start the next in the queue if the last step wasn't forced. + // Timers currently will call their complete callbacks, which + // will dequeue but only if they were gotoEnd. + if ( dequeue || !gotoEnd ) { + jQuery.dequeue( this, type ); + } + } ); + }, + finish: function( type ) { + if ( type !== false ) { + type = type || "fx"; + } + return this.each( function() { + var index, + data = dataPriv.get( this ), + queue = data[ type + "queue" ], + hooks = data[ type + "queueHooks" ], + timers = jQuery.timers, + length = queue ? queue.length : 0; + + // Enable finishing flag on private data + data.finish = true; + + // Empty the queue first + jQuery.queue( this, type, [] ); + + if ( hooks && hooks.stop ) { + hooks.stop.call( this, true ); + } + + // Look for any active animations, and finish them + for ( index = timers.length; index--; ) { + if ( timers[ index ].elem === this && timers[ index ].queue === type ) { + timers[ index ].anim.stop( true ); + timers.splice( index, 1 ); + } + } + + // Look for any animations in the old queue and finish them + for ( index = 0; index < length; index++ ) { + if ( queue[ index ] && queue[ index ].finish ) { + queue[ index ].finish.call( this ); + } + } + + // Turn off finishing flag + delete data.finish; + } ); + } +} ); + +jQuery.each( [ "toggle", "show", "hide" ], function( i, name ) { + var cssFn = jQuery.fn[ name ]; + jQuery.fn[ name ] = function( speed, easing, callback ) { + return speed == null || typeof speed === "boolean" ? + cssFn.apply( this, arguments ) : + this.animate( genFx( name, true ), speed, easing, callback ); + }; +} ); + +// Generate shortcuts for custom animations +jQuery.each( { + slideDown: genFx( "show" ), + slideUp: genFx( "hide" ), + slideToggle: genFx( "toggle" ), + fadeIn: { opacity: "show" }, + fadeOut: { opacity: "hide" }, + fadeToggle: { opacity: "toggle" } +}, function( name, props ) { + jQuery.fn[ name ] = function( speed, easing, callback ) { + return this.animate( props, speed, easing, callback ); + }; +} ); + +jQuery.timers = []; +jQuery.fx.tick = function() { + var timer, + i = 0, + timers = jQuery.timers; + + fxNow = Date.now(); + + for ( ; i < timers.length; i++ ) { + timer = timers[ i ]; + + // Run the timer and safely remove it when done (allowing for external removal) + if ( !timer() && timers[ i ] === timer ) { + timers.splice( i--, 1 ); + } + } + + if ( !timers.length ) { + jQuery.fx.stop(); + } + fxNow = undefined; +}; + +jQuery.fx.timer = function( timer ) { + jQuery.timers.push( timer ); + jQuery.fx.start(); +}; + +jQuery.fx.interval = 13; +jQuery.fx.start = function() { + if ( inProgress ) { + return; + } + + inProgress = true; + schedule(); +}; + +jQuery.fx.stop = function() { + inProgress = null; +}; + +jQuery.fx.speeds = { + slow: 600, + fast: 200, + + // Default speed + _default: 400 +}; + + +// Based off of the plugin by Clint Helfers, with permission. +// https://web.archive.org/web/20100324014747/http://blindsignals.com/index.php/2009/07/jquery-delay/ +jQuery.fn.delay = function( time, type ) { + time = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time; + type = type || "fx"; + + return this.queue( type, function( next, hooks ) { + var timeout = window.setTimeout( next, time ); + hooks.stop = function() { + window.clearTimeout( timeout ); + }; + } ); +}; + + +( function() { + var input = document.createElement( "input" ), + select = document.createElement( "select" ), + opt = select.appendChild( document.createElement( "option" ) ); + + input.type = "checkbox"; + + // Support: Android <=4.3 only + // Default value for a checkbox should be "on" + support.checkOn = input.value !== ""; + + // Support: IE <=11 only + // Must access selectedIndex to make default options select + support.optSelected = opt.selected; + + // Support: IE <=11 only + // An input loses its value after becoming a radio + input = document.createElement( "input" ); + input.value = "t"; + input.type = "radio"; + support.radioValue = input.value === "t"; +} )(); + + +var boolHook, + attrHandle = jQuery.expr.attrHandle; + +jQuery.fn.extend( { attr: function( name, value ) { - return jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 ); + return access( this, jQuery.attr, name, value, arguments.length > 1 ); }, removeAttr: function( name ) { - return this.each(function() { + return this.each( function() { jQuery.removeAttr( this, name ); - }); + } ); + } +} ); + +jQuery.extend( { + attr: function( elem, name, value ) { + var ret, hooks, + nType = elem.nodeType; + + // Don't get/set attributes on text, comment and attribute nodes + if ( nType === 3 || nType === 8 || nType === 2 ) { + return; + } + + // Fallback to prop when attributes are not supported + if ( typeof elem.getAttribute === "undefined" ) { + return jQuery.prop( elem, name, value ); + } + + // Attribute hooks are determined by the lowercase version + // Grab necessary hook if one is defined + if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { + hooks = jQuery.attrHooks[ name.toLowerCase() ] || + ( jQuery.expr.match.bool.test( name ) ? boolHook : undefined ); + } + + if ( value !== undefined ) { + if ( value === null ) { + jQuery.removeAttr( elem, name ); + return; + } + + if ( hooks && "set" in hooks && + ( ret = hooks.set( elem, value, name ) ) !== undefined ) { + return ret; + } + + elem.setAttribute( name, value + "" ); + return value; + } + + if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { + return ret; + } + + ret = jQuery.find.attr( elem, name ); + + // Non-existent attributes return null, we normalize to undefined + return ret == null ? undefined : ret; }, + attrHooks: { + type: { + set: function( elem, value ) { + if ( !support.radioValue && value === "radio" && + nodeName( elem, "input" ) ) { + var val = elem.value; + elem.setAttribute( "type", value ); + if ( val ) { + elem.value = val; + } + return value; + } + } + } + }, + + removeAttr: function( elem, value ) { + var name, + i = 0, + + // Attribute names can contain non-HTML whitespace characters + // https://html.spec.whatwg.org/multipage/syntax.html#attributes-2 + attrNames = value && value.match( rnothtmlwhite ); + + if ( attrNames && elem.nodeType === 1 ) { + while ( ( name = attrNames[ i++ ] ) ) { + elem.removeAttribute( name ); + } + } + } +} ); + +// Hooks for boolean attributes +boolHook = { + set: function( elem, value, name ) { + if ( value === false ) { + + // Remove boolean attributes when set to false + jQuery.removeAttr( elem, name ); + } else { + elem.setAttribute( name, name ); + } + return name; + } +}; + +jQuery.each( jQuery.expr.match.bool.source.match( /\w+/g ), function( i, name ) { + var getter = attrHandle[ name ] || jQuery.find.attr; + + attrHandle[ name ] = function( elem, name, isXML ) { + var ret, handle, + lowercaseName = name.toLowerCase(); + + if ( !isXML ) { + + // Avoid an infinite loop by temporarily removing this function from the getter + handle = attrHandle[ lowercaseName ]; + attrHandle[ lowercaseName ] = ret; + ret = getter( elem, name, isXML ) != null ? + lowercaseName : + null; + attrHandle[ lowercaseName ] = handle; + } + return ret; + }; +} ); + + + + +var rfocusable = /^(?:input|select|textarea|button)$/i, + rclickable = /^(?:a|area)$/i; + +jQuery.fn.extend( { prop: function( name, value ) { - return jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 ); + return access( this, jQuery.prop, name, value, arguments.length > 1 ); }, removeProp: function( name ) { - name = jQuery.propFix[ name ] || name; - return this.each(function() { - // try/catch handles cases where IE balks (such as removing a property on window) - try { - this[ name ] = undefined; - delete this[ name ]; - } catch( e ) {} - }); - }, + return this.each( function() { + delete this[ jQuery.propFix[ name ] || name ]; + } ); + } +} ); - addClass: function( value ) { - var classes, elem, cur, clazz, j, - i = 0, - len = this.length, - proceed = typeof value === "string" && value; +jQuery.extend( { + prop: function( elem, name, value ) { + var ret, hooks, + nType = elem.nodeType; - if ( jQuery.isFunction( value ) ) { - return this.each(function( j ) { - jQuery( this ).addClass( value.call( this, j, this.className ) ); - }); + // Don't get/set properties on text, comment and attribute nodes + if ( nType === 3 || nType === 8 || nType === 2 ) { + return; } - if ( proceed ) { - // The disjunction here is for better compressibility (see removeClass) - classes = ( value || "" ).match( core_rnotwhite ) || []; + if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { - for ( ; i < len; i++ ) { - elem = this[ i ]; - cur = elem.nodeType === 1 && ( elem.className ? - ( " " + elem.className + " " ).replace( rclass, " " ) : - " " - ); + // Fix name and attach hooks + name = jQuery.propFix[ name ] || name; + hooks = jQuery.propHooks[ name ]; + } + + if ( value !== undefined ) { + if ( hooks && "set" in hooks && + ( ret = hooks.set( elem, value, name ) ) !== undefined ) { + return ret; + } + + return ( elem[ name ] = value ); + } + + if ( hooks && "get" in hooks && ( ret = hooks.get( elem, name ) ) !== null ) { + return ret; + } + + return elem[ name ]; + }, + + propHooks: { + tabIndex: { + get: function( elem ) { + + // Support: IE <=9 - 11 only + // elem.tabIndex doesn't always return the + // correct value when it hasn't been explicitly set + // https://web.archive.org/web/20141116233347/http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ + // Use proper attribute retrieval(#12072) + var tabindex = jQuery.find.attr( elem, "tabindex" ); + + if ( tabindex ) { + return parseInt( tabindex, 10 ); + } + + if ( + rfocusable.test( elem.nodeName ) || + rclickable.test( elem.nodeName ) && + elem.href + ) { + return 0; + } + + return -1; + } + } + }, + + propFix: { + "for": "htmlFor", + "class": "className" + } +} ); + +// Support: IE <=11 only +// Accessing the selectedIndex property +// forces the browser to respect setting selected +// on the option +// The getter ensures a default option is selected +// when in an optgroup +// eslint rule "no-unused-expressions" is disabled for this code +// since it considers such accessions noop +if ( !support.optSelected ) { + jQuery.propHooks.selected = { + get: function( elem ) { + + /* eslint no-unused-expressions: "off" */ + + var parent = elem.parentNode; + if ( parent && parent.parentNode ) { + parent.parentNode.selectedIndex; + } + return null; + }, + set: function( elem ) { + + /* eslint no-unused-expressions: "off" */ + + var parent = elem.parentNode; + if ( parent ) { + parent.selectedIndex; + + if ( parent.parentNode ) { + parent.parentNode.selectedIndex; + } + } + } + }; +} + +jQuery.each( [ + "tabIndex", + "readOnly", + "maxLength", + "cellSpacing", + "cellPadding", + "rowSpan", + "colSpan", + "useMap", + "frameBorder", + "contentEditable" +], function() { + jQuery.propFix[ this.toLowerCase() ] = this; +} ); + + + + + // Strip and collapse whitespace according to HTML spec + // https://infra.spec.whatwg.org/#strip-and-collapse-ascii-whitespace + function stripAndCollapse( value ) { + var tokens = value.match( rnothtmlwhite ) || []; + return tokens.join( " " ); + } + + +function getClass( elem ) { + return elem.getAttribute && elem.getAttribute( "class" ) || ""; +} + +function classesToArray( value ) { + if ( Array.isArray( value ) ) { + return value; + } + if ( typeof value === "string" ) { + return value.match( rnothtmlwhite ) || []; + } + return []; +} + +jQuery.fn.extend( { + addClass: function( value ) { + var classes, elem, cur, curValue, clazz, j, finalValue, + i = 0; + + if ( isFunction( value ) ) { + return this.each( function( j ) { + jQuery( this ).addClass( value.call( this, j, getClass( this ) ) ); + } ); + } + + classes = classesToArray( value ); + + if ( classes.length ) { + while ( ( elem = this[ i++ ] ) ) { + curValue = getClass( elem ); + cur = elem.nodeType === 1 && ( " " + stripAndCollapse( curValue ) + " " ); if ( cur ) { j = 0; - while ( (clazz = classes[j++]) ) { + while ( ( clazz = classes[ j++ ] ) ) { if ( cur.indexOf( " " + clazz + " " ) < 0 ) { cur += clazz + " "; } } - elem.className = jQuery.trim( cur ); + // Only assign if different to avoid unneeded rendering. + finalValue = stripAndCollapse( cur ); + if ( curValue !== finalValue ) { + elem.setAttribute( "class", finalValue ); + } } } } @@ -4119,36 +7840,43 @@ jQuery.fn.extend({ }, removeClass: function( value ) { - var classes, elem, cur, clazz, j, - i = 0, - len = this.length, - proceed = arguments.length === 0 || typeof value === "string" && value; + var classes, elem, cur, curValue, clazz, j, finalValue, + i = 0; - if ( jQuery.isFunction( value ) ) { - return this.each(function( j ) { - jQuery( this ).removeClass( value.call( this, j, this.className ) ); - }); + if ( isFunction( value ) ) { + return this.each( function( j ) { + jQuery( this ).removeClass( value.call( this, j, getClass( this ) ) ); + } ); } - if ( proceed ) { - classes = ( value || "" ).match( core_rnotwhite ) || []; - for ( ; i < len; i++ ) { - elem = this[ i ]; + if ( !arguments.length ) { + return this.attr( "class", "" ); + } + + classes = classesToArray( value ); + + if ( classes.length ) { + while ( ( elem = this[ i++ ] ) ) { + curValue = getClass( elem ); + // This expression is here for better compressibility (see addClass) - cur = elem.nodeType === 1 && ( elem.className ? - ( " " + elem.className + " " ).replace( rclass, " " ) : - "" - ); + cur = elem.nodeType === 1 && ( " " + stripAndCollapse( curValue ) + " " ); if ( cur ) { j = 0; - while ( (clazz = classes[j++]) ) { + while ( ( clazz = classes[ j++ ] ) ) { + // Remove *all* instances - while ( cur.indexOf( " " + clazz + " " ) >= 0 ) { + while ( cur.indexOf( " " + clazz + " " ) > -1 ) { cur = cur.replace( " " + clazz + " ", " " ); } } - elem.className = value ? jQuery.trim( cur ) : ""; + + // Only assign if different to avoid unneeded rendering. + finalValue = stripAndCollapse( cur ); + if ( curValue !== finalValue ) { + elem.setAttribute( "class", finalValue ); + } } } } @@ -4157,28 +7885,35 @@ jQuery.fn.extend({ }, toggleClass: function( value, stateVal ) { - var type = typeof value; + var type = typeof value, + isValidValue = type === "string" || Array.isArray( value ); - if ( typeof stateVal === "boolean" && type === "string" ) { + if ( typeof stateVal === "boolean" && isValidValue ) { return stateVal ? this.addClass( value ) : this.removeClass( value ); } - if ( jQuery.isFunction( value ) ) { - return this.each(function( i ) { - jQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal ); - }); + if ( isFunction( value ) ) { + return this.each( function( i ) { + jQuery( this ).toggleClass( + value.call( this, i, getClass( this ), stateVal ), + stateVal + ); + } ); } - return this.each(function() { - if ( type === "string" ) { - // toggle individual class names - var className, - i = 0, - self = jQuery( this ), - classNames = value.match( core_rnotwhite ) || []; + return this.each( function() { + var className, i, self, classNames; - while ( (className = classNames[ i++ ]) ) { - // check each className given, space separated list + if ( isValidValue ) { + + // Toggle individual class names + i = 0; + self = jQuery( this ); + classNames = classesToArray( value ); + + while ( ( className = classNames[ i++ ] ) ) { + + // Check each className given, space separated list if ( self.hasClass( className ) ) { self.removeClass( className ); } else { @@ -4187,68 +7922,91 @@ jQuery.fn.extend({ } // Toggle whole class name - } else if ( type === core_strundefined || type === "boolean" ) { - if ( this.className ) { - // store className if set - jQuery._data( this, "__className__", this.className ); + } else if ( value === undefined || type === "boolean" ) { + className = getClass( this ); + if ( className ) { + + // Store className if set + dataPriv.set( this, "__className__", className ); } - // If the element has a class name or if we're passed "false", + // If the element has a class name or if we're passed `false`, // then remove the whole classname (if there was one, the above saved it). // Otherwise bring back whatever was previously saved (if anything), // falling back to the empty string if nothing was stored. - this.className = this.className || value === false ? "" : jQuery._data( this, "__className__" ) || ""; + if ( this.setAttribute ) { + this.setAttribute( "class", + className || value === false ? + "" : + dataPriv.get( this, "__className__" ) || "" + ); + } } - }); + } ); }, hasClass: function( selector ) { - var className = " " + selector + " ", - i = 0, - l = this.length; - for ( ; i < l; i++ ) { - if ( this[i].nodeType === 1 && (" " + this[i].className + " ").replace(rclass, " ").indexOf( className ) >= 0 ) { - return true; + var className, elem, + i = 0; + + className = " " + selector + " "; + while ( ( elem = this[ i++ ] ) ) { + if ( elem.nodeType === 1 && + ( " " + stripAndCollapse( getClass( elem ) ) + " " ).indexOf( className ) > -1 ) { + return true; } } return false; - }, + } +} ); + + + +var rreturn = /\r/g; + +jQuery.fn.extend( { val: function( value ) { - var ret, hooks, isFunction, - elem = this[0]; + var hooks, ret, valueIsFunction, + elem = this[ 0 ]; if ( !arguments.length ) { if ( elem ) { - hooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ]; + hooks = jQuery.valHooks[ elem.type ] || + jQuery.valHooks[ elem.nodeName.toLowerCase() ]; - if ( hooks && "get" in hooks && (ret = hooks.get( elem, "value" )) !== undefined ) { + if ( hooks && + "get" in hooks && + ( ret = hooks.get( elem, "value" ) ) !== undefined + ) { return ret; } ret = elem.value; - return typeof ret === "string" ? - // handle most common string cases - ret.replace(rreturn, "") : - // handle cases where value is null/undef or number - ret == null ? "" : ret; + // Handle most common string cases + if ( typeof ret === "string" ) { + return ret.replace( rreturn, "" ); + } + + // Handle cases where value is null/undef or number + return ret == null ? "" : ret; } return; } - isFunction = jQuery.isFunction( value ); + valueIsFunction = isFunction( value ); - return this.each(function( i ) { + return this.each( function( i ) { var val; if ( this.nodeType !== 1 ) { return; } - if ( isFunction ) { + if ( valueIsFunction ) { val = value.call( this, i, jQuery( this ).val() ); } else { val = value; @@ -4257,56 +8015,70 @@ jQuery.fn.extend({ // Treat null/undefined as ""; convert numbers to string if ( val == null ) { val = ""; + } else if ( typeof val === "number" ) { val += ""; - } else if ( jQuery.isArray( val ) ) { - val = jQuery.map(val, function ( value ) { + + } else if ( Array.isArray( val ) ) { + val = jQuery.map( val, function( value ) { return value == null ? "" : value + ""; - }); + } ); } hooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ]; // If set returns undefined, fall back to normal setting - if ( !hooks || !("set" in hooks) || hooks.set( this, val, "value" ) === undefined ) { + if ( !hooks || !( "set" in hooks ) || hooks.set( this, val, "value" ) === undefined ) { this.value = val; } - }); + } ); } -}); +} ); -jQuery.extend({ +jQuery.extend( { valHooks: { option: { get: function( elem ) { - // Use proper attribute retrieval(#6932, #12072) + var val = jQuery.find.attr( elem, "value" ); return val != null ? val : - elem.text; + + // Support: IE <=10 - 11 only + // option.text throws exceptions (#14686, #14858) + // Strip and collapse whitespace + // https://html.spec.whatwg.org/#strip-and-collapse-whitespace + stripAndCollapse( jQuery.text( elem ) ); } }, select: { get: function( elem ) { - var value, option, + var value, option, i, options = elem.options, index = elem.selectedIndex, - one = elem.type === "select-one" || index < 0, + one = elem.type === "select-one", values = one ? null : [], - max = one ? index + 1 : options.length, - i = index < 0 ? - max : - one ? index : 0; + max = one ? index + 1 : options.length; + + if ( index < 0 ) { + i = max; + + } else { + i = one ? index : 0; + } // Loop through all the selected options for ( ; i < max; i++ ) { option = options[ i ]; - // oldIE doesn't update selected after form reset (#2551) + // Support: IE <=9 only + // IE8-9 doesn't update selected after form reset (#2551) if ( ( option.selected || i === index ) && + // Don't return options that are disabled or in a disabled optgroup - ( jQuery.support.optDisabled ? !option.disabled : option.getAttribute("disabled") === null ) && - ( !option.parentNode.disabled || !jQuery.nodeName( option.parentNode, "optgroup" ) ) ) { + !option.disabled && + ( !option.parentNode.disabled || + !nodeName( option.parentNode, "optgroup" ) ) ) { // Get the specific value for the option value = jQuery( option ).val(); @@ -4332,600 +8104,68 @@ jQuery.extend({ while ( i-- ) { option = options[ i ]; - if ( (option.selected = jQuery.inArray( jQuery(option).val(), values ) >= 0) ) { + + /* eslint-disable no-cond-assign */ + + if ( option.selected = + jQuery.inArray( jQuery.valHooks.option.get( option ), values ) > -1 + ) { optionSet = true; } + + /* eslint-enable no-cond-assign */ } - // force browsers to behave consistently when non-matching value is set + // Force browsers to behave consistently when non-matching value is set if ( !optionSet ) { elem.selectedIndex = -1; } return values; } } - }, - - attr: function( elem, name, value ) { - var hooks, ret, - nType = elem.nodeType; - - // don't get/set attributes on text, comment and attribute nodes - if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { - return; - } - - // Fallback to prop when attributes are not supported - if ( typeof elem.getAttribute === core_strundefined ) { - return jQuery.prop( elem, name, value ); - } - - // All attributes are lowercase - // Grab necessary hook if one is defined - if ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) { - name = name.toLowerCase(); - hooks = jQuery.attrHooks[ name ] || - ( jQuery.expr.match.bool.test( name ) ? boolHook : nodeHook ); - } - - if ( value !== undefined ) { - - if ( value === null ) { - jQuery.removeAttr( elem, name ); - - } else if ( hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) { - return ret; - - } else { - elem.setAttribute( name, value + "" ); - return value; - } - - } else if ( hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ) { - return ret; - - } else { - ret = jQuery.find.attr( elem, name ); - - // Non-existent attributes return null, we normalize to undefined - return ret == null ? - undefined : - ret; - } - }, - - removeAttr: function( elem, value ) { - var name, propName, - i = 0, - attrNames = value && value.match( core_rnotwhite ); - - if ( attrNames && elem.nodeType === 1 ) { - while ( (name = attrNames[i++]) ) { - propName = jQuery.propFix[ name ] || name; - - // Boolean attributes get special treatment (#10870) - if ( jQuery.expr.match.bool.test( name ) ) { - // Set corresponding property to false - if ( getSetInput && getSetAttribute || !ruseDefault.test( name ) ) { - elem[ propName ] = false; - // Support: IE<9 - // Also clear defaultChecked/defaultSelected (if appropriate) - } else { - elem[ jQuery.camelCase( "default-" + name ) ] = - elem[ propName ] = false; - } - - // See #9699 for explanation of this approach (setting first, then removal) - } else { - jQuery.attr( elem, name, "" ); - } - - elem.removeAttribute( getSetAttribute ? name : propName ); - } - } - }, - - attrHooks: { - type: { - set: function( elem, value ) { - if ( !jQuery.support.radioValue && value === "radio" && jQuery.nodeName(elem, "input") ) { - // Setting the type on a radio button after the value resets the value in IE6-9 - // Reset value to default in case type is set after value during creation - var val = elem.value; - elem.setAttribute( "type", value ); - if ( val ) { - elem.value = val; - } - return value; - } - } - } - }, - - propFix: { - "for": "htmlFor", - "class": "className" - }, - - prop: function( elem, name, value ) { - var ret, hooks, notxml, - nType = elem.nodeType; - - // don't get/set properties on text, comment and attribute nodes - if ( !elem || nType === 3 || nType === 8 || nType === 2 ) { - return; - } - - notxml = nType !== 1 || !jQuery.isXMLDoc( elem ); - - if ( notxml ) { - // Fix name and attach hooks - name = jQuery.propFix[ name ] || name; - hooks = jQuery.propHooks[ name ]; - } - - if ( value !== undefined ) { - return hooks && "set" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ? - ret : - ( elem[ name ] = value ); - - } else { - return hooks && "get" in hooks && (ret = hooks.get( elem, name )) !== null ? - ret : - elem[ name ]; - } - }, - - propHooks: { - tabIndex: { - get: function( elem ) { - // elem.tabIndex doesn't always return the correct value when it hasn't been explicitly set - // http://fluidproject.org/blog/2008/01/09/getting-setting-and-removing-tabindex-values-with-javascript/ - // Use proper attribute retrieval(#12072) - var tabindex = jQuery.find.attr( elem, "tabindex" ); - - return tabindex ? - parseInt( tabindex, 10 ) : - rfocusable.test( elem.nodeName ) || rclickable.test( elem.nodeName ) && elem.href ? - 0 : - -1; - } - } } -}); - -// Hooks for boolean attributes -boolHook = { - set: function( elem, value, name ) { - if ( value === false ) { - // Remove boolean attributes when set to false - jQuery.removeAttr( elem, name ); - } else if ( getSetInput && getSetAttribute || !ruseDefault.test( name ) ) { - // IE<8 needs the *property* name - elem.setAttribute( !getSetAttribute && jQuery.propFix[ name ] || name, name ); - - // Use defaultChecked and defaultSelected for oldIE - } else { - elem[ jQuery.camelCase( "default-" + name ) ] = elem[ name ] = true; - } - - return name; - } -}; -jQuery.each( jQuery.expr.match.bool.source.match( /\w+/g ), function( i, name ) { - var getter = jQuery.expr.attrHandle[ name ] || jQuery.find.attr; - - jQuery.expr.attrHandle[ name ] = getSetInput && getSetAttribute || !ruseDefault.test( name ) ? - function( elem, name, isXML ) { - var fn = jQuery.expr.attrHandle[ name ], - ret = isXML ? - undefined : - /* jshint eqeqeq: false */ - (jQuery.expr.attrHandle[ name ] = undefined) != - getter( elem, name, isXML ) ? - - name.toLowerCase() : - null; - jQuery.expr.attrHandle[ name ] = fn; - return ret; - } : - function( elem, name, isXML ) { - return isXML ? - undefined : - elem[ jQuery.camelCase( "default-" + name ) ] ? - name.toLowerCase() : - null; - }; -}); - -// fix oldIE attroperties -if ( !getSetInput || !getSetAttribute ) { - jQuery.attrHooks.value = { - set: function( elem, value, name ) { - if ( jQuery.nodeName( elem, "input" ) ) { - // Does not return so that setAttribute is also used - elem.defaultValue = value; - } else { - // Use nodeHook if defined (#1954); otherwise setAttribute is fine - return nodeHook && nodeHook.set( elem, value, name ); - } - } - }; -} - -// IE6/7 do not support getting/setting some attributes with get/setAttribute -if ( !getSetAttribute ) { - - // Use this for any attribute in IE6/7 - // This fixes almost every IE6/7 issue - nodeHook = { - set: function( elem, value, name ) { - // Set the existing or create a new attribute node - var ret = elem.getAttributeNode( name ); - if ( !ret ) { - elem.setAttributeNode( - (ret = elem.ownerDocument.createAttribute( name )) - ); - } - - ret.value = value += ""; - - // Break association with cloned elements by also using setAttribute (#9646) - return name === "value" || value === elem.getAttribute( name ) ? - value : - undefined; - } - }; - jQuery.expr.attrHandle.id = jQuery.expr.attrHandle.name = jQuery.expr.attrHandle.coords = - // Some attributes are constructed with empty-string values when not defined - function( elem, name, isXML ) { - var ret; - return isXML ? - undefined : - (ret = elem.getAttributeNode( name )) && ret.value !== "" ? - ret.value : - null; - }; - jQuery.valHooks.button = { - get: function( elem, name ) { - var ret = elem.getAttributeNode( name ); - return ret && ret.specified ? - ret.value : - undefined; - }, - set: nodeHook.set - }; - - // Set contenteditable to false on removals(#10429) - // Setting to empty string throws an error as an invalid value - jQuery.attrHooks.contenteditable = { - set: function( elem, value, name ) { - nodeHook.set( elem, value === "" ? false : value, name ); - } - }; - - // Set width and height to auto instead of 0 on empty string( Bug #8150 ) - // This is for removals - jQuery.each([ "width", "height" ], function( i, name ) { - jQuery.attrHooks[ name ] = { - set: function( elem, value ) { - if ( value === "" ) { - elem.setAttribute( name, "auto" ); - return value; - } - } - }; - }); -} - - -// Some attributes require a special call on IE -// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx -if ( !jQuery.support.hrefNormalized ) { - // href/src property should get the full normalized URL (#10299/#12915) - jQuery.each([ "href", "src" ], function( i, name ) { - jQuery.propHooks[ name ] = { - get: function( elem ) { - return elem.getAttribute( name, 4 ); - } - }; - }); -} - -if ( !jQuery.support.style ) { - jQuery.attrHooks.style = { - get: function( elem ) { - // Return undefined in the case of empty string - // Note: IE uppercases css property names, but if we were to .toLowerCase() - // .cssText, that would destroy case senstitivity in URL's, like in "background" - return elem.style.cssText || undefined; - }, - set: function( elem, value ) { - return ( elem.style.cssText = value + "" ); - } - }; -} - -// Safari mis-reports the default selected property of an option -// Accessing the parent's selectedIndex property fixes it -if ( !jQuery.support.optSelected ) { - jQuery.propHooks.selected = { - get: function( elem ) { - var parent = elem.parentNode; - - if ( parent ) { - parent.selectedIndex; - - // Make sure that it also works with optgroups, see #5701 - if ( parent.parentNode ) { - parent.parentNode.selectedIndex; - } - } - return null; - } - }; -} - -jQuery.each([ - "tabIndex", - "readOnly", - "maxLength", - "cellSpacing", - "cellPadding", - "rowSpan", - "colSpan", - "useMap", - "frameBorder", - "contentEditable" -], function() { - jQuery.propFix[ this.toLowerCase() ] = this; -}); - -// IE6/7 call enctype encoding -if ( !jQuery.support.enctype ) { - jQuery.propFix.enctype = "encoding"; -} +} ); // Radios and checkboxes getter/setter -jQuery.each([ "radio", "checkbox" ], function() { +jQuery.each( [ "radio", "checkbox" ], function() { jQuery.valHooks[ this ] = { set: function( elem, value ) { - if ( jQuery.isArray( value ) ) { - return ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 ); + if ( Array.isArray( value ) ) { + return ( elem.checked = jQuery.inArray( jQuery( elem ).val(), value ) > -1 ); } } }; - if ( !jQuery.support.checkOn ) { + if ( !support.checkOn ) { jQuery.valHooks[ this ].get = function( elem ) { - // Support: Webkit - // "" is returned instead of "on" if a value isn't specified - return elem.getAttribute("value") === null ? "on" : elem.value; + return elem.getAttribute( "value" ) === null ? "on" : elem.value; }; } -}); -var rformElems = /^(?:input|select|textarea)$/i, - rkeyEvent = /^key/, - rmouseEvent = /^(?:mouse|contextmenu)|click/, - rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, - rtypenamespace = /^([^.]*)(?:\.(.+)|)$/; +} ); -function returnTrue() { - return true; -} -function returnFalse() { - return false; -} -function safeActiveElement() { - try { - return document.activeElement; - } catch ( err ) { } -} -/* - * Helper functions for managing events -- not part of the public interface. - * Props to Dean Edwards' addEvent library for many of the ideas. - */ -jQuery.event = { +// Return jQuery for attributes-only inclusion - global: {}, - add: function( elem, types, handler, data, selector ) { - var tmp, events, t, handleObjIn, - special, eventHandle, handleObj, - handlers, type, namespaces, origType, - elemData = jQuery._data( elem ); +support.focusin = "onfocusin" in window; - // Don't attach events to noData or text/comment nodes (but allow plain objects) - if ( !elemData ) { - return; - } - // Caller can pass in an object of custom data in lieu of the handler - if ( handler.handler ) { - handleObjIn = handler; - handler = handleObjIn.handler; - selector = handleObjIn.selector; - } +var rfocusMorph = /^(?:focusinfocus|focusoutblur)$/, + stopPropagationCallback = function( e ) { + e.stopPropagation(); + }; - // Make sure that the handler has a unique ID, used to find/remove it later - if ( !handler.guid ) { - handler.guid = jQuery.guid++; - } - - // Init the element's event structure and main handler, if this is the first - if ( !(events = elemData.events) ) { - events = elemData.events = {}; - } - if ( !(eventHandle = elemData.handle) ) { - eventHandle = elemData.handle = function( e ) { - // Discard the second event of a jQuery.event.trigger() and - // when an event is called after a page has unloaded - return typeof jQuery !== core_strundefined && (!e || jQuery.event.triggered !== e.type) ? - jQuery.event.dispatch.apply( eventHandle.elem, arguments ) : - undefined; - }; - // Add elem as a property of the handle fn to prevent a memory leak with IE non-native events - eventHandle.elem = elem; - } - - // Handle multiple events separated by a space - types = ( types || "" ).match( core_rnotwhite ) || [""]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // There *must* be a type, no attaching namespace-only handlers - if ( !type ) { - continue; - } - - // If event changes its type, use the special event handlers for the changed type - special = jQuery.event.special[ type ] || {}; - - // If selector defined, determine special event api type, otherwise given type - type = ( selector ? special.delegateType : special.bindType ) || type; - - // Update special based on newly reset type - special = jQuery.event.special[ type ] || {}; - - // handleObj is passed to all event handlers - handleObj = jQuery.extend({ - type: type, - origType: origType, - data: data, - handler: handler, - guid: handler.guid, - selector: selector, - needsContext: selector && jQuery.expr.match.needsContext.test( selector ), - namespace: namespaces.join(".") - }, handleObjIn ); - - // Init the event handler queue if we're the first - if ( !(handlers = events[ type ]) ) { - handlers = events[ type ] = []; - handlers.delegateCount = 0; - - // Only use addEventListener/attachEvent if the special events handler returns false - if ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) { - // Bind the global event handler to the element - if ( elem.addEventListener ) { - elem.addEventListener( type, eventHandle, false ); - - } else if ( elem.attachEvent ) { - elem.attachEvent( "on" + type, eventHandle ); - } - } - } - - if ( special.add ) { - special.add.call( elem, handleObj ); - - if ( !handleObj.handler.guid ) { - handleObj.handler.guid = handler.guid; - } - } - - // Add to the element's handler list, delegates in front - if ( selector ) { - handlers.splice( handlers.delegateCount++, 0, handleObj ); - } else { - handlers.push( handleObj ); - } - - // Keep track of which events have ever been used, for event optimization - jQuery.event.global[ type ] = true; - } - - // Nullify elem to prevent memory leaks in IE - elem = null; - }, - - // Detach an event or set of events from an element - remove: function( elem, types, handler, selector, mappedTypes ) { - var j, handleObj, tmp, - origCount, t, events, - special, handlers, type, - namespaces, origType, - elemData = jQuery.hasData( elem ) && jQuery._data( elem ); - - if ( !elemData || !(events = elemData.events) ) { - return; - } - - // Once for each type.namespace in types; type may be omitted - types = ( types || "" ).match( core_rnotwhite ) || [""]; - t = types.length; - while ( t-- ) { - tmp = rtypenamespace.exec( types[t] ) || []; - type = origType = tmp[1]; - namespaces = ( tmp[2] || "" ).split( "." ).sort(); - - // Unbind all events (on this namespace, if provided) for the element - if ( !type ) { - for ( type in events ) { - jQuery.event.remove( elem, type + types[ t ], handler, selector, true ); - } - continue; - } - - special = jQuery.event.special[ type ] || {}; - type = ( selector ? special.delegateType : special.bindType ) || type; - handlers = events[ type ] || []; - tmp = tmp[2] && new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ); - - // Remove matching events - origCount = j = handlers.length; - while ( j-- ) { - handleObj = handlers[ j ]; - - if ( ( mappedTypes || origType === handleObj.origType ) && - ( !handler || handler.guid === handleObj.guid ) && - ( !tmp || tmp.test( handleObj.namespace ) ) && - ( !selector || selector === handleObj.selector || selector === "**" && handleObj.selector ) ) { - handlers.splice( j, 1 ); - - if ( handleObj.selector ) { - handlers.delegateCount--; - } - if ( special.remove ) { - special.remove.call( elem, handleObj ); - } - } - } - - // Remove generic event handler if we removed something and no more handlers exist - // (avoids potential for endless recursion during removal of special event handlers) - if ( origCount && !handlers.length ) { - if ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) { - jQuery.removeEvent( elem, type, elemData.handle ); - } - - delete events[ type ]; - } - } - - // Remove the expando if it's no longer used - if ( jQuery.isEmptyObject( events ) ) { - delete elemData.handle; - - // removeData also checks for emptiness and clears the expando if empty - // so use it instead of delete - jQuery._removeData( elem, "events" ); - } - }, +jQuery.extend( jQuery.event, { trigger: function( event, data, elem, onlyHandlers ) { - var handle, ontype, cur, - bubbleType, special, tmp, i, - eventPath = [ elem || document ], - type = core_hasOwn.call( event, "type" ) ? event.type : event, - namespaces = core_hasOwn.call( event, "namespace" ) ? event.namespace.split(".") : []; - cur = tmp = elem = elem || document; + var i, cur, tmp, bubbleType, ontype, handle, special, lastElement, + eventPath = [ elem || document ], + type = hasOwn.call( event, "type" ) ? event.type : event, + namespaces = hasOwn.call( event, "namespace" ) ? event.namespace.split( "." ) : []; + + cur = lastElement = tmp = elem = elem || document; // Don't do events on text and comment nodes if ( elem.nodeType === 3 || elem.nodeType === 8 ) { @@ -4937,13 +8177,14 @@ jQuery.event = { return; } - if ( type.indexOf(".") >= 0 ) { + if ( type.indexOf( "." ) > -1 ) { + // Namespaced trigger; create a regexp to match event type in handle() - namespaces = type.split("."); + namespaces = type.split( "." ); type = namespaces.shift(); namespaces.sort(); } - ontype = type.indexOf(":") < 0 && "on" + type; + ontype = type.indexOf( ":" ) < 0 && "on" + type; // Caller can pass in a jQuery.Event object, Object, or just an event type string event = event[ jQuery.expando ] ? @@ -4952,9 +8193,9 @@ jQuery.event = { // Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true) event.isTrigger = onlyHandlers ? 2 : 3; - event.namespace = namespaces.join("."); - event.namespace_re = event.namespace ? - new RegExp( "(^|\\.)" + namespaces.join("\\.(?:.*\\.|)") + "(\\.|$)" ) : + event.namespace = namespaces.join( "." ); + event.rnamespace = event.namespace ? + new RegExp( "(^|\\.)" + namespaces.join( "\\.(?:.*\\.|)" ) + "(\\.|$)" ) : null; // Clean up the event in case it is being reused @@ -4976,7 +8217,7 @@ jQuery.event = { // Determine event propagation path in advance, per W3C events spec (#9951) // Bubble up to document, then to window; watch for a global ownerDocument var (#9724) - if ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) { + if ( !onlyHandlers && !special.noBubble && !isWindow( elem ) ) { bubbleType = special.delegateType || type; if ( !rfocusMorph.test( bubbleType + type ) ) { @@ -4988,29 +8229,33 @@ jQuery.event = { } // Only add window if we got to document (e.g., not plain obj or detached DOM) - if ( tmp === (elem.ownerDocument || document) ) { + if ( tmp === ( elem.ownerDocument || document ) ) { eventPath.push( tmp.defaultView || tmp.parentWindow || window ); } } // Fire handlers on the event path i = 0; - while ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) { - + while ( ( cur = eventPath[ i++ ] ) && !event.isPropagationStopped() ) { + lastElement = cur; event.type = i > 1 ? bubbleType : special.bindType || type; // jQuery handler - handle = ( jQuery._data( cur, "events" ) || {} )[ event.type ] && jQuery._data( cur, "handle" ); + handle = ( dataPriv.get( cur, "events" ) || {} )[ event.type ] && + dataPriv.get( cur, "handle" ); if ( handle ) { handle.apply( cur, data ); } // Native handler handle = ontype && cur[ ontype ]; - if ( handle && jQuery.acceptData( cur ) && handle.apply && handle.apply( cur, data ) === false ) { - event.preventDefault(); + if ( handle && handle.apply && acceptData( cur ) ) { + event.result = handle.apply( cur, data ); + if ( event.result === false ) { + event.preventDefault(); + } } } event.type = type; @@ -5018,13 +8263,13 @@ jQuery.event = { // If nobody prevented the default action, do it now if ( !onlyHandlers && !event.isDefaultPrevented() ) { - if ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) && - jQuery.acceptData( elem ) ) { + if ( ( !special._default || + special._default.apply( eventPath.pop(), data ) === false ) && + acceptData( elem ) ) { - // Call a native DOM method on the target with the same name name as the event. - // Can't use an .isFunction() check here because IE6/7 fails that test. + // Call a native DOM method on the target with the same name as the event. // Don't do default actions on window, that's where global variables be (#6170) - if ( ontype && elem[ type ] && !jQuery.isWindow( elem ) ) { + if ( ontype && isFunction( elem[ type ] ) && !isWindow( elem ) ) { // Don't re-trigger an onFOO event when we call its FOO() method tmp = elem[ ontype ]; @@ -5035,12 +8280,17 @@ jQuery.event = { // Prevent re-triggering of the same event, since we already bubbled it above jQuery.event.triggered = type; - try { - elem[ type ](); - } catch ( e ) { - // IE<9 dies on focus/blur to hidden element (#1486,#12518) - // only reproducible on winXP IE8 native, not IE9 in IE8 mode + + if ( event.isPropagationStopped() ) { + lastElement.addEventListener( type, stopPropagationCallback ); } + + elem[ type ](); + + if ( event.isPropagationStopped() ) { + lastElement.removeEventListener( type, stopPropagationCallback ); + } + jQuery.event.triggered = undefined; if ( tmp ) { @@ -5053,2466 +8303,180 @@ jQuery.event = { return event.result; }, - dispatch: function( event ) { - - // Make a writable jQuery.Event from the native event object - event = jQuery.event.fix( event ); - - var i, ret, handleObj, matched, j, - handlerQueue = [], - args = core_slice.call( arguments ), - handlers = ( jQuery._data( this, "events" ) || {} )[ event.type ] || [], - special = jQuery.event.special[ event.type ] || {}; - - // Use the fix-ed jQuery.Event rather than the (read-only) native event - args[0] = event; - event.delegateTarget = this; - - // Call the preDispatch hook for the mapped type, and let it bail if desired - if ( special.preDispatch && special.preDispatch.call( this, event ) === false ) { - return; - } - - // Determine handlers - handlerQueue = jQuery.event.handlers.call( this, event, handlers ); - - // Run delegates first; they may want to stop propagation beneath us - i = 0; - while ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) { - event.currentTarget = matched.elem; - - j = 0; - while ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) { - - // Triggered event must either 1) have no namespace, or - // 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace). - if ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) { - - event.handleObj = handleObj; - event.data = handleObj.data; - - ret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler ) - .apply( matched.elem, args ); - - if ( ret !== undefined ) { - if ( (event.result = ret) === false ) { - event.preventDefault(); - event.stopPropagation(); - } - } - } - } - } - - // Call the postDispatch hook for the mapped type - if ( special.postDispatch ) { - special.postDispatch.call( this, event ); - } - - return event.result; - }, - - handlers: function( event, handlers ) { - var sel, handleObj, matches, i, - handlerQueue = [], - delegateCount = handlers.delegateCount, - cur = event.target; - - // Find delegate handlers - // Black-hole SVG instance trees (#13180) - // Avoid non-left-click bubbling in Firefox (#3861) - if ( delegateCount && cur.nodeType && (!event.button || event.type !== "click") ) { - - /* jshint eqeqeq: false */ - for ( ; cur != this; cur = cur.parentNode || this ) { - /* jshint eqeqeq: true */ - - // Don't check non-elements (#13208) - // Don't process clicks on disabled elements (#6911, #8165, #11382, #11764) - if ( cur.nodeType === 1 && (cur.disabled !== true || event.type !== "click") ) { - matches = []; - for ( i = 0; i < delegateCount; i++ ) { - handleObj = handlers[ i ]; - - // Don't conflict with Object.prototype properties (#13203) - sel = handleObj.selector + " "; - - if ( matches[ sel ] === undefined ) { - matches[ sel ] = handleObj.needsContext ? - jQuery( sel, this ).index( cur ) >= 0 : - jQuery.find( sel, this, null, [ cur ] ).length; - } - if ( matches[ sel ] ) { - matches.push( handleObj ); - } - } - if ( matches.length ) { - handlerQueue.push({ elem: cur, handlers: matches }); - } - } - } - } - - // Add the remaining (directly-bound) handlers - if ( delegateCount < handlers.length ) { - handlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) }); - } - - return handlerQueue; - }, - - fix: function( event ) { - if ( event[ jQuery.expando ] ) { - return event; - } - - // Create a writable copy of the event object and normalize some properties - var i, prop, copy, - type = event.type, - originalEvent = event, - fixHook = this.fixHooks[ type ]; - - if ( !fixHook ) { - this.fixHooks[ type ] = fixHook = - rmouseEvent.test( type ) ? this.mouseHooks : - rkeyEvent.test( type ) ? this.keyHooks : - {}; - } - copy = fixHook.props ? this.props.concat( fixHook.props ) : this.props; - - event = new jQuery.Event( originalEvent ); - - i = copy.length; - while ( i-- ) { - prop = copy[ i ]; - event[ prop ] = originalEvent[ prop ]; - } - - // Support: IE<9 - // Fix target property (#1925) - if ( !event.target ) { - event.target = originalEvent.srcElement || document; - } - - // Support: Chrome 23+, Safari? - // Target should not be a text node (#504, #13143) - if ( event.target.nodeType === 3 ) { - event.target = event.target.parentNode; - } - - // Support: IE<9 - // For mouse/key events, metaKey==false if it's undefined (#3368, #11328) - event.metaKey = !!event.metaKey; - - return fixHook.filter ? fixHook.filter( event, originalEvent ) : event; - }, - - // Includes some event props shared by KeyEvent and MouseEvent - props: "altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "), - - fixHooks: {}, - - keyHooks: { - props: "char charCode key keyCode".split(" "), - filter: function( event, original ) { - - // Add which for key events - if ( event.which == null ) { - event.which = original.charCode != null ? original.charCode : original.keyCode; - } - - return event; - } - }, - - mouseHooks: { - props: "button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "), - filter: function( event, original ) { - var body, eventDoc, doc, - button = original.button, - fromElement = original.fromElement; - - // Calculate pageX/Y if missing and clientX/Y available - if ( event.pageX == null && original.clientX != null ) { - eventDoc = event.target.ownerDocument || document; - doc = eventDoc.documentElement; - body = eventDoc.body; - - event.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 ); - event.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 ); - } - - // Add relatedTarget, if necessary - if ( !event.relatedTarget && fromElement ) { - event.relatedTarget = fromElement === event.target ? original.toElement : fromElement; - } - - // Add which for click: 1 === left; 2 === middle; 3 === right - // Note: button is not normalized, so don't use it - if ( !event.which && button !== undefined ) { - event.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) ); - } - - return event; - } - }, - - special: { - load: { - // Prevent triggered image.load events from bubbling to window.load - noBubble: true - }, - focus: { - // Fire native event if possible so blur/focus sequence is correct - trigger: function() { - if ( this !== safeActiveElement() && this.focus ) { - try { - this.focus(); - return false; - } catch ( e ) { - // Support: IE<9 - // If we error on focus to hidden element (#1486, #12518), - // let .trigger() run the handlers - } - } - }, - delegateType: "focusin" - }, - blur: { - trigger: function() { - if ( this === safeActiveElement() && this.blur ) { - this.blur(); - return false; - } - }, - delegateType: "focusout" - }, - click: { - // For checkbox, fire native event so checked state will be right - trigger: function() { - if ( jQuery.nodeName( this, "input" ) && this.type === "checkbox" && this.click ) { - this.click(); - return false; - } - }, - - // For cross-browser consistency, don't fire native .click() on links - _default: function( event ) { - return jQuery.nodeName( event.target, "a" ); - } - }, - - beforeunload: { - postDispatch: function( event ) { - - // Even when returnValue equals to undefined Firefox will still show alert - if ( event.result !== undefined ) { - event.originalEvent.returnValue = event.result; - } - } - } - }, - - simulate: function( type, elem, event, bubble ) { - // Piggyback on a donor event to simulate a different one. - // Fake originalEvent to avoid donor's stopPropagation, but if the - // simulated event prevents default then we do the same on the donor. + // Piggyback on a donor event to simulate a different one + // Used only for `focus(in | out)` events + simulate: function( type, elem, event ) { var e = jQuery.extend( new jQuery.Event(), event, { type: type, - isSimulated: true, - originalEvent: {} + isSimulated: true } ); - if ( bubble ) { - jQuery.event.trigger( e, null, elem ); - } else { - jQuery.event.dispatch.call( elem, e ); - } - if ( e.isDefaultPrevented() ) { - event.preventDefault(); - } - } -}; -jQuery.removeEvent = document.removeEventListener ? - function( elem, type, handle ) { - if ( elem.removeEventListener ) { - elem.removeEventListener( type, handle, false ); - } - } : - function( elem, type, handle ) { - var name = "on" + type; - - if ( elem.detachEvent ) { - - // #8545, #7054, preventing memory leaks for custom events in IE6-8 - // detachEvent needed property on element, by name of that event, to properly expose it to GC - if ( typeof elem[ name ] === core_strundefined ) { - elem[ name ] = null; - } - - elem.detachEvent( name, handle ); - } - }; - -jQuery.Event = function( src, props ) { - // Allow instantiation without the 'new' keyword - if ( !(this instanceof jQuery.Event) ) { - return new jQuery.Event( src, props ); + jQuery.event.trigger( e, null, elem ); } - // Event object - if ( src && src.type ) { - this.originalEvent = src; - this.type = src.type; +} ); - // Events bubbling up the document may have been marked as prevented - // by a handler lower down the tree; reflect the correct value. - this.isDefaultPrevented = ( src.defaultPrevented || src.returnValue === false || - src.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse; - - // Event type - } else { - this.type = src; - } - - // Put explicitly provided properties onto the event object - if ( props ) { - jQuery.extend( this, props ); - } - - // Create a timestamp if incoming event doesn't have one - this.timeStamp = src && src.timeStamp || jQuery.now(); - - // Mark it as fixed - this[ jQuery.expando ] = true; -}; - -// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding -// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html -jQuery.Event.prototype = { - isDefaultPrevented: returnFalse, - isPropagationStopped: returnFalse, - isImmediatePropagationStopped: returnFalse, - - preventDefault: function() { - var e = this.originalEvent; - - this.isDefaultPrevented = returnTrue; - if ( !e ) { - return; - } - - // If preventDefault exists, run it on the original event - if ( e.preventDefault ) { - e.preventDefault(); - - // Support: IE - // Otherwise set the returnValue property of the original event to false - } else { - e.returnValue = false; - } - }, - stopPropagation: function() { - var e = this.originalEvent; - - this.isPropagationStopped = returnTrue; - if ( !e ) { - return; - } - // If stopPropagation exists, run it on the original event - if ( e.stopPropagation ) { - e.stopPropagation(); - } - - // Support: IE - // Set the cancelBubble property of the original event to true - e.cancelBubble = true; - }, - stopImmediatePropagation: function() { - this.isImmediatePropagationStopped = returnTrue; - this.stopPropagation(); - } -}; - -// Create mouseenter/leave events using mouseover/out and event-time checks -jQuery.each({ - mouseenter: "mouseover", - mouseleave: "mouseout" -}, function( orig, fix ) { - jQuery.event.special[ orig ] = { - delegateType: fix, - bindType: fix, - - handle: function( event ) { - var ret, - target = this, - related = event.relatedTarget, - handleObj = event.handleObj; - - // For mousenter/leave call the handler if related is outside the target. - // NB: No relatedTarget if the mouse left/entered the browser window - if ( !related || (related !== target && !jQuery.contains( target, related )) ) { - event.type = handleObj.origType; - ret = handleObj.handler.apply( this, arguments ); - event.type = fix; - } - return ret; - } - }; -}); - -// IE submit delegation -if ( !jQuery.support.submitBubbles ) { - - jQuery.event.special.submit = { - setup: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Lazy-add a submit handler when a descendant form may potentially be submitted - jQuery.event.add( this, "click._submit keypress._submit", function( e ) { - // Node name check avoids a VML-related crash in IE (#9807) - var elem = e.target, - form = jQuery.nodeName( elem, "input" ) || jQuery.nodeName( elem, "button" ) ? elem.form : undefined; - if ( form && !jQuery._data( form, "submitBubbles" ) ) { - jQuery.event.add( form, "submit._submit", function( event ) { - event._submit_bubble = true; - }); - jQuery._data( form, "submitBubbles", true ); - } - }); - // return undefined since we don't need an event listener - }, - - postDispatch: function( event ) { - // If form was submitted by the user, bubble the event up the tree - if ( event._submit_bubble ) { - delete event._submit_bubble; - if ( this.parentNode && !event.isTrigger ) { - jQuery.event.simulate( "submit", this.parentNode, event, true ); - } - } - }, - - teardown: function() { - // Only need this for delegated form submit events - if ( jQuery.nodeName( this, "form" ) ) { - return false; - } - - // Remove delegated handlers; cleanData eventually reaps submit handlers attached above - jQuery.event.remove( this, "._submit" ); - } - }; -} - -// IE change delegation and checkbox/radio fix -if ( !jQuery.support.changeBubbles ) { - - jQuery.event.special.change = { - - setup: function() { - - if ( rformElems.test( this.nodeName ) ) { - // IE doesn't fire change on a check/radio until blur; trigger it on click - // after a propertychange. Eat the blur-change in special.change.handle. - // This still fires onchange a second time for check/radio after blur. - if ( this.type === "checkbox" || this.type === "radio" ) { - jQuery.event.add( this, "propertychange._change", function( event ) { - if ( event.originalEvent.propertyName === "checked" ) { - this._just_changed = true; - } - }); - jQuery.event.add( this, "click._change", function( event ) { - if ( this._just_changed && !event.isTrigger ) { - this._just_changed = false; - } - // Allow triggered, simulated change events (#11500) - jQuery.event.simulate( "change", this, event, true ); - }); - } - return false; - } - // Delegated event; lazy-add a change handler on descendant inputs - jQuery.event.add( this, "beforeactivate._change", function( e ) { - var elem = e.target; - - if ( rformElems.test( elem.nodeName ) && !jQuery._data( elem, "changeBubbles" ) ) { - jQuery.event.add( elem, "change._change", function( event ) { - if ( this.parentNode && !event.isSimulated && !event.isTrigger ) { - jQuery.event.simulate( "change", this.parentNode, event, true ); - } - }); - jQuery._data( elem, "changeBubbles", true ); - } - }); - }, - - handle: function( event ) { - var elem = event.target; - - // Swallow native change events from checkbox/radio, we already triggered them above - if ( this !== elem || event.isSimulated || event.isTrigger || (elem.type !== "radio" && elem.type !== "checkbox") ) { - return event.handleObj.handler.apply( this, arguments ); - } - }, - - teardown: function() { - jQuery.event.remove( this, "._change" ); - - return !rformElems.test( this.nodeName ); - } - }; -} - -// Create "bubbling" focus and blur events -if ( !jQuery.support.focusinBubbles ) { - jQuery.each({ focus: "focusin", blur: "focusout" }, function( orig, fix ) { - - // Attach a single capturing handler while someone wants focusin/focusout - var attaches = 0, - handler = function( event ) { - jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true ); - }; - - jQuery.event.special[ fix ] = { - setup: function() { - if ( attaches++ === 0 ) { - document.addEventListener( orig, handler, true ); - } - }, - teardown: function() { - if ( --attaches === 0 ) { - document.removeEventListener( orig, handler, true ); - } - } - }; - }); -} - -jQuery.fn.extend({ - - on: function( types, selector, data, fn, /*INTERNAL*/ one ) { - var type, origFn; - - // Types can be a map of types/handlers - if ( typeof types === "object" ) { - // ( types-Object, selector, data ) - if ( typeof selector !== "string" ) { - // ( types-Object, data ) - data = data || selector; - selector = undefined; - } - for ( type in types ) { - this.on( type, selector, data, types[ type ], one ); - } - return this; - } - - if ( data == null && fn == null ) { - // ( types, fn ) - fn = selector; - data = selector = undefined; - } else if ( fn == null ) { - if ( typeof selector === "string" ) { - // ( types, selector, fn ) - fn = data; - data = undefined; - } else { - // ( types, data, fn ) - fn = data; - data = selector; - selector = undefined; - } - } - if ( fn === false ) { - fn = returnFalse; - } else if ( !fn ) { - return this; - } - - if ( one === 1 ) { - origFn = fn; - fn = function( event ) { - // Can use an empty set, since event contains the info - jQuery().off( event ); - return origFn.apply( this, arguments ); - }; - // Use same guid so caller can remove using origFn - fn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ ); - } - return this.each( function() { - jQuery.event.add( this, types, fn, data, selector ); - }); - }, - one: function( types, selector, data, fn ) { - return this.on( types, selector, data, fn, 1 ); - }, - off: function( types, selector, fn ) { - var handleObj, type; - if ( types && types.preventDefault && types.handleObj ) { - // ( event ) dispatched jQuery.Event - handleObj = types.handleObj; - jQuery( types.delegateTarget ).off( - handleObj.namespace ? handleObj.origType + "." + handleObj.namespace : handleObj.origType, - handleObj.selector, - handleObj.handler - ); - return this; - } - if ( typeof types === "object" ) { - // ( types-object [, selector] ) - for ( type in types ) { - this.off( type, selector, types[ type ] ); - } - return this; - } - if ( selector === false || typeof selector === "function" ) { - // ( types [, fn] ) - fn = selector; - selector = undefined; - } - if ( fn === false ) { - fn = returnFalse; - } - return this.each(function() { - jQuery.event.remove( this, types, fn, selector ); - }); - }, +jQuery.fn.extend( { trigger: function( type, data ) { - return this.each(function() { + return this.each( function() { jQuery.event.trigger( type, data, this ); - }); + } ); }, triggerHandler: function( type, data ) { - var elem = this[0]; + var elem = this[ 0 ]; if ( elem ) { return jQuery.event.trigger( type, data, elem, true ); } } -}); -var isSimple = /^.[^:#\[\.,]*$/, - rparentsprev = /^(?:parents|prev(?:Until|All))/, - rneedsContext = jQuery.expr.match.needsContext, - // methods guaranteed to produce a unique set when starting from a unique set - guaranteedUnique = { - children: true, - contents: true, - next: true, - prev: true - }; +} ); -jQuery.fn.extend({ - find: function( selector ) { - var i, - ret = [], - self = this, - len = self.length; - if ( typeof selector !== "string" ) { - return this.pushStack( jQuery( selector ).filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( self[ i ], this ) ) { - return true; - } +// Support: Firefox <=44 +// Firefox doesn't have focus(in | out) events +// Related ticket - https://bugzilla.mozilla.org/show_bug.cgi?id=687787 +// +// Support: Chrome <=48 - 49, Safari <=9.0 - 9.1 +// focus(in | out) events fire after focus & blur events, +// which is spec violation - http://www.w3.org/TR/DOM-Level-3-Events/#events-focusevent-event-order +// Related ticket - https://bugs.chromium.org/p/chromium/issues/detail?id=449857 +if ( !support.focusin ) { + jQuery.each( { focus: "focusin", blur: "focusout" }, function( orig, fix ) { + + // Attach a single capturing handler on the document while someone wants focusin/focusout + var handler = function( event ) { + jQuery.event.simulate( fix, event.target, jQuery.event.fix( event ) ); + }; + + jQuery.event.special[ fix ] = { + setup: function() { + var doc = this.ownerDocument || this, + attaches = dataPriv.access( doc, fix ); + + if ( !attaches ) { + doc.addEventListener( orig, handler, true ); } - }) ); - } + dataPriv.access( doc, fix, ( attaches || 0 ) + 1 ); + }, + teardown: function() { + var doc = this.ownerDocument || this, + attaches = dataPriv.access( doc, fix ) - 1; - for ( i = 0; i < len; i++ ) { - jQuery.find( selector, self[ i ], ret ); - } + if ( !attaches ) { + doc.removeEventListener( orig, handler, true ); + dataPriv.remove( doc, fix ); - // Needed because $( selector, context ) becomes $( context ).find( selector ) - ret = this.pushStack( len > 1 ? jQuery.unique( ret ) : ret ); - ret.selector = this.selector ? this.selector + " " + selector : selector; - return ret; - }, - - has: function( target ) { - var i, - targets = jQuery( target, this ), - len = targets.length; - - return this.filter(function() { - for ( i = 0; i < len; i++ ) { - if ( jQuery.contains( this, targets[i] ) ) { - return true; - } - } - }); - }, - - not: function( selector ) { - return this.pushStack( winnow(this, selector || [], true) ); - }, - - filter: function( selector ) { - return this.pushStack( winnow(this, selector || [], false) ); - }, - - is: function( selector ) { - return !!winnow( - this, - - // If this is a positional/relative selector, check membership in the returned set - // so $("p:first").is("p:last") won't return true for a doc with two "p". - typeof selector === "string" && rneedsContext.test( selector ) ? - jQuery( selector ) : - selector || [], - false - ).length; - }, - - closest: function( selectors, context ) { - var cur, - i = 0, - l = this.length, - ret = [], - pos = rneedsContext.test( selectors ) || typeof selectors !== "string" ? - jQuery( selectors, context || this.context ) : - 0; - - for ( ; i < l; i++ ) { - for ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) { - // Always skip document fragments - if ( cur.nodeType < 11 && (pos ? - pos.index(cur) > -1 : - - // Don't pass non-elements to Sizzle - cur.nodeType === 1 && - jQuery.find.matchesSelector(cur, selectors)) ) { - - cur = ret.push( cur ); - break; - } - } - } - - return this.pushStack( ret.length > 1 ? jQuery.unique( ret ) : ret ); - }, - - // Determine the position of an element within - // the matched set of elements - index: function( elem ) { - - // No argument, return index in parent - if ( !elem ) { - return ( this[0] && this[0].parentNode ) ? this.first().prevAll().length : -1; - } - - // index in selector - if ( typeof elem === "string" ) { - return jQuery.inArray( this[0], jQuery( elem ) ); - } - - // Locate the position of the desired element - return jQuery.inArray( - // If it receives a jQuery object, the first element is used - elem.jquery ? elem[0] : elem, this ); - }, - - add: function( selector, context ) { - var set = typeof selector === "string" ? - jQuery( selector, context ) : - jQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ), - all = jQuery.merge( this.get(), set ); - - return this.pushStack( jQuery.unique(all) ); - }, - - addBack: function( selector ) { - return this.add( selector == null ? - this.prevObject : this.prevObject.filter(selector) - ); - } -}); - -function sibling( cur, dir ) { - do { - cur = cur[ dir ]; - } while ( cur && cur.nodeType !== 1 ); - - return cur; -} - -jQuery.each({ - parent: function( elem ) { - var parent = elem.parentNode; - return parent && parent.nodeType !== 11 ? parent : null; - }, - parents: function( elem ) { - return jQuery.dir( elem, "parentNode" ); - }, - parentsUntil: function( elem, i, until ) { - return jQuery.dir( elem, "parentNode", until ); - }, - next: function( elem ) { - return sibling( elem, "nextSibling" ); - }, - prev: function( elem ) { - return sibling( elem, "previousSibling" ); - }, - nextAll: function( elem ) { - return jQuery.dir( elem, "nextSibling" ); - }, - prevAll: function( elem ) { - return jQuery.dir( elem, "previousSibling" ); - }, - nextUntil: function( elem, i, until ) { - return jQuery.dir( elem, "nextSibling", until ); - }, - prevUntil: function( elem, i, until ) { - return jQuery.dir( elem, "previousSibling", until ); - }, - siblings: function( elem ) { - return jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem ); - }, - children: function( elem ) { - return jQuery.sibling( elem.firstChild ); - }, - contents: function( elem ) { - return jQuery.nodeName( elem, "iframe" ) ? - elem.contentDocument || elem.contentWindow.document : - jQuery.merge( [], elem.childNodes ); - } -}, function( name, fn ) { - jQuery.fn[ name ] = function( until, selector ) { - var ret = jQuery.map( this, fn, until ); - - if ( name.slice( -5 ) !== "Until" ) { - selector = until; - } - - if ( selector && typeof selector === "string" ) { - ret = jQuery.filter( selector, ret ); - } - - if ( this.length > 1 ) { - // Remove duplicates - if ( !guaranteedUnique[ name ] ) { - ret = jQuery.unique( ret ); - } - - // Reverse order for parents* and prev-derivatives - if ( rparentsprev.test( name ) ) { - ret = ret.reverse(); - } - } - - return this.pushStack( ret ); - }; -}); - -jQuery.extend({ - filter: function( expr, elems, not ) { - var elem = elems[ 0 ]; - - if ( not ) { - expr = ":not(" + expr + ")"; - } - - return elems.length === 1 && elem.nodeType === 1 ? - jQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] : - jQuery.find.matches( expr, jQuery.grep( elems, function( elem ) { - return elem.nodeType === 1; - })); - }, - - dir: function( elem, dir, until ) { - var matched = [], - cur = elem[ dir ]; - - while ( cur && cur.nodeType !== 9 && (until === undefined || cur.nodeType !== 1 || !jQuery( cur ).is( until )) ) { - if ( cur.nodeType === 1 ) { - matched.push( cur ); - } - cur = cur[dir]; - } - return matched; - }, - - sibling: function( n, elem ) { - var r = []; - - for ( ; n; n = n.nextSibling ) { - if ( n.nodeType === 1 && n !== elem ) { - r.push( n ); - } - } - - return r; - } -}); - -// Implement the identical functionality for filter and not -function winnow( elements, qualifier, not ) { - if ( jQuery.isFunction( qualifier ) ) { - return jQuery.grep( elements, function( elem, i ) { - /* jshint -W018 */ - return !!qualifier.call( elem, i, elem ) !== not; - }); - - } - - if ( qualifier.nodeType ) { - return jQuery.grep( elements, function( elem ) { - return ( elem === qualifier ) !== not; - }); - - } - - if ( typeof qualifier === "string" ) { - if ( isSimple.test( qualifier ) ) { - return jQuery.filter( qualifier, elements, not ); - } - - qualifier = jQuery.filter( qualifier, elements ); - } - - return jQuery.grep( elements, function( elem ) { - return ( jQuery.inArray( elem, qualifier ) >= 0 ) !== not; - }); -} -function createSafeFragment( document ) { - var list = nodeNames.split( "|" ), - safeFrag = document.createDocumentFragment(); - - if ( safeFrag.createElement ) { - while ( list.length ) { - safeFrag.createElement( - list.pop() - ); - } - } - return safeFrag; -} - -var nodeNames = "abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|" + - "header|hgroup|mark|meter|nav|output|progress|section|summary|time|video", - rinlinejQuery = / jQuery\d+="(?:null|\d+)"/g, - rnoshimcache = new RegExp("<(?:" + nodeNames + ")[\\s/>]", "i"), - rleadingWhitespace = /^\s+/, - rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi, - rtagName = /<([\w:]+)/, - rtbody = /\s*$/g, - - // We have to close these tags to support XHTML (#13200) - wrapMap = { - option: [ 1, "" ], - legend: [ 1, "
", "
" ], - area: [ 1, "", "" ], - param: [ 1, "", "" ], - thead: [ 1, "", "
" ], - tr: [ 2, "", "
" ], - col: [ 2, "", "
" ], - td: [ 3, "", "
" ], - - // IE6-8 can't serialize link, script, style, or any html5 (NoScope) tags, - // unless wrapped in a div with non-breaking characters in front of it. - _default: jQuery.support.htmlSerialize ? [ 0, "", "" ] : [ 1, "X
", "
" ] - }, - safeFragment = createSafeFragment( document ), - fragmentDiv = safeFragment.appendChild( document.createElement("div") ); - -wrapMap.optgroup = wrapMap.option; -wrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.thead; -wrapMap.th = wrapMap.td; - -jQuery.fn.extend({ - text: function( value ) { - return jQuery.access( this, function( value ) { - return value === undefined ? - jQuery.text( this ) : - this.empty().append( ( this[0] && this[0].ownerDocument || document ).createTextNode( value ) ); - }, null, value, arguments.length ); - }, - - append: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.appendChild( elem ); - } - }); - }, - - prepend: function() { - return this.domManip( arguments, function( elem ) { - if ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) { - var target = manipulationTarget( this, elem ); - target.insertBefore( elem, target.firstChild ); - } - }); - }, - - before: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this ); - } - }); - }, - - after: function() { - return this.domManip( arguments, function( elem ) { - if ( this.parentNode ) { - this.parentNode.insertBefore( elem, this.nextSibling ); - } - }); - }, - - // keepData is for internal use only--do not document - remove: function( selector, keepData ) { - var elem, - elems = selector ? jQuery.filter( selector, this ) : this, - i = 0; - - for ( ; (elem = elems[i]) != null; i++ ) { - - if ( !keepData && elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem ) ); - } - - if ( elem.parentNode ) { - if ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) { - setGlobalEval( getAll( elem, "script" ) ); - } - elem.parentNode.removeChild( elem ); - } - } - - return this; - }, - - empty: function() { - var elem, - i = 0; - - for ( ; (elem = this[i]) != null; i++ ) { - // Remove element nodes and prevent memory leaks - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - } - - // Remove any remaining nodes - while ( elem.firstChild ) { - elem.removeChild( elem.firstChild ); - } - - // If this is a select, ensure that it displays empty (#12336) - // Support: IE<9 - if ( elem.options && jQuery.nodeName( elem, "select" ) ) { - elem.options.length = 0; - } - } - - return this; - }, - - clone: function( dataAndEvents, deepDataAndEvents ) { - dataAndEvents = dataAndEvents == null ? false : dataAndEvents; - deepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents; - - return this.map( function () { - return jQuery.clone( this, dataAndEvents, deepDataAndEvents ); - }); - }, - - html: function( value ) { - return jQuery.access( this, function( value ) { - var elem = this[0] || {}, - i = 0, - l = this.length; - - if ( value === undefined ) { - return elem.nodeType === 1 ? - elem.innerHTML.replace( rinlinejQuery, "" ) : - undefined; - } - - // See if we can take a shortcut and just use innerHTML - if ( typeof value === "string" && !rnoInnerhtml.test( value ) && - ( jQuery.support.htmlSerialize || !rnoshimcache.test( value ) ) && - ( jQuery.support.leadingWhitespace || !rleadingWhitespace.test( value ) ) && - !wrapMap[ ( rtagName.exec( value ) || ["", ""] )[1].toLowerCase() ] ) { - - value = value.replace( rxhtmlTag, "<$1>" ); - - try { - for (; i < l; i++ ) { - // Remove element nodes and prevent memory leaks - elem = this[i] || {}; - if ( elem.nodeType === 1 ) { - jQuery.cleanData( getAll( elem, false ) ); - elem.innerHTML = value; - } - } - - elem = 0; - - // If using innerHTML throws an exception, use the fallback method - } catch(e) {} - } - - if ( elem ) { - this.empty().append( value ); - } - }, null, value, arguments.length ); - }, - - replaceWith: function() { - var - // Snapshot the DOM in case .domManip sweeps something relevant into its fragment - args = jQuery.map( this, function( elem ) { - return [ elem.nextSibling, elem.parentNode ]; - }), - i = 0; - - // Make the changes, replacing each context element with the new content - this.domManip( arguments, function( elem ) { - var next = args[ i++ ], - parent = args[ i++ ]; - - if ( parent ) { - // Don't use the snapshot next if it has moved (#13810) - if ( next && next.parentNode !== parent ) { - next = this.nextSibling; - } - jQuery( this ).remove(); - parent.insertBefore( elem, next ); - } - // Allow new content to include elements from the context set - }, true ); - - // Force removal if there was no new content (e.g., from empty arguments) - return i ? this : this.remove(); - }, - - detach: function( selector ) { - return this.remove( selector, true ); - }, - - domManip: function( args, callback, allowIntersection ) { - - // Flatten any nested arrays - args = core_concat.apply( [], args ); - - var first, node, hasScripts, - scripts, doc, fragment, - i = 0, - l = this.length, - set = this, - iNoClone = l - 1, - value = args[0], - isFunction = jQuery.isFunction( value ); - - // We can't cloneNode fragments that contain checked, in WebKit - if ( isFunction || !( l <= 1 || typeof value !== "string" || jQuery.support.checkClone || !rchecked.test( value ) ) ) { - return this.each(function( index ) { - var self = set.eq( index ); - if ( isFunction ) { - args[0] = value.call( this, index, self.html() ); - } - self.domManip( args, callback, allowIntersection ); - }); - } - - if ( l ) { - fragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, !allowIntersection && this ); - first = fragment.firstChild; - - if ( fragment.childNodes.length === 1 ) { - fragment = first; - } - - if ( first ) { - scripts = jQuery.map( getAll( fragment, "script" ), disableScript ); - hasScripts = scripts.length; - - // Use the original fragment for the last item instead of the first because it can end up - // being emptied incorrectly in certain situations (#8070). - for ( ; i < l; i++ ) { - node = fragment; - - if ( i !== iNoClone ) { - node = jQuery.clone( node, true, true ); - - // Keep references to cloned scripts for later restoration - if ( hasScripts ) { - jQuery.merge( scripts, getAll( node, "script" ) ); - } - } - - callback.call( this[i], node, i ); - } - - if ( hasScripts ) { - doc = scripts[ scripts.length - 1 ].ownerDocument; - - // Reenable scripts - jQuery.map( scripts, restoreScript ); - - // Evaluate executable scripts on first document insertion - for ( i = 0; i < hasScripts; i++ ) { - node = scripts[ i ]; - if ( rscriptType.test( node.type || "" ) && - !jQuery._data( node, "globalEval" ) && jQuery.contains( doc, node ) ) { - - if ( node.src ) { - // Hope ajax is available... - jQuery._evalUrl( node.src ); - } else { - jQuery.globalEval( ( node.text || node.textContent || node.innerHTML || "" ).replace( rcleanScript, "" ) ); - } - } - } - } - - // Fix #11809: Avoid leaking memory - fragment = first = null; - } - } - - return this; - } -}); - -// Support: IE<8 -// Manipulating tables requires a tbody -function manipulationTarget( elem, content ) { - return jQuery.nodeName( elem, "table" ) && - jQuery.nodeName( content.nodeType === 1 ? content : content.firstChild, "tr" ) ? - - elem.getElementsByTagName("tbody")[0] || - elem.appendChild( elem.ownerDocument.createElement("tbody") ) : - elem; -} - -// Replace/restore the type attribute of script elements for safe DOM manipulation -function disableScript( elem ) { - elem.type = (jQuery.find.attr( elem, "type" ) !== null) + "/" + elem.type; - return elem; -} -function restoreScript( elem ) { - var match = rscriptTypeMasked.exec( elem.type ); - if ( match ) { - elem.type = match[1]; - } else { - elem.removeAttribute("type"); - } - return elem; -} - -// Mark scripts as having already been evaluated -function setGlobalEval( elems, refElements ) { - var elem, - i = 0; - for ( ; (elem = elems[i]) != null; i++ ) { - jQuery._data( elem, "globalEval", !refElements || jQuery._data( refElements[i], "globalEval" ) ); - } -} - -function cloneCopyEvent( src, dest ) { - - if ( dest.nodeType !== 1 || !jQuery.hasData( src ) ) { - return; - } - - var type, i, l, - oldData = jQuery._data( src ), - curData = jQuery._data( dest, oldData ), - events = oldData.events; - - if ( events ) { - delete curData.handle; - curData.events = {}; - - for ( type in events ) { - for ( i = 0, l = events[ type ].length; i < l; i++ ) { - jQuery.event.add( dest, type, events[ type ][ i ] ); - } - } - } - - // make the cloned public data object a copy from the original - if ( curData.data ) { - curData.data = jQuery.extend( {}, curData.data ); - } -} - -function fixCloneNodeIssues( src, dest ) { - var nodeName, e, data; - - // We do not need to do anything for non-Elements - if ( dest.nodeType !== 1 ) { - return; - } - - nodeName = dest.nodeName.toLowerCase(); - - // IE6-8 copies events bound via attachEvent when using cloneNode. - if ( !jQuery.support.noCloneEvent && dest[ jQuery.expando ] ) { - data = jQuery._data( dest ); - - for ( e in data.events ) { - jQuery.removeEvent( dest, e, data.handle ); - } - - // Event data gets referenced instead of copied if the expando gets copied too - dest.removeAttribute( jQuery.expando ); - } - - // IE blanks contents when cloning scripts, and tries to evaluate newly-set text - if ( nodeName === "script" && dest.text !== src.text ) { - disableScript( dest ).text = src.text; - restoreScript( dest ); - - // IE6-10 improperly clones children of object elements using classid. - // IE10 throws NoModificationAllowedError if parent is null, #12132. - } else if ( nodeName === "object" ) { - if ( dest.parentNode ) { - dest.outerHTML = src.outerHTML; - } - - // This path appears unavoidable for IE9. When cloning an object - // element in IE9, the outerHTML strategy above is not sufficient. - // If the src has innerHTML and the destination does not, - // copy the src.innerHTML into the dest.innerHTML. #10324 - if ( jQuery.support.html5Clone && ( src.innerHTML && !jQuery.trim(dest.innerHTML) ) ) { - dest.innerHTML = src.innerHTML; - } - - } else if ( nodeName === "input" && manipulation_rcheckableType.test( src.type ) ) { - // IE6-8 fails to persist the checked state of a cloned checkbox - // or radio button. Worse, IE6-7 fail to give the cloned element - // a checked appearance if the defaultChecked value isn't also set - - dest.defaultChecked = dest.checked = src.checked; - - // IE6-7 get confused and end up setting the value of a cloned - // checkbox/radio button to an empty string instead of "on" - if ( dest.value !== src.value ) { - dest.value = src.value; - } - - // IE6-8 fails to return the selected option to the default selected - // state when cloning options - } else if ( nodeName === "option" ) { - dest.defaultSelected = dest.selected = src.defaultSelected; - - // IE6-8 fails to set the defaultValue to the correct value when - // cloning other types of input fields - } else if ( nodeName === "input" || nodeName === "textarea" ) { - dest.defaultValue = src.defaultValue; - } -} - -jQuery.each({ - appendTo: "append", - prependTo: "prepend", - insertBefore: "before", - insertAfter: "after", - replaceAll: "replaceWith" -}, function( name, original ) { - jQuery.fn[ name ] = function( selector ) { - var elems, - i = 0, - ret = [], - insert = jQuery( selector ), - last = insert.length - 1; - - for ( ; i <= last; i++ ) { - elems = i === last ? this : this.clone(true); - jQuery( insert[i] )[ original ]( elems ); - - // Modern browsers can apply jQuery collections as arrays, but oldIE needs a .get() - core_push.apply( ret, elems.get() ); - } - - return this.pushStack( ret ); - }; -}); - -function getAll( context, tag ) { - var elems, elem, - i = 0, - found = typeof context.getElementsByTagName !== core_strundefined ? context.getElementsByTagName( tag || "*" ) : - typeof context.querySelectorAll !== core_strundefined ? context.querySelectorAll( tag || "*" ) : - undefined; - - if ( !found ) { - for ( found = [], elems = context.childNodes || context; (elem = elems[i]) != null; i++ ) { - if ( !tag || jQuery.nodeName( elem, tag ) ) { - found.push( elem ); - } else { - jQuery.merge( found, getAll( elem, tag ) ); - } - } - } - - return tag === undefined || tag && jQuery.nodeName( context, tag ) ? - jQuery.merge( [ context ], found ) : - found; -} - -// Used in buildFragment, fixes the defaultChecked property -function fixDefaultChecked( elem ) { - if ( manipulation_rcheckableType.test( elem.type ) ) { - elem.defaultChecked = elem.checked; - } -} - -jQuery.extend({ - clone: function( elem, dataAndEvents, deepDataAndEvents ) { - var destElements, node, clone, i, srcElements, - inPage = jQuery.contains( elem.ownerDocument, elem ); - - if ( jQuery.support.html5Clone || jQuery.isXMLDoc(elem) || !rnoshimcache.test( "<" + elem.nodeName + ">" ) ) { - clone = elem.cloneNode( true ); - - // IE<=8 does not properly clone detached, unknown element nodes - } else { - fragmentDiv.innerHTML = elem.outerHTML; - fragmentDiv.removeChild( clone = fragmentDiv.firstChild ); - } - - if ( (!jQuery.support.noCloneEvent || !jQuery.support.noCloneChecked) && - (elem.nodeType === 1 || elem.nodeType === 11) && !jQuery.isXMLDoc(elem) ) { - - // We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2 - destElements = getAll( clone ); - srcElements = getAll( elem ); - - // Fix all IE cloning issues - for ( i = 0; (node = srcElements[i]) != null; ++i ) { - // Ensure that the destination node is not null; Fixes #9587 - if ( destElements[i] ) { - fixCloneNodeIssues( node, destElements[i] ); - } - } - } - - // Copy the events from the original to the clone - if ( dataAndEvents ) { - if ( deepDataAndEvents ) { - srcElements = srcElements || getAll( elem ); - destElements = destElements || getAll( clone ); - - for ( i = 0; (node = srcElements[i]) != null; i++ ) { - cloneCopyEvent( node, destElements[i] ); - } - } else { - cloneCopyEvent( elem, clone ); - } - } - - // Preserve script evaluation history - destElements = getAll( clone, "script" ); - if ( destElements.length > 0 ) { - setGlobalEval( destElements, !inPage && getAll( elem, "script" ) ); - } - - destElements = srcElements = node = null; - - // Return the cloned set - return clone; - }, - - buildFragment: function( elems, context, scripts, selection ) { - var j, elem, contains, - tmp, tag, tbody, wrap, - l = elems.length, - - // Ensure a safe fragment - safe = createSafeFragment( context ), - - nodes = [], - i = 0; - - for ( ; i < l; i++ ) { - elem = elems[ i ]; - - if ( elem || elem === 0 ) { - - // Add nodes directly - if ( jQuery.type( elem ) === "object" ) { - jQuery.merge( nodes, elem.nodeType ? [ elem ] : elem ); - - // Convert non-html into a text node - } else if ( !rhtml.test( elem ) ) { - nodes.push( context.createTextNode( elem ) ); - - // Convert html into DOM nodes } else { - tmp = tmp || safe.appendChild( context.createElement("div") ); - - // Deserialize a standard representation - tag = ( rtagName.exec( elem ) || ["", ""] )[1].toLowerCase(); - wrap = wrapMap[ tag ] || wrapMap._default; - - tmp.innerHTML = wrap[1] + elem.replace( rxhtmlTag, "<$1>" ) + wrap[2]; - - // Descend through wrappers to the right content - j = wrap[0]; - while ( j-- ) { - tmp = tmp.lastChild; - } - - // Manually add leading whitespace removed by IE - if ( !jQuery.support.leadingWhitespace && rleadingWhitespace.test( elem ) ) { - nodes.push( context.createTextNode( rleadingWhitespace.exec( elem )[0] ) ); - } - - // Remove IE's autoinserted from table fragments - if ( !jQuery.support.tbody ) { - - // String was a , *may* have spurious - elem = tag === "table" && !rtbody.test( elem ) ? - tmp.firstChild : - - // String was a bare or - wrap[1] === "
" && !rtbody.test( elem ) ? - tmp : - 0; - - j = elem && elem.childNodes.length; - while ( j-- ) { - if ( jQuery.nodeName( (tbody = elem.childNodes[j]), "tbody" ) && !tbody.childNodes.length ) { - elem.removeChild( tbody ); - } - } - } - - jQuery.merge( nodes, tmp.childNodes ); - - // Fix #12392 for WebKit and IE > 9 - tmp.textContent = ""; - - // Fix #12392 for oldIE - while ( tmp.firstChild ) { - tmp.removeChild( tmp.firstChild ); - } - - // Remember the top-level container for proper cleanup - tmp = safe.lastChild; - } - } - } - - // Fix #11356: Clear elements from fragment - if ( tmp ) { - safe.removeChild( tmp ); - } - - // Reset defaultChecked for any radios and checkboxes - // about to be appended to the DOM in IE 6/7 (#8060) - if ( !jQuery.support.appendChecked ) { - jQuery.grep( getAll( nodes, "input" ), fixDefaultChecked ); - } - - i = 0; - while ( (elem = nodes[ i++ ]) ) { - - // #4087 - If origin and destination elements are the same, and this is - // that element, do not do anything - if ( selection && jQuery.inArray( elem, selection ) !== -1 ) { - continue; - } - - contains = jQuery.contains( elem.ownerDocument, elem ); - - // Append to fragment - tmp = getAll( safe.appendChild( elem ), "script" ); - - // Preserve script evaluation history - if ( contains ) { - setGlobalEval( tmp ); - } - - // Capture executables - if ( scripts ) { - j = 0; - while ( (elem = tmp[ j++ ]) ) { - if ( rscriptType.test( elem.type || "" ) ) { - scripts.push( elem ); - } - } - } - } - - tmp = null; - - return safe; - }, - - cleanData: function( elems, /* internal */ acceptData ) { - var elem, type, id, data, - i = 0, - internalKey = jQuery.expando, - cache = jQuery.cache, - deleteExpando = jQuery.support.deleteExpando, - special = jQuery.event.special; - - for ( ; (elem = elems[i]) != null; i++ ) { - - if ( acceptData || jQuery.acceptData( elem ) ) { - - id = elem[ internalKey ]; - data = id && cache[ id ]; - - if ( data ) { - if ( data.events ) { - for ( type in data.events ) { - if ( special[ type ] ) { - jQuery.event.remove( elem, type ); - - // This is a shortcut to avoid jQuery.event.remove's overhead - } else { - jQuery.removeEvent( elem, type, data.handle ); - } - } - } - - // Remove cache only if it was not already removed by jQuery.event.remove - if ( cache[ id ] ) { - - delete cache[ id ]; - - // IE does not allow us to delete expando properties from nodes, - // nor does it have a removeAttribute function on Document nodes; - // we must handle all of these cases - if ( deleteExpando ) { - delete elem[ internalKey ]; - - } else if ( typeof elem.removeAttribute !== core_strundefined ) { - elem.removeAttribute( internalKey ); - - } else { - elem[ internalKey ] = null; - } - - core_deletedIds.push( id ); - } - } - } - } - }, - - _evalUrl: function( url ) { - return jQuery.ajax({ - url: url, - type: "GET", - dataType: "script", - async: false, - global: false, - "throws": true - }); - } -}); -jQuery.fn.extend({ - wrapAll: function( html ) { - if ( jQuery.isFunction( html ) ) { - return this.each(function(i) { - jQuery(this).wrapAll( html.call(this, i) ); - }); - } - - if ( this[0] ) { - // The elements to wrap the target around - var wrap = jQuery( html, this[0].ownerDocument ).eq(0).clone(true); - - if ( this[0].parentNode ) { - wrap.insertBefore( this[0] ); - } - - wrap.map(function() { - var elem = this; - - while ( elem.firstChild && elem.firstChild.nodeType === 1 ) { - elem = elem.firstChild; - } - - return elem; - }).append( this ); - } - - return this; - }, - - wrapInner: function( html ) { - if ( jQuery.isFunction( html ) ) { - return this.each(function(i) { - jQuery(this).wrapInner( html.call(this, i) ); - }); - } - - return this.each(function() { - var self = jQuery( this ), - contents = self.contents(); - - if ( contents.length ) { - contents.wrapAll( html ); - - } else { - self.append( html ); - } - }); - }, - - wrap: function( html ) { - var isFunction = jQuery.isFunction( html ); - - return this.each(function(i) { - jQuery( this ).wrapAll( isFunction ? html.call(this, i) : html ); - }); - }, - - unwrap: function() { - return this.parent().each(function() { - if ( !jQuery.nodeName( this, "body" ) ) { - jQuery( this ).replaceWith( this.childNodes ); - } - }).end(); - } -}); -var iframe, getStyles, curCSS, - ralpha = /alpha\([^)]*\)/i, - ropacity = /opacity\s*=\s*([^)]*)/, - rposition = /^(top|right|bottom|left)$/, - // swappable if display is none or starts with table except "table", "table-cell", or "table-caption" - // see here for display values: https://developer.mozilla.org/en-US/docs/CSS/display - rdisplayswap = /^(none|table(?!-c[ea]).+)/, - rmargin = /^margin/, - rnumsplit = new RegExp( "^(" + core_pnum + ")(.*)$", "i" ), - rnumnonpx = new RegExp( "^(" + core_pnum + ")(?!px)[a-z%]+$", "i" ), - rrelNum = new RegExp( "^([+-])=(" + core_pnum + ")", "i" ), - elemdisplay = { BODY: "block" }, - - cssShow = { position: "absolute", visibility: "hidden", display: "block" }, - cssNormalTransform = { - letterSpacing: 0, - fontWeight: 400 - }, - - cssExpand = [ "Top", "Right", "Bottom", "Left" ], - cssPrefixes = [ "Webkit", "O", "Moz", "ms" ]; - -// return a css property mapped to a potentially vendor prefixed property -function vendorPropName( style, name ) { - - // shortcut for names that are not vendor prefixed - if ( name in style ) { - return name; - } - - // check for vendor prefixed names - var capName = name.charAt(0).toUpperCase() + name.slice(1), - origName = name, - i = cssPrefixes.length; - - while ( i-- ) { - name = cssPrefixes[ i ] + capName; - if ( name in style ) { - return name; - } - } - - return origName; -} - -function isHidden( elem, el ) { - // isHidden might be called from jQuery#filter function; - // in that case, element will be second argument - elem = el || elem; - return jQuery.css( elem, "display" ) === "none" || !jQuery.contains( elem.ownerDocument, elem ); -} - -function showHide( elements, show ) { - var display, elem, hidden, - values = [], - index = 0, - length = elements.length; - - for ( ; index < length; index++ ) { - elem = elements[ index ]; - if ( !elem.style ) { - continue; - } - - values[ index ] = jQuery._data( elem, "olddisplay" ); - display = elem.style.display; - if ( show ) { - // Reset the inline display of this element to learn if it is - // being hidden by cascaded rules or not - if ( !values[ index ] && display === "none" ) { - elem.style.display = ""; - } - - // Set elements which have been overridden with display: none - // in a stylesheet to whatever the default browser style is - // for such an element - if ( elem.style.display === "" && isHidden( elem ) ) { - values[ index ] = jQuery._data( elem, "olddisplay", css_defaultDisplay(elem.nodeName) ); - } - } else { - - if ( !values[ index ] ) { - hidden = isHidden( elem ); - - if ( display && display !== "none" || !hidden ) { - jQuery._data( elem, "olddisplay", hidden ? display : jQuery.css( elem, "display" ) ); - } - } - } - } - - // Set the display of most of the elements in a second loop - // to avoid the constant reflow - for ( index = 0; index < length; index++ ) { - elem = elements[ index ]; - if ( !elem.style ) { - continue; - } - if ( !show || elem.style.display === "none" || elem.style.display === "" ) { - elem.style.display = show ? values[ index ] || "" : "none"; - } - } - - return elements; -} - -jQuery.fn.extend({ - css: function( name, value ) { - return jQuery.access( this, function( elem, name, value ) { - var len, styles, - map = {}, - i = 0; - - if ( jQuery.isArray( name ) ) { - styles = getStyles( elem ); - len = name.length; - - for ( ; i < len; i++ ) { - map[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles ); - } - - return map; - } - - return value !== undefined ? - jQuery.style( elem, name, value ) : - jQuery.css( elem, name ); - }, name, value, arguments.length > 1 ); - }, - show: function() { - return showHide( this, true ); - }, - hide: function() { - return showHide( this ); - }, - toggle: function( state ) { - if ( typeof state === "boolean" ) { - return state ? this.show() : this.hide(); - } - - return this.each(function() { - if ( isHidden( this ) ) { - jQuery( this ).show(); - } else { - jQuery( this ).hide(); - } - }); - } -}); - -jQuery.extend({ - // Add in style property hooks for overriding the default - // behavior of getting and setting a style property - cssHooks: { - opacity: { - get: function( elem, computed ) { - if ( computed ) { - // We should always get a number back from opacity - var ret = curCSS( elem, "opacity" ); - return ret === "" ? "1" : ret; - } - } - } - }, - - // Don't automatically add "px" to these possibly-unitless properties - cssNumber: { - "columnCount": true, - "fillOpacity": true, - "fontWeight": true, - "lineHeight": true, - "opacity": true, - "order": true, - "orphans": true, - "widows": true, - "zIndex": true, - "zoom": true - }, - - // Add in properties whose names you wish to fix before - // setting or getting the value - cssProps: { - // normalize float css property - "float": jQuery.support.cssFloat ? "cssFloat" : "styleFloat" - }, - - // Get and set the style property on a DOM Node - style: function( elem, name, value, extra ) { - // Don't set styles on text and comment nodes - if ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) { - return; - } - - // Make sure that we're working with the right name - var ret, type, hooks, - origName = jQuery.camelCase( name ), - style = elem.style; - - name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( style, origName ) ); - - // gets hook for the prefixed version - // followed by the unprefixed version - hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; - - // Check if we're setting a value - if ( value !== undefined ) { - type = typeof value; - - // convert relative number strings (+= or -=) to relative numbers. #7345 - if ( type === "string" && (ret = rrelNum.exec( value )) ) { - value = ( ret[1] + 1 ) * ret[2] + parseFloat( jQuery.css( elem, name ) ); - // Fixes bug #9237 - type = "number"; - } - - // Make sure that NaN and null values aren't set. See: #7116 - if ( value == null || type === "number" && isNaN( value ) ) { - return; - } - - // If a number was passed in, add 'px' to the (except for certain CSS properties) - if ( type === "number" && !jQuery.cssNumber[ origName ] ) { - value += "px"; - } - - // Fixes #8908, it can be done more correctly by specifing setters in cssHooks, - // but it would mean to define eight (for every problematic property) identical functions - if ( !jQuery.support.clearCloneStyle && value === "" && name.indexOf("background") === 0 ) { - style[ name ] = "inherit"; - } - - // If a hook was provided, use that value, otherwise just set the specified value - if ( !hooks || !("set" in hooks) || (value = hooks.set( elem, value, extra )) !== undefined ) { - - // Wrapped to prevent IE from throwing errors when 'invalid' values are provided - // Fixes bug #5509 - try { - style[ name ] = value; - } catch(e) {} - } - - } else { - // If a hook was provided get the non-computed value from there - if ( hooks && "get" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) { - return ret; - } - - // Otherwise just get the value from the style object - return style[ name ]; - } - }, - - css: function( elem, name, extra, styles ) { - var num, val, hooks, - origName = jQuery.camelCase( name ); - - // Make sure that we're working with the right name - name = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( elem.style, origName ) ); - - // gets hook for the prefixed version - // followed by the unprefixed version - hooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ]; - - // If a hook was provided get the computed value from there - if ( hooks && "get" in hooks ) { - val = hooks.get( elem, true, extra ); - } - - // Otherwise, if a way to get the computed value exists, use that - if ( val === undefined ) { - val = curCSS( elem, name, styles ); - } - - //convert "normal" to computed value - if ( val === "normal" && name in cssNormalTransform ) { - val = cssNormalTransform[ name ]; - } - - // Return, converting to number if forced or a qualifier was provided and val looks numeric - if ( extra === "" || extra ) { - num = parseFloat( val ); - return extra === true || jQuery.isNumeric( num ) ? num || 0 : val; - } - return val; - } -}); - -// NOTE: we've included the "window" in window.getComputedStyle -// because jsdom on node.js will break without it. -if ( window.getComputedStyle ) { - getStyles = function( elem ) { - return window.getComputedStyle( elem, null ); - }; - - curCSS = function( elem, name, _computed ) { - var width, minWidth, maxWidth, - computed = _computed || getStyles( elem ), - - // getPropertyValue is only needed for .css('filter') in IE9, see #12537 - ret = computed ? computed.getPropertyValue( name ) || computed[ name ] : undefined, - style = elem.style; - - if ( computed ) { - - if ( ret === "" && !jQuery.contains( elem.ownerDocument, elem ) ) { - ret = jQuery.style( elem, name ); - } - - // A tribute to the "awesome hack by Dean Edwards" - // Chrome < 17 and Safari 5.0 uses "computed value" instead of "used value" for margin-right - // Safari 5.1.7 (at least) returns percentage for a larger set of values, but width seems to be reliably pixels - // this is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values - if ( rnumnonpx.test( ret ) && rmargin.test( name ) ) { - - // Remember the original values - width = style.width; - minWidth = style.minWidth; - maxWidth = style.maxWidth; - - // Put in the new values to get a computed value out - style.minWidth = style.maxWidth = style.width = ret; - ret = computed.width; - - // Revert the changed values - style.width = width; - style.minWidth = minWidth; - style.maxWidth = maxWidth; - } - } - - return ret; - }; -} else if ( document.documentElement.currentStyle ) { - getStyles = function( elem ) { - return elem.currentStyle; - }; - - curCSS = function( elem, name, _computed ) { - var left, rs, rsLeft, - computed = _computed || getStyles( elem ), - ret = computed ? computed[ name ] : undefined, - style = elem.style; - - // Avoid setting ret to empty string here - // so we don't default to auto - if ( ret == null && style && style[ name ] ) { - ret = style[ name ]; - } - - // From the awesome hack by Dean Edwards - // http://erik.eae.net/archives/2007/07/27/18.54.15/#comment-102291 - - // If we're not dealing with a regular pixel number - // but a number that has a weird ending, we need to convert it to pixels - // but not position css attributes, as those are proportional to the parent element instead - // and we can't measure the parent instead because it might trigger a "stacking dolls" problem - if ( rnumnonpx.test( ret ) && !rposition.test( name ) ) { - - // Remember the original values - left = style.left; - rs = elem.runtimeStyle; - rsLeft = rs && rs.left; - - // Put in the new values to get a computed value out - if ( rsLeft ) { - rs.left = elem.currentStyle.left; - } - style.left = name === "fontSize" ? "1em" : ret; - ret = style.pixelLeft + "px"; - - // Revert the changed values - style.left = left; - if ( rsLeft ) { - rs.left = rsLeft; - } - } - - return ret === "" ? "auto" : ret; - }; -} - -function setPositiveNumber( elem, value, subtract ) { - var matches = rnumsplit.exec( value ); - return matches ? - // Guard against undefined "subtract", e.g., when used as in cssHooks - Math.max( 0, matches[ 1 ] - ( subtract || 0 ) ) + ( matches[ 2 ] || "px" ) : - value; -} - -function augmentWidthOrHeight( elem, name, extra, isBorderBox, styles ) { - var i = extra === ( isBorderBox ? "border" : "content" ) ? - // If we already have the right measurement, avoid augmentation - 4 : - // Otherwise initialize for horizontal or vertical properties - name === "width" ? 1 : 0, - - val = 0; - - for ( ; i < 4; i += 2 ) { - // both box models exclude margin, so add it if we want it - if ( extra === "margin" ) { - val += jQuery.css( elem, extra + cssExpand[ i ], true, styles ); - } - - if ( isBorderBox ) { - // border-box includes padding, so remove it if we want content - if ( extra === "content" ) { - val -= jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); - } - - // at this point, extra isn't border nor margin, so remove border - if ( extra !== "margin" ) { - val -= jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); - } - } else { - // at this point, extra isn't content, so add padding - val += jQuery.css( elem, "padding" + cssExpand[ i ], true, styles ); - - // at this point, extra isn't content nor padding, so add border - if ( extra !== "padding" ) { - val += jQuery.css( elem, "border" + cssExpand[ i ] + "Width", true, styles ); - } - } - } - - return val; -} - -function getWidthOrHeight( elem, name, extra ) { - - // Start with offset property, which is equivalent to the border-box value - var valueIsBorderBox = true, - val = name === "width" ? elem.offsetWidth : elem.offsetHeight, - styles = getStyles( elem ), - isBorderBox = jQuery.support.boxSizing && jQuery.css( elem, "boxSizing", false, styles ) === "border-box"; - - // some non-html elements return undefined for offsetWidth, so check for null/undefined - // svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285 - // MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668 - if ( val <= 0 || val == null ) { - // Fall back to computed then uncomputed css if necessary - val = curCSS( elem, name, styles ); - if ( val < 0 || val == null ) { - val = elem.style[ name ]; - } - - // Computed unit is not pixels. Stop here and return. - if ( rnumnonpx.test(val) ) { - return val; - } - - // we need the check for style in case a browser which returns unreliable values - // for getComputedStyle silently falls back to the reliable elem.style - valueIsBorderBox = isBorderBox && ( jQuery.support.boxSizingReliable || val === elem.style[ name ] ); - - // Normalize "", auto, and prepare for extra - val = parseFloat( val ) || 0; - } - - // use the active box-sizing model to add/subtract irrelevant styles - return ( val + - augmentWidthOrHeight( - elem, - name, - extra || ( isBorderBox ? "border" : "content" ), - valueIsBorderBox, - styles - ) - ) + "px"; -} - -// Try to determine the default display value of an element -function css_defaultDisplay( nodeName ) { - var doc = document, - display = elemdisplay[ nodeName ]; - - if ( !display ) { - display = actualDisplay( nodeName, doc ); - - // If the simple way fails, read from inside an iframe - if ( display === "none" || !display ) { - // Use the already-created iframe if possible - iframe = ( iframe || - jQuery("