true if the key is deprecated and
+ false otherwise.]]>
+ null if
+ no such property exists. If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null.
+
+ Values are processed for variable expansion
+ before being returned.
+
+ As a side effect get loads the properties from the sources if called for
+ the first time as a lazy init.
+
+ @param name the property name, will be trimmed before get value.
+ @return the value of the name or its replacing property,
+ or null if no such property exists.]]>
+ name exists without value]]>
+ String,
+ null if no such property exists.
+ If the key is deprecated, it returns the value of
+ the first key which replaces the deprecated key and is not null
+
+ Values are processed for variable expansion
+ before being returned.
+
+ @param name the property name.
+ @return the value of the name or its replacing property,
+ or null if no such property exists.]]>
+ String,
+ defaultValue if no such property exists.
+ See @{Configuration#getTrimmed} for more details.
+
+ @param name the property name.
+ @param defaultValue the property default value.
+ @return the value of the name or defaultValue
+ if it is not set.]]>
+ name property or
+ its replacing property and null if no such property exists.]]>
+ name property. If
+ name is deprecated or there is a deprecated name associated to it,
+ it sets the value to both names. Name will be trimmed before put into
+ configuration.
+
+ @param name property name.
+ @param value property value.]]>
+ name property. If
+ name is deprecated, it also sets the value to
+ the keys that replace the deprecated key. Name will be trimmed before put
+ into configuration.
+
+ @param name property name.
+ @param value property value.
+ @param source the place that this configuration value came from
+ (For debugging).
+ @throws IllegalArgumentException when the value or name is null.]]>
+ defaultValue is returned.
+
+ @param name property name, will be trimmed before get value.
+ @param defaultValue default value.
+ @return property value, or defaultValue if the property
+ doesn't exist.]]>
+ int.
+
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid int,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as an int,
+ or defaultValue.]]>
+ int values.
+
+ If no such property exists, an empty array is returned.
+
+ @param name property name
+ @return property value interpreted as an array of comma-delimited
+ int values]]>
+ int.
+
+ @param name property name.
+ @param value int value of the property.]]>
+ long.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid long,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a long,
+ or defaultValue.]]>
+ long or
+ human readable format. If no such property exists, the provided default
+ value is returned, or if the specified value is not a valid
+ long or human readable format, then an error is thrown. You
+ can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ t(tera), p(peta), e(exa)
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a long,
+ or defaultValue.]]>
+ long.
+
+ @param name property name.
+ @param value long value of the property.]]>
+ float.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid float,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a float,
+ or defaultValue.]]>
+ float.
+
+ @param name property name.
+ @param value property value.]]>
+ double.
+ If no such property exists, the provided default value is returned,
+ or if the specified value is not a valid double,
+ then an error is thrown.
+
+ @param name property name.
+ @param defaultValue default value.
+ @throws NumberFormatException when the value is invalid
+ @return property value as a double,
+ or defaultValue.]]>
+ double.
+
+ @param name property name.
+ @param value property value.]]>
+ boolean.
+ If no such property is specified, or if the specified value is not a valid
+ boolean, then defaultValue is returned.
+
+ @param name property name.
+ @param defaultValue default value.
+ @return property value as a boolean,
+ or defaultValue.]]>
+ boolean.
+
+ @param name property name.
+ @param value boolean value of the property.]]>
+ set(<name>, value.toString()).
+ @param name property name
+ @param value new value
+ @param set(<name>, value + <time suffix>).
+ @param name Property name
+ @param value Time duration
+ @param unit Unit of time]]>
+ Pattern.
+ If no such property is specified, or if the specified value is not a valid
+ Pattern, then DefaultValue is returned.
+ Note that the returned value is NOT trimmed by this method.
+
+ @param name property name
+ @param defaultValue default value
+ @return property value as a compiled Pattern, or defaultValue]]>
+ Strings.
+ If no such property is specified then empty collection is returned.
+
+ This is an optimized version of {@link #getStrings(String)}
+
+ @param name property name.
+ @return property value as a collection of Strings.]]>
+
Strings.
+ If no such property is specified then null is returned.
+
+ @param name property name.
+ @return property value as an array of Strings,
+ or null.]]>
+ Strings.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of Strings,
+ or default value.]]>
+ Strings, trimmed of the leading and trailing whitespace.
+ If no such property is specified then empty Collection is returned.
+
+ @param name property name.
+ @return property value as a collection of Strings, or empty Collection]]>
+ Strings, trimmed of the leading and trailing whitespace.
+ If no such property is specified then an empty array is returned.
+
+ @param name property name.
+ @return property value as an array of trimmed Strings,
+ or empty array.]]>
+ Strings, trimmed of the leading and trailing whitespace.
+ If no such property is specified then default value is returned.
+
+ @param name property name.
+ @param defaultValue The default value
+ @return property value as an array of trimmed Strings,
+ or default value.]]>
+ InetSocketAddress. If hostProperty is
+ null, addressProperty will be used. This
+ is useful for cases where we want to differentiate between host
+ bind address and address clients should use to establish connection.
+
+ @param hostProperty bind host property name.
+ @param addressProperty address property name.
+ @param defaultAddressValue the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ InetSocketAddress.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ host:port.
+ @param name property name.
+ @param addr inetSocketAddress addr.]]>
+ host:port. The wildcard
+ address is replaced with the local host's address. If the host and address
+ properties are configured the host component of the address will be combined
+ with the port component of the addr to generate the address. This is to allow
+ optional control over which host name is used in multi-home bind-host
+ cases where a host can have multiple names
+ @param hostProperty the bind-host configuration name
+ @param addressProperty the service address configuration name
+ @param defaultAddressValue the service default address configuration value
+ @param addr InetSocketAddress of the service listener
+ @return InetSocketAddress for clients to connect]]>
+ host:port. The wildcard
+ address is replaced with the local host's address.
+ @param name property name.
+ @param addr InetSocketAddress of a listener to store in the given property
+ @return InetSocketAddress for clients to connect]]>
+ Class.
+ The value of the property specifies a list of comma separated class names.
+ If no such property is specified, then defaultValue is
+ returned.
+
+ @param name the property name.
+ @param defaultValue default value.
+ @return property value as a Class[],
+ or defaultValue.]]>
+ Class.
+ If no such property is specified, then defaultValue is
+ returned.
+
+ @param name the conf key name.
+ @param defaultValue default value.
+ @return property value as a Class,
+ or defaultValue.]]>
+ Class
+ implementing the interface specified by xface.
+
+ If no such property is specified, then defaultValue is
+ returned.
+
+ An exception is thrown if the returned class does not implement the named
+ interface.
+
+ @param name the conf key name.
+ @param defaultValue default value.
+ @param xface the interface implemented by the named class.
+ @param Interface class type.
+ @return property value as a Class,
+ or defaultValue.]]>
+ List
+ of objects implementing the interface specified by xface.
+
+ An exception is thrown if any of the classes does not exist, or if it does
+ not implement the named interface.
+
+ @param name the property name.
+ @param xface the interface implemented by the classes named by
+ name.
+ @param Interface class type.
+ @return a List of objects implementing xface.]]>
+ theClass implementing the given interface xface.
+
+ An exception is thrown if theClass does not implement the
+ interface xface.
+
+ @param name property name.
+ @param theClass property value.
+ @param xface the interface implemented by the named class.]]>
+
+ {
+ "property": {
+ "key" : "key1",
+ "value" : "value1",
+ "isFinal" : "key1.isFinal",
+ "resource" : "key1.resource"
+ }
+ }
+
+
+ { "properties" :
+ [ { key : "key1",
+ value : "value1",
+ isFinal : "key1.isFinal",
+ resource : "key1.resource" },
+ { key : "key2",
+ value : "value2",
+ isFinal : "ke2.isFinal",
+ resource : "key2.resource" }
+ ]
+ }
+
+ + @param config the configuration + @param propertyName property name + @param out the Writer to write to + @throws IOException raised on errors performing I/O. + @throws IllegalArgumentException when property name is not + empty and the property is not found in configuration]]> +
+ + @param config the configuration + @param out the Writer to write to + @throws IOException raised on errors performing I/O.]]> +
false
+ to turn it off.]]>
+ Configurations are specified by resources. A resource contains a set of
+ name/value pairs as XML data. Each resource is named by either a
+ String or by a {@link Path}. If named by a String,
+ then the classpath is examined for a file with that name. If named by a
+ Path, then the local filesystem is examined directly, without
+ referring to the classpath.
+
+
Unless explicitly turned off, Hadoop by default specifies two + resources, loaded in-order from the classpath:
Configuration parameters may be declared final. + Once a resource declares a value final, no subsequently-loaded + resource can alter that value. + For example, one might define a final parameter with: +
+ <property>
+ <name>dfs.hosts.include</name>
+ <value>/etc/hadoop/conf/hosts.include</value>
+ <final>true</final>
+ </property>Value strings are first processed for variable expansion. The + available properties are:
For example, if a configuration resource contains the following property + definitions: +
+ <property>
+ <name>basedir</name>
+ <value>/user/${user.name}</value>
+ </property>
+
+ <property>
+ <name>tempdir</name>
+ <value>${basedir}/tmp</value>
+ </property>
+
+ <property>
+ <name>otherdir</name>
+ <value>${env.BASE_DIR}/other</value>
+ </property>
+ When conf.get("tempdir") is called, then ${basedir} + will be resolved to another property in this Configuration, while + ${user.name} would then ordinarily be resolved to the value + of the System property with that name. +
When conf.get("otherdir") is called, then ${env.BASE_DIR} + will be resolved to the value of the ${BASE_DIR} environment variable. + It supports ${env.NAME:-default} and ${env.NAME-default} notations. + The former is resolved to "default" if ${NAME} environment variable is undefined + or its value is empty. + The latter behaves the same way only if ${NAME} is undefined. +
By default, warnings will be given to any deprecated configuration + parameters and these are suppressible by configuring + log4j.logger.org.apache.hadoop.conf.Configuration.deprecation in + log4j.properties file. + +
Optionally we can tag related properties together by using tag + attributes. System tags are defined by hadoop.tags.system property. Users + can define there own custom tags in hadoop.tags.custom property. + +
For example, we can tag existing property as: +
+ <property>
+ <name>dfs.replication</name>
+ <value>3</value>
+ <tag>HDFS,REQUIRED</tag>
+ </property>
+
+ <property>
+ <name>dfs.data.transfer.protection</name>
+ <value>3</value>
+ <tag>HDFS,SECURITY</tag>
+ </property>
+ Properties marked with tags can be retrieved with conf + .getAllPropertiesByTag("HDFS") or conf.getAllPropertiesByTags + (Arrays.asList("YARN","SECURITY")).
]]> +KeyProvider implementations must be thread safe.]]>
+ uri is not supported.]]>
+
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+ "host2:9866","host3:9866","host4:9866","host5:9866"})
+
+
+ Please refer to
+ {@link FileSystem#getFileBlockLocations(FileStatus, long, long)} or
+ {@link FileContext#getFileBlockLocations(Path, long, long)}
+ for more examples.]]>
+ + In the case of an exception, the state of the buffer (the contents of the + buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is + undefined, and callers should be prepared to recover from this + eventuality. +
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with + {@link StreamCapabilities#PREADBYTEBUFFER} to check if the underlying + stream supports this interface, otherwise they might get a + {@link UnsupportedOperationException}. +
+ Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. +
+ This does not change the current offset of a file, and is thread-safe. + + @param position position within file + @param buf the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if reached + end-of-stream + @throws IOException if there is some error performing the read]]> +
+ In the case of an exception, the state of the buffer (the contents of the + buffer, the {@code buf.position()}, the {@code buf.limit()}, etc.) is + undefined, and callers should be prepared to recover from this + eventuality. +
+ Callers should use {@link StreamCapabilities#hasCapability(String)} with + {@link StreamCapabilities#READBYTEBUFFER} to check if the underlying + stream supports this interface, otherwise they might get a + {@link UnsupportedOperationException}. +
+ Implementations should treat 0-length requests as legitimate, and must not + signal an error upon their receipt. + + @param buf + the ByteBuffer to receive the results of the read operation. + @return the number of bytes read, possibly zero, or -1 if + reach end-of-stream + @throws IOException + if there is some error performing the read]]> +
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.APPEND)
+
+ + + Use the CreateFlag as follows: +
absOrFqPath could
+ not be instantiated.]]>
+ f is not valid]]>
+ f already exists
+ @throws FileNotFoundException If parent of f does not exist
+ and createParent is false
+ @throws ParentNotDirectoryException If parent of f is not a
+ directory.
+ @throws UnsupportedFileSystemException If file system for f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+
+ RuntimeExceptions:
+ @throws InvalidPathException If path f is not valid]]>
+ dir does not exist
+ and createParent is false
+ @throws ParentNotDirectoryException If parent of dir is not a
+ directory
+ @throws UnsupportedFileSystemException If file system for dir
+ is not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+
+ RuntimeExceptions:
+ @throws InvalidPathException If path dir is not valid]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+
+ RuntimeExceptions:
+ @throws InvalidPathException If path f is invalid
+
+ @return if delete success true, not false.]]>
+ f
+ is not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+ @return input stream.]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+ @return output stream.]]>
+ true if the file has been truncated to the desired
+ newLength and is immediately available to be reused for
+ write operations such as append, or
+ false if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+
+ @throws AccessControlException If access is denied
+ @throws FileNotFoundException If file f does not exist
+ @throws UnsupportedFileSystemException If file system for f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ + If OVERWRITE option is not passed as an argument, rename fails if the dst + already exists. +
+ If OVERWRITE option is passed as an argument, rename overwrites the dst if + it is a file or an empty directory. Rename fails if dst is a non-empty + directory. +
+ Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for details +
+
+ @param src path to be renamed
+ @param dst new path after rename
+ @param options rename options.
+
+ @throws AccessControlException If access is denied
+ @throws FileAlreadyExistsException If dst already exists and
+ options has {@link Options.Rename#OVERWRITE}
+ option false.
+ @throws FileNotFoundException If src does not exist
+ @throws ParentNotDirectoryException If parent of dst is not a
+ directory
+ @throws UnsupportedFileSystemException If file system for src
+ and dst is not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+
f
+ is not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server
+
+ RuntimeExceptions:
+ @throws HadoopIllegalArgumentException If username or
+ groupname is invalid.]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred]]>
+ f is
+ not supported
+ @throws IOException If the given path does not refer to a symlink
+ or an I/O error occurred]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ link already exists
+ @throws FileNotFoundException If target does not exist
+ @throws ParentNotDirectoryException If parent of link is not a
+ directory.
+ @throws UnsupportedFileSystemException If file system for
+ target or link is not supported
+ @throws IOException If an I/O error occurred]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ f is
+ not supported
+ @throws IOException If an I/O error occurred
+
+ Exceptions applicable to file systems accessed over RPC:
+ @throws RpcClientException If an exception occurred in the RPC client
+ @throws RpcServerException If an exception occurred in the RPC server
+ @throws UnexpectedServerException If server implementation throws
+ undeclared exception to RPC server]]>
+ + + Hadoop also supports working-directory-relative names, which are paths + relative to the current working directory (similar to Unix). The working + directory can be in a different file system than the default FS. +
+ Thus, Hadoop path names can be specified as one of the following: +
+ The file system related server-side defaults are: +
UnsupportedOperationException.
+
+ @return the protocol scheme for this FileSystem.
+ @throws UnsupportedOperationException if the operation is unsupported
+ (default).]]>
+
+ BlockLocation( { "localhost:9866" }, { "localhost" }, 0, file.getLen())
+
+
+ In HDFS, if file is three-replicated, the returned array contains
+ elements like:
+
+ BlockLocation(offset: 0, length: BLOCK_SIZE,
+ hosts: {"host1:9866", "host2:9866, host3:9866"})
+ BlockLocation(offset: BLOCK_SIZE, length: BLOCK_SIZE,
+ hosts: {"host2:9866", "host3:9866, host4:9866"})
+
+
+ And if a file is erasure-coded, the returned BlockLocation are logical
+ block groups.
+
+ Suppose we have a RS_3_2 coded file (3 data units and 2 parity units).
+ 1. If the file size is less than one stripe size, say 2 * CELL_SIZE, then
+ there will be one BlockLocation returned, with 0 offset, actual file size
+ and 4 hosts (2 data blocks and 2 parity blocks) hosting the actual blocks.
+ 3. If the file size is less than one group size but greater than one
+ stripe size, then there will be one BlockLocation returned, with 0 offset,
+ actual file size with 5 hosts (3 data blocks and 2 parity blocks) hosting
+ the actual blocks.
+ 4. If the file size is greater than one group size, 3 * BLOCK_SIZE + 123
+ for example, then the result will be like:
+
+ BlockLocation(offset: 0, length: 3 * BLOCK_SIZE, hosts: {"host1:9866",
+ "host2:9866","host3:9866","host4:9866","host5:9866"})
+ BlockLocation(offset: 3 * BLOCK_SIZE, length: 123, hosts: {"host1:9866",
+ "host4:9866", "host5:9866"})
+
+
+ @param file FilesStatus to get data from
+ @param start offset into the given file
+ @param len length for which to get locations for
+ @throws IOException IO failure
+ @return block location array.]]>
+ + If OVERWRITE option is not passed as an argument, rename fails + if the dst already exists. +
++ If OVERWRITE option is passed as an argument, rename overwrites + the dst if it is a file or an empty directory. Rename fails if dst is + a non-empty directory. +
+ Note that atomicity of rename is dependent on the file system + implementation. Please refer to the file system documentation for + details. This default implementation is non atomic. ++ This method is deprecated since it is a temporary method added to + support the transition from FileSystem to FileContext for user + applications. +
+ + @param src path to be renamed + @param dst new path after rename + @param options rename options. + @throws FileNotFoundException src path does not exist, or the parent + path of dst does not exist. + @throws FileAlreadyExistsException dest path exists and is a file + @throws ParentNotDirectoryException if the parent path of dest is not + a directory + @throws IOException on failure]]> +true if the file has been truncated to the desired
+ newLength and is immediately available to be reused for
+ write operations such as append, or
+ false if a background process of adjusting the length of
+ the last block has been started, and clients should wait for it to
+ complete before proceeding with further file updates.
+ @throws IOException IO failure
+ @throws UnsupportedOperationException if the operation is unsupported
+ (default).]]>
+ + Will not return null. Expect IOException upon access error. + @param f given path + @return the statuses of the files/directories in the given patch + @throws FileNotFoundException when the path does not exist + @throws IOException see specific implementation]]> +
+ A filename pattern is composed of regular characters and + special pattern matching characters, which are: + +
+ The local implementation is {@link LocalFileSystem} and distributed + implementation is DistributedFileSystem. There are other implementations + for object stores and (outside the Apache Hadoop codebase), + third party filesystems. +
+ Notes ++ Important note for developers +
+ If you are making changes here to the public API or protected methods, + you must review the following subclasses and make sure that + they are filtering/passing through new methods as appropriate. + + {@link FilterFileSystem}: methods are passed through. If not, + then {@code TestFilterFileSystem.MustNotImplement} must be + updated with the unsupported interface. + Furthermore, if the new API's support is probed for via + {@link #hasPathCapability(Path, String)} then + {@link FilterFileSystem#hasPathCapability(Path, String)} + must return false, always. ++ {@link ChecksumFileSystem}: checksums are created and + verified. +
+ {@code TestHarFileSystem} will need its {@code MustNotImplement} + interface updated. + ++ There are some external places your changes will break things. + Do co-ordinate changes here. +
+ + HBase: HBoss ++ Hive: HiveShim23 +
+ {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}]]> ++ The operation is recursive, and the deleteSource operation takes place + as each subdirectory is copied. Therefore, if an operation fails partway + through, the source tree may be partially deleted. +
+ @param srcFS source filesystem + @param srcStatus status of source + @param dstFS destination filesystem + @param dst path of source + @param deleteSource delete the source? + @param overwrite overwrite files at destination? + @param conf configuration to use when opening files + @return true if the operation succeeded. + @throws IOException failure]]> +FilterFileSystem
+ itself simply overrides all methods of
+ FileSystem with versions that
+ pass all requests to the contained file
+ system. Subclasses of FilterFileSystem
+ may further override some of these methods
+ and may also provide additional methods
+ and fields.]]>
+ file]]>
+ pathname
+ should be included]]>
+ + If a file is changed while the readVectored() operation is in progress, the output is + undefined. Some ranges may have old data, some may have new and some may have both. +
++ While a readVectored() operation is in progress, normal read api calls may block. +
+ @param ranges the byte ranges to read + @param allocate the function to allocate ByteBuffer + @throws IOException any IOE.]]> +ftp]]>
+ + Consult the filesystem specification document for the requirements + of an implementation of this interface.]]> +
+ This is designed to affordable to use in log statements. + @param source source of statistics -may be null. + @return an object whose toString() operation returns the current values.]]> +
+ This is for use in log statements where for the cost of creation + of this entry is low; it is affordable to use in log statements. + @param statistics statistics to stringify -may be null. + @return an object whose toString() operation returns the current values.]]> +
+ It is annotated for correct serializations with jackson2. +
]]> ++ The instance can be serialized, and its + {@code toString()} method lists all the values. + @param statistics statistics + @return a snapshot of the current values.]]> +
+ If a statistic has 0 samples then it is considered to be empty. +
++ All 'empty' statistics are equivalent, independent of the sum value. +
++ For non-empty statistics, sum and sample values must match + for equality. +
++ It is serializable and annotated for correct serializations with jackson2. +
++ Thread safety. The operations to add/copy sample data, are thread safe. +
++ So is the {@link #mean()} method. This ensures that when + used to aggregated statistics, the aggregate value and sample + count are set and evaluated consistently. +
++ Other methods marked as synchronized because Findbugs overreacts + to the idea that some operations to update sum and sample count + are synchronized, but that things like equals are not. +
]]> ++ Fencing is configured by the operator as an ordered list of methods to + attempt. Each method will be tried in turn, and the next in the list + will only be attempted if the previous one fails. See {@link NodeFencer} + for more information. +
+ If an implementation also implements {@link Configurable} then its
+ setConf method will be called upon instantiation.]]>
+
+ Compared with ObjectWritable, this class is much more effective,
+ because ObjectWritable will append the class declaration as a String
+ into the output file in every Key-Value pair.
+
+ Generic Writable implements {@link Configurable} interface, so that it will be + configured by the framework. The configuration is passed to the wrapped objects + implementing {@link Configurable} interface before deserialization. +
+ + how to use it:getTypes(), defines
+ the classes which will be wrapped in GenericObject in application.
+ Attention: this classes defined in getTypes() method, must
+ implement Writable interface.
+
+ public class GenericObject extends GenericWritable {
+
+ private static Class[] CLASSES = {
+ ClassType1.class,
+ ClassType2.class,
+ ClassType3.class,
+ };
+
+ protected Class[] getTypes() {
+ return CLASSES;
+ }
+
+ }
+ data file,
+ containing all keys and values in the map, and a smaller index
+ file, containing a fraction of the keys. The fraction is determined by
+ {@link Writer#getIndexInterval()}.
+
+ The index file is read entirely into memory. Thus key implementations + should try to keep themselves small. + +
Map files are created by adding entries in-order. To maintain a large + database, perform updates by copying the previous version of a database and + merging in a sorted change list, to create a new version of the database in + a new file. Sorting large change lists can be done with {@link + SequenceFile.Sorter}.]]> +
SequenceFile provides {@link SequenceFile.Writer},
+ {@link SequenceFile.Reader} and {@link Sorter} classes for writing,
+ reading and sorting respectively.
SequenceFile Writers based on the
+ {@link CompressionType} used to compress key/value pairs:
+ Writer : Uncompressed records.
+ RecordCompressWriter : Record-compressed files, only compress
+ values.
+ BlockCompressWriter : Block-compressed files, both keys &
+ values are collected in 'blocks'
+ separately and compressed. The size of
+ the 'block' is configurable.
+ The actual compression algorithm used to compress key and/or values can be + specified by using the appropriate {@link CompressionCodec}.
+ +The recommended way is to use the static createWriter methods
+ provided by the SequenceFile to chose the preferred format.
The {@link SequenceFile.Reader} acts as the bridge and can read any of the
+ above SequenceFile formats.
Essentially there are 3 different formats for SequenceFiles
+ depending on the CompressionType specified. All of them share a
+ common header described below.
+
+
CompressionCodec class which is used for
+ compression of keys and/or values (if compression is
+ enabled).
+ 100 kilobytes or so.
+ 100 kilobytes or so.
+ The compressed blocks of key lengths and value lengths consist of the + actual lengths of individual keys/values encoded in ZeroCompressedInteger + format.
+ + @see CompressionCodec]]> +start. The starting
+ position is measured in bytes and the return value is in
+ terms of byte position in the buffer. The backing buffer is
+ not converted to a string for this operation.
+
+ @param what input what.
+ @param start input start.
+ @return byte position of the first occurrence of the search
+ string in the UTF-8 buffer or -1 if not found]]>
+ new byte[0]).]]>
+ Also includes utilities for + serializing/deserialing a string, coding/decoding a string, checking if a + byte array contains valid UTF8 code, calculating the length of an encoded + string.]]> +
DataOuput to serialize this object into.
+ @throws IOException any other problem for write.]]>
+ For efficiency, implementations should attempt to re-use storage in the + existing object where possible.
+ + @param inDataInput to deseriablize this object from.
+ @throws IOException any other problem for readFields.]]>
+ key or value type in the Hadoop Map-Reduce
+ framework implements this interface.
+
+ Implementations typically implement a static read(DataInput)
+ method which constructs a new instance, calls {@link #readFields(DataInput)}
+ and returns the instance.
Example:
+
+ public class MyWritable implements Writable {
+ // Some data
+ private int counter;
+ private long timestamp;
+
+ // Default constructor to allow (de)serialization
+ MyWritable() { }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(counter);
+ out.writeLong(timestamp);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ counter = in.readInt();
+ timestamp = in.readLong();
+ }
+
+ public static MyWritable read(DataInput in) throws IOException {
+ MyWritable w = new MyWritable();
+ w.readFields(in);
+ return w;
+ }
+ }
+ WritableComparables can be compared to each other, typically
+ via Comparators. Any type which is to be used as a
+ key in the Hadoop Map-Reduce framework should implement this
+ interface.
+
+ Note that hashCode() is frequently used in Hadoop to partition
+ keys. It's important that your implementation of hashCode() returns the same
+ result across different instances of the JVM. Note also that the default
+ hashCode() implementation in Object does not
+ satisfy this property.
Example:
+
+ public class MyWritableComparable implements
+ WritableComparable{@literal } {
+ // Some data
+ private int counter;
+ private long timestamp;
+
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(counter);
+ out.writeLong(timestamp);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ counter = in.readInt();
+ timestamp = in.readLong();
+ }
+
+ public int compareTo(MyWritableComparable o) {
+ int thisValue = this.value;
+ int thatValue = o.value;
+ return (thisValue < thatValue ? -1 : (thisValue==thatValue ? 0 : 1));
+ }
+
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + counter;
+ result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
+ return result
+ }
+ }
+ One may optimize compare-intensive operations by overriding + {@link #compare(byte[],int,int,byte[],int,int)}. Static utility methods are + provided to assist in optimized implementations of this method.]]> +
Compressor
+ @param conf the Configuration object which contains confs for creating or reinit the compressor
+ @return Compressor for the given
+ CompressionCodec from the pool or a new one]]>
+ Decompressor
+ @return Decompressor for the given
+ CompressionCodec the pool or a new one]]>
+ + The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec object]]> +
+ The code alias is the short class name (without the package name). + If the short class name ends with 'Codec', then there are two aliases for + the codec, the complete short class name and the short class name without + the 'Codec' ending. For example for the 'GzipCodec' codec class name the + alias are 'gzip' and 'gzipcodec'. + + @param codecName the canonical class name of the codec + @return the codec class]]> +
b[] remain unmodified until
+ the caller is explicitly notified--via {@link #needsInput()}--that the
+ buffer may be safely modified. With this requirement, an extra
+ buffer-copy can be avoided.)
+
+ @param b Input data
+ @param off Start offset
+ @param len Length]]>
+ true if the input data buffer is empty and
+ {@link #setInput(byte[], int, int)} should be called in
+ order to provide more input.]]>
+ true if a preset dictionary is needed for decompression]]>
+ true and {@link #getRemaining()}
+ returns a positive value. finished() will be reset with the
+ {@link #reset()} method.
+ @return true if the end of the decompressed
+ data output stream has been reached.]]>
+ true and getRemaining() returns
+ a zero value, indicates that the end of data stream has been reached and
+ is not a concatenated data stream.
+ @return The number of bytes remaining in the compressed data buffer.]]>
+ false when reset() is called.]]>
+ + The behavior of TFile can be customized by the following variables through + Configuration: +
+ Suggestions on performance optimization. +
+To add a new serialization framework write an implementation of +{@link org.apache.hadoop.io.serializer.Serialization} and add its name to the +"io.serializations" property. +
]]> ++Use {@link org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization} for +serialization of classes generated by Avro's 'specific' compiler. +
+ ++Use {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} for +other classes. +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization} work for +any class which is either in the package list configured via +{@link org.apache.hadoop.io.serializer.avro.AvroReflectSerialization#AVRO_REFLECT_PACKAGES} +or implement {@link org.apache.hadoop.io.serializer.avro.AvroReflectSerializable} +interface. +
]]> +basepath property. All metrics will be logged to a file in the
+ current interval's directory in a file named <hostname>.log, where
+ <hostname> is the name of the host on which the metrics logging
+ process is running. The base path is set by the
+ <prefix>.sink.<instance>.basepath property. The
+ time zone used to create the current interval's directory name is GMT. If
+ the basepath property isn't specified, it will default to
+ "/tmp", which is the temp directory on whatever default file
+ system is configured for the cluster.
+
+ The <prefix>.sink.<instance>.ignore-error
+ property controls whether an exception is thrown when an error is encountered
+ writing a log file. The default value is true. When set to
+ false, file errors are quietly swallowed.
The roll-interval property sets the amount of time before
+ rolling the directory. The default value is 1 hour. The roll interval may
+ not be less than 1 minute. The property's value should be given as
+ number unit, where number is an integer value, and
+ unit is a valid unit. Valid units are minute, hour,
+ and day. The units are case insensitive and may be abbreviated or
+ plural. If no units are specified, hours are assumed. For example,
+ "2", "2h", "2 hour", and
+ "2 hours" are all valid ways to specify two hours.
The roll-offset-interval-millis property sets the upper
+ bound on a random time interval (in milliseconds) that is used to delay
+ before the initial roll. All subsequent rolls will happen an integer
+ number of roll intervals after the initial roll, hence retaining the original
+ offset. The purpose of this property is to insert some variance in the roll
+ times so that large clusters using this sink on every node don't cause a
+ performance impact on HDFS by rolling simultaneously. The default value is
+ 30000 (30s). When writing to HDFS, as a rule of thumb, the roll offset in
+ millis should be no less than the number of sink instances times 5.
+
+
The primary use of this class is for logging to HDFS. As it uses + {@link org.apache.hadoop.fs.FileSystem} to access the target file system, + however, it can be used to write to the local file system, Amazon S3, or any + other supported file system. The base path for the sink will determine the + file system used. An unqualified path will write to the default file system + set by the configuration.
+ +Not all file systems support the ability to append to files. In file
+ systems without the ability to append to files, only one writer can write to
+ a file at a time. To allow for concurrent writes from multiple daemons on a
+ single host, the source property is used to set unique headers
+ for the log files. The property should be set to the name of
+ the source daemon, e.g. namenode. The value of the
+ source property should typically be the same as the property's
+ prefix. If this property is not set, the source is taken to be
+ unknown.
Instead of appending to an existing file, by default the sink + will create a new file with a suffix of ".<n>", where + n is the next lowest integer that isn't already used in a file name, + similar to the Hadoop daemon logs. NOTE: the file with the highest + sequence number is the newest file, unlike the Hadoop daemon logs.
+ +For file systems that allow append, the sink supports appending to the
+ existing file instead. If the allow-append property is set to
+ true, the sink will instead append to the existing file on file systems that
+ support appends. By default, the allow-append property is
+ false.
Note that when writing to HDFS with allow-append set to true,
+ there is a minimum acceptable number of data nodes. If the number of data
+ nodes drops below that minimum, the append will succeed, but reading the
+ data will fail with an IOException in the DataStreamer class. The minimum
+ number of data nodes required for a successful append is generally 2 or
+ 3.
Note also that when writing to HDFS, the file size information is not + updated until the file is closed (at the end of the interval) even though + the data is being written successfully. This is a known HDFS limitation that + exists because of the performance cost of updating the metadata. See + HDFS-5478.
+ +When using this sink in a secure (Kerberos) environment, two additional
+ properties must be set: keytab-key and
+ principal-key. keytab-key should contain the key by
+ which the keytab file can be found in the configuration, for example,
+ yarn.nodemanager.keytab. principal-key should
+ contain the key by which the principal can be found in the configuration,
+ for example, yarn.nodemanager.principal.]]>
+
+ *.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink + [prefix].sink.statsd.server.host= + [prefix].sink.statsd.server.port= + [prefix].sink.statsd.skip.hostname=true|false (optional) + [prefix].sink.statsd.service.name=NameNode (name you want for service) +]]> +
+
+ This class does not extend the Configured
+ base class, and should not be changed to do so, as it causes problems
+ for subclasses. The constructor of the Configured calls
+ the {@link #setConf(Configuration)} method, which will call into the
+ subclasses before they have been fully constructed.]]>
+
RawScriptBasedMapping that performs
+ the work: reading the configuration parameters, executing any defined
+ script, handling errors and such like. The outer
+ class extends {@link CachedDNSToSwitchMapping} to cache the delegated
+ queries.
+ + This DNS mapper's {@link #isSingleSwitch()} predicate returns + true if and only if a script is defined.]]> +
+ This class uses the configuration parameter {@code + net.topology.table.file.name} to locate the mapping file. +
++ Calls to {@link #resolve(List)} will look up the address as defined in the + mapping file. If no entry corresponding to the address is found, the value + {@code /default-rack} is returned. +
]]> ++ An instance of the default {@link DelegationTokenAuthenticator} will be + used.]]> +
null the default one will be used.]]>
+ null the default one will be used.
+ @param connConfigurator a connection configurator.]]>
+ TRUE if the token is transmitted in the
+ URL query string, FALSE if the delegation token is transmitted
+ using the {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP
+ header.]]>
+ FALSE if the delegation token is transmitted using the
+ {@link DelegationTokenAuthenticator#DELEGATION_TOKEN_HEADER} HTTP header.]]>
+ doAs parameter is not NULL,
+ the request will be done on behalf of the specified doAs user.
+
+ @param url the URL to connect to. Only HTTP/S URLs are supported.
+ @param token the authentication token being used for the user.
+ @param doAs user to do the the request on behalf of, if NULL the request is
+ as self.
+ @return an authenticated {@link HttpURLConnection}.
+ @throws IOException if an IO error occurred.
+ @throws AuthenticationException if an authentication exception occurred.]]>
+ + The authentication mechanisms supported by default are Hadoop Simple + authentication (also known as pseudo authentication) and Kerberos SPNEGO + authentication. +
+ Additional authentication mechanisms can be supported via {@link + DelegationTokenAuthenticator} implementations. +
+ The default {@link DelegationTokenAuthenticator} is the {@link + KerberosDelegationTokenAuthenticator} class which supports + automatic fallback from Kerberos SPNEGO to Hadoop Simple authentication via + the {@link PseudoDelegationTokenAuthenticator} class. +
+ AuthenticatedURL instances are not thread-safe.]]>
+
+ It falls back to the {@link PseudoDelegationTokenAuthenticator} if the HTTP + endpoint does not trigger a SPNEGO authentication]]> +
+ This mimics the model of Hadoop Simple authentication trusting the + {@link UserGroupInformation#getCurrentUser()} value.]]> +
+ The service state is checked before the operation begins. + This process is not thread safe. + @param service a service or null]]> +
+ This permits implementations to change the configuration before + the init operation. As the ServiceLauncher only creates + an instance of the base {@link Configuration} class, it is + recommended to instantiate any subclass (such as YarnConfiguration) + that injects new resources. +
+ @param config the initial configuration build up by the + service launcher. + @param args list of arguments passed to the command line + after any launcher-specific commands have been stripped. + @return the configuration to init the service with. + Recommended: pass down the config parameter with any changes + @throws Exception any problem]]> +
+ If an exception is raised, the policy is: +
+ Approximate HTTP equivalent: {@code 505: Version Not Supported}]]> +
+ Many of the exit codes are designed to resemble HTTP error codes, + squashed into a single byte. e.g 44 , "not found" is the equivalent + of 404. The various 2XX HTTP error codes aren't followed; + the Unix standard of "0" for success is used. +
+ 0-10: general command issues + 30-39: equivalent to the 3XX responses, where those responses are + considered errors by the application. + 40-49: client-side/CLI/config problems + 50-59: service-side problems. + 60+ : application specific error codes +]]> +
+ If the last argument is a throwable, it becomes the cause of the exception. + It will also be used as a parameter for the format. + @param exitCode exit code + @param format format for message to use in exception + @param args list of arguments]]> +
Progressable
+ to explicitly report progress to the Hadoop framework. This is especially
+ important for operations which take significant amount of time since,
+ in-lieu of the reported progress, the framework has to assume that an error
+ has occurred and time-out the operation.]]>
+ Class is to be obtained
+ @return the correctly typed Class of the given object.]]>
+ ".sh" otherwise.
+
+ @param parent File parent directory
+ @param basename String script file basename
+ @return File referencing the script in the directory]]>
+ ".sh" otherwise.
+
+ @param basename String script file basename
+ @return String script file name]]>
+ Shell
+ processes and destroys them one by one. This method is thread safe]]>
+ + @deprecated use one of the exception-raising getter methods, + specifically {@link #getWinUtilsPath()} or {@link #getWinUtilsFile()}]]> +
du or
+ df. It also offers facilities to gate commands by
+ time-intervals.]]>
+ ShutdownHookManager singleton.]]>
+ TimeUnit]]>
+ + The JVM runs ShutdownHooks in a non-deterministic order or in parallel. + This class registers a single JVM shutdownHook and run all the + shutdownHooks registered to it (to this class) in order based on their + priority. + + Unless a hook was registered with a shutdown explicitly set through + {@link #addShutdownHook(Runnable, int, long, TimeUnit)}, + the shutdown time allocated to it is set by the configuration option + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT} in + {@code core-site.xml}, with a default value of + {@link CommonConfigurationKeysPublic#SERVICE_SHUTDOWN_TIMEOUT_DEFAULT} + seconds.]]> +
Tool, is the standard for any Map-Reduce tool/application.
+ The tool/application should delegate the handling of
+
+ standard command-line options to {@link ToolRunner#run(Tool, String[])}
+ and only handle its custom arguments.
+
+ Here is how a typical Tool is implemented:
+ public class MyApp extends Configured implements Tool {
+
+ public int run(String[] args) throws Exception {
+ // Configuration processed by ToolRunner
+ Configuration conf = getConf();
+
+ // Create a JobConf using the processed conf
+ JobConf job = new JobConf(conf, MyApp.class);
+
+ // Process custom command-line options
+ Path in = new Path(args[1]);
+ Path out = new Path(args[2]);
+
+ // Specify various job-specific parameters
+ job.setJobName("my-app");
+ job.setInputPath(in);
+ job.setOutputPath(out);
+ job.setMapperClass(MyMapper.class);
+ job.setReducerClass(MyReducer.class);
+
+ // Submit the job, then poll for progress until the job is complete
+ RunningJob runningJob = JobClient.runJob(job);
+ if (runningJob.isSuccessful()) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ // Let ToolRunner handle generic command-line options
+ int res = ToolRunner.run(new Configuration(), new MyApp(), args);
+
+ System.exit(res);
+ }
+ }
+ Configuration, or builds one if null.
+
+ Sets the Tool's configuration with the possibly modified
+ version of the conf.
+
+ @param conf Configuration for the Tool.
+ @param tool Tool to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.
+ @throws Exception Exception.]]>
+ Configuration.
+
+ Equivalent to run(tool.getConf(), tool, args).
+
+ @param tool Tool to run.
+ @param args command-line arguments to the tool.
+ @return exit code of the {@link Tool#run(String[])} method.
+ @throws Exception exception.]]>
+ ToolRunner can be used to run classes implementing
+ Tool interface. It works in conjunction with
+ {@link GenericOptionsParser} to parse the
+
+ generic hadoop command line arguments and modifies the
+ Configuration of the Tool. The
+ application-specific options are passed along without being modified.
+
+
+ @see Tool
+ @see GenericOptionsParser]]>
+ + The Bloom filter is a data structure that was introduced in 1970 and that has been adopted by + the networking research community in the past decade thanks to the bandwidth efficiencies that it + offers for the transmission of set membership information between networked hosts. A sender encodes + the information into a bit vector, the Bloom filter, that is more compact than a conventional + representation. Computation and space costs for construction are linear in the number of elements. + The receiver uses the filter to test whether various elements are members of the set. Though the + filter will occasionally return a false positive, it will never return a false negative. When creating + the filter, the sender can choose its desired point in a trade-off between the false positive rate and the size. + +
+ Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Space/Time Trade-Offs in Hash Coding with Allowable Errors]]> +
+ Invariant: nothing happens if the specified key does not belong to this counter Bloom filter. + @param key The key to remove.]]> +
NOTE: due to the bucket size of this filter, inserting the same
+ key more than 15 times will cause an overflow at all filter positions
+ associated with this key, and it will significantly increase the error
+ rate for this and other keys. For this reason the filter can only be
+ used to store small count values 0 <= N << 15.
+ @param key key to be tested
+ @return 0 if the key is not present. Otherwise, a positive value v will
+ be returned such that v == count with probability equal to the
+ error rate of this filter, and v > count otherwise.
+ Additionally, if the filter experienced an underflow as a result of
+ {@link #delete(Key)} operation, the return value may be lower than the
+ count with the probability of the false negative rate of such
+ filter.]]>
+
+ A counting Bloom filter is an improvement to standard a Bloom filter as it + allows dynamic additions and deletions of set membership information. This + is achieved through the use of a counting vector instead of a bit vector. +
+ Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + + @see Summary cache: a scalable wide-area web cache sharing protocol]]> +
+ A dynamic Bloom filter (DBF) makes use of a s * m bit matrix but
+ each of the s rows is a standard Bloom filter. The creation
+ process of a DBF is iterative. At the start, the DBF is a 1 * m
+ bit matrix, i.e., it is composed of a single standard Bloom filter.
+ It assumes that nr elements are recorded in the
+ initial bit vector, where nr {@literal <=} n
+ (n is the cardinality of the set A to record in
+ the filter).
+
+ As the size of A grows during the execution of the application,
+ several keys must be inserted in the DBF. When inserting a key into the DBF,
+ one must first get an active Bloom filter in the matrix. A Bloom filter is
+ active when the number of recorded keys, nr, is
+ strictly less than the current cardinality of A, n.
+ If an active Bloom filter is found, the key is inserted and
+ nr is incremented by one. On the other hand, if there
+ is no active Bloom filter, a new one is created (i.e., a new row is added to
+ the matrix) according to the current size of A and the element
+ is added in this new Bloom filter and the nr value of
+ this new Bloom filter is set to one. A given key is said to belong to the
+ DBF if the k positions are set to one in one of the matrix rows.
+
+ Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + + @see Theory and Network Applications of Dynamic Bloom Filters]]> +
+ Invariant: if the false positive is null, nothing happens.
+ @param key The false positive key to add.]]>
+
+ It allows the removal of selected false positives at the cost of introducing + random false negatives, and with the benefit of eliminating some random false + positives at the same time. + +
+ Originally created by + European Commission One-Lab Project 034819. + + @see Filter The general behavior of a filter + @see BloomFilter A Bloom filter + @see RemoveScheme The different selective clearing algorithms + + @see Retouched Bloom Filters: Allowing Networked Applications to Trade Off Selected False Positives Against False Negatives]]> +
+ One key feature is that the {@link #awaitFuture(Future)} and + {@link #awaitFuture(Future, long, TimeUnit)} calls will + extract and rethrow exceptions raised in the future's execution, + including extracting the inner IOException of any + {@code UncheckedIOException} raised in the future. + This makes it somewhat easier to execute IOException-raising + code inside futures. +
]]> ++ The number of entries processed is returned, as it is useful to + know this, especially during tests or when reporting values + to users. +
+ This does not close the iterator afterwards. + @param source iterator source + @param consumer consumer of the values. + @return the number of elements processed + @param+ One notable feature is that the + {@link #foreach(RemoteIterator, ConsumerRaisingIOE)} method will + LOG at debug any IOStatistics provided by the iterator, if such + statistics are provided. There's no attempt at retrieval and logging + if the LOG is not set to debug, so it is a zero cost feature unless + the logger {@code org.apache.hadoop.fs.functional.RemoteIterators} + is at DEBUG. +
+ Based on the S3A Listing code, and some some work on moving other code + to using iterative listings so as to pick up the statistics.]]> +
- * {"name":"test","time":1318429136789,"date":"2011-10-12 15:18:56,789","level":"INFO","thread":"main","message":"test message"}
- *
- *
- * An event with an error will contain data similar to that below (which has been reformatted to be multi-line).
- *
- *
- * {
- * "name":"testException",
- * "time":1318429136789,
- * "date":"2011-10-12 15:18:56,789",
- * "level":"INFO",
- * "thread":"quoted\"",
- * "message":"new line\n and {}",
- * "exceptionclass":"java.net.NoRouteToHostException",
- * "stack":[
- * "java.net.NoRouteToHostException: that box caught fire 3 years ago",
- * "\tat org.apache.hadoop.log.TestLog4Json.testException(TestLog4Json.java:49)",
- * "\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)",
- * "\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)",
- * "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)",
- * "\tat java.lang.reflect.Method.invoke(Method.java:597)",
- * "\tat junit.framework.TestCase.runTest(TestCase.java:168)",
- * "\tat junit.framework.TestCase.runBare(TestCase.java:134)",
- * "\tat junit.framework.TestResult$1.protect(TestResult.java:110)",
- * "\tat junit.framework.TestResult.runProtected(TestResult.java:128)",
- * "\tat junit.framework.TestResult.run(TestResult.java:113)",
- * "\tat junit.framework.TestCase.run(TestCase.java:124)",
- * "\tat junit.framework.TestSuite.runTest(TestSuite.java:232)",
- * "\tat junit.framework.TestSuite.run(TestSuite.java:227)",
- * "\tat org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:83)",
- * "\tat org.apache.maven.surefire.junit4.JUnit4TestSet.execute(JUnit4TestSet.java:59)",
- * "\tat org.apache.maven.surefire.suite.AbstractDirectoryTestSuite.executeTestSet(AbstractDirectoryTestSuite.java:120)",
- * "\tat org.apache.maven.surefire.suite.AbstractDirectoryTestSuite.execute(AbstractDirectoryTestSuite.java:145)",
- * "\tat org.apache.maven.surefire.Surefire.run(Surefire.java:104)",
- * "\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)",
- * "\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)",
- * "\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)",
- * "\tat java.lang.reflect.Method.invoke(Method.java:597)",
- * "\tat org.apache.maven.surefire.booter.SurefireBooter.runSuitesInProcess(SurefireBooter.java:290)",
- * "\tat org.apache.maven.surefire.booter.SurefireBooter.main(SurefireBooter.java:1017)"
- * ]
- * }
- *
- */
-public class Log4Json extends Layout {
-
- /**
- * Jackson factories are thread safe when constructing parsers and generators.
- * They are not thread safe in configure methods; if there is to be any
- * configuration it must be done in a static initializer block.
- */
- private static final JsonFactory factory = new MappingJsonFactory();
- private static final ObjectReader READER = new ObjectMapper(factory).reader();
- public static final String DATE = "date";
- public static final String EXCEPTION_CLASS = "exceptionclass";
- public static final String LEVEL = "level";
- public static final String MESSAGE = "message";
- public static final String NAME = "name";
- public static final String STACK = "stack";
- public static final String THREAD = "thread";
- public static final String TIME = "time";
- public static final String JSON_TYPE = "application/json";
-
- private final DateFormat dateFormat;
-
- public Log4Json() {
- dateFormat = new ISO8601DateFormat();
- }
-
-
- /**
- * @return the mime type of JSON
- */
- @Override
- public String getContentType() {
- return JSON_TYPE;
- }
-
- @Override
- public String format(LoggingEvent event) {
- try {
- return toJson(event);
- } catch (IOException e) {
- //this really should not happen, and rather than throw an exception
- //which may hide the real problem, the log class is printed
- //in JSON format. The classname is used to ensure valid JSON is
- //returned without playing escaping games
- return "{ \"logfailure\":\"" + e.getClass().toString() + "\"}";
- }
- }
-
- /**
- * Convert an event to JSON
- *
- * @param event the event -must not be null
- * @return a string value
- * @throws IOException on problems generating the JSON
- */
- public String toJson(LoggingEvent event) throws IOException {
- StringWriter writer = new StringWriter();
- toJson(writer, event);
- return writer.toString();
- }
-
- /**
- * Convert an event to JSON
- *
- * @param writer the destination writer
- * @param event the event -must not be null
- * @return the writer
- * @throws IOException on problems generating the JSON
- */
- public Writer toJson(final Writer writer, final LoggingEvent event)
- throws IOException {
- ThrowableInformation ti = event.getThrowableInformation();
- toJson(writer,
- event.getLoggerName(),
- event.getTimeStamp(),
- event.getLevel().toString(),
- event.getThreadName(),
- event.getRenderedMessage(),
- ti);
- return writer;
- }
-
- /**
- * Build a JSON entry from the parameters. This is public for testing.
- *
- * @param writer destination
- * @param loggerName logger name
- * @param timeStamp time_t value
- * @param level level string
- * @param threadName name of the thread
- * @param message rendered message
- * @param ti nullable thrown information
- * @return the writer
- * @throws IOException on any problem
- */
- public Writer toJson(final Writer writer,
- final String loggerName,
- final long timeStamp,
- final String level,
- final String threadName,
- final String message,
- final ThrowableInformation ti) throws IOException {
- JsonGenerator json = factory.createGenerator(writer);
- json.writeStartObject();
- json.writeStringField(NAME, loggerName);
- json.writeNumberField(TIME, timeStamp);
- Date date = new Date(timeStamp);
- json.writeStringField(DATE, dateFormat.format(date));
- json.writeStringField(LEVEL, level);
- json.writeStringField(THREAD, threadName);
- json.writeStringField(MESSAGE, message);
- if (ti != null) {
- //there is some throwable info, but if the log event has been sent over the wire,
- //there may not be a throwable inside it, just a summary.
- Throwable thrown = ti.getThrowable();
- String eclass = (thrown != null) ?
- thrown.getClass().getName()
- : "";
- json.writeStringField(EXCEPTION_CLASS, eclass);
- String[] stackTrace = ti.getThrowableStrRep();
- json.writeArrayFieldStart(STACK);
- for (String row : stackTrace) {
- json.writeString(row);
- }
- json.writeEndArray();
- }
- json.writeEndObject();
- json.flush();
- json.close();
- return writer;
- }
-
- /**
- * This appender does not ignore throwables
- *
- * @return false, always
- */
- @Override
- public boolean ignoresThrowable() {
- return false;
- }
-
- /**
- * Do nothing
- */
- @Override
- public void activateOptions() {
- }
-
- /**
- * For use in tests
- *
- * @param json incoming JSON to parse
- * @return a node tree
- * @throws IOException on any parsing problems
- */
- public static ContainerNode parse(String json) throws IOException {
- JsonNode jsonNode = READER.readTree(json);
- if (!(jsonNode instanceof ContainerNode)) {
- throw new IOException("Wrong JSON data: " + json);
- }
- return (ContainerNode) jsonNode;
- }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
index 6785e2f672a..32879597a9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -34,6 +34,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -44,6 +46,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.GenericsUtil;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -338,14 +341,18 @@ public class LogLevel {
out.println(MARKER
+ "Submitted Class Name: " + logName + "Class<T>) of the
* argument of type T.
@@ -87,12 +96,27 @@ public class GenericsUtil {
if (clazz == null) {
return false;
}
- Logger log = LoggerFactory.getLogger(clazz);
+ return isLog4jLogger(clazz.getName());
+ }
+
+ /**
+ * Determine whether the log of the given logger is of Log4J implementation.
+ *
+ * @param logger the logger name, usually class name as string.
+ * @return true if the logger uses Log4J implementation.
+ */
+ public static boolean isLog4jLogger(String logger) {
+ if (logger == null || !IS_LOG4J_LOGGER.get()) {
+ return false;
+ }
+ Logger log = LoggerFactory.getLogger(logger);
try {
- Class log4jClass = Class.forName("org.slf4j.impl.Log4jLoggerAdapter");
+ Class log4jClass = Class.forName(SLF4J_LOG4J_ADAPTER_CLASS);
return log4jClass.isInstance(log);
} catch (ClassNotFoundException e) {
+ IS_LOG4J_LOGGER.set(false);
return false;
}
}
+
}
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index a551e3ae15f..0777fc42abe 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -592,17 +592,19 @@ StateStoreMetrics
-----------------
StateStoreMetrics shows the statistics of the State Store component in Router-based federation.
-| Name | Description |
-|:---- |:---- |
-| `ReadsNumOps` | Number of GET transactions for State Store within an interval time of metric |
-| `ReadsAvgTime` | Average time of GET transactions for State Store in milliseconds |
-| `WritesNumOps` | Number of PUT transactions for State Store within an interval time of metric |
-| `WritesAvgTime` | Average time of PUT transactions for State Store in milliseconds |
-| `RemovesNumOps` | Number of REMOVE transactions for State Store within an interval time of metric |
-| `RemovesAvgTime` | Average time of REMOVE transactions for State Store in milliseconds |
-| `FailuresNumOps` | Number of failed transactions for State Store within an interval time of metric |
-| `FailuresAvgTime` | Average time of failed transactions for State Store in milliseconds |
-| `Cache`*BaseRecord*`Size` | Number of store records to cache in State Store |
+| Name | Description |
+|:------------------------------------------|:-----------------------------------------------------------------------------------|
+| `ReadsNumOps` | Number of GET transactions for State Store within an interval time of metric |
+| `ReadsAvgTime` | Average time of GET transactions for State Store in milliseconds |
+| `WritesNumOps` | Number of PUT transactions for State Store within an interval time of metric |
+| `WritesAvgTime` | Average time of PUT transactions for State Store in milliseconds |
+| `RemovesNumOps` | Number of REMOVE transactions for State Store within an interval time of metric |
+| `RemovesAvgTime` | Average time of REMOVE transactions for State Store in milliseconds |
+| `FailuresNumOps` | Number of failed transactions for State Store within an interval time of metric |
+| `FailuresAvgTime` | Average time of failed transactions for State Store in milliseconds |
+| `Cache`*BaseRecord*`Size` | Number of store records to cache in State Store |
+| `Cache`*BaseRecord*`LoadNumOps` | Number of times store records are loaded in the State Store Cache from State Store |
+| `Cache`*BaseRecord*`LoadAvgTime` | Average time of loading State Store Cache from State Store in milliseconds |
yarn context
============
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
index bbea16855e5..8153dce5c3f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
@@ -157,8 +157,7 @@ The following instructions are to run a MapReduce job locally. If you want to ex
4. Make the HDFS directories required to execute MapReduce jobs:
- $ bin/hdfs dfs -mkdir /user
- $ bin/hdfs dfs -mkdir /user/The most important difference is that unlike GFS, Hadoop DFS files +have strictly one writer at any one time. Bytes are always appended +to the end of the writer's stream. There is no notion of "record appends" +or "mutations" that are then checked or reordered. Writers simply emit +a byte stream. That byte stream is guaranteed to be stored in the +order written.
]]> +- * For example, - * log4j.appender.ASYNCAPPENDER=org.apache.log4j.AsyncAppender - * log4j.appender.ASYNCAPPENDER.blocking=false - */ - @Deprecated - public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_KEY = - "dfs.namenode.audit.log.async.blocking"; - @Deprecated - public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_DEFAULT = true; - - /** - * Deprecated. Use log4j properties instead. - * Set value to Async appender "bufferSize" property as part of log4j properties configuration. - *
- * For example,
- * log4j.appender.ASYNCAPPENDER=org.apache.log4j.AsyncAppender
- * log4j.appender.ASYNCAPPENDER.bufferSize=128
- */
- @Deprecated
- public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_KEY =
- "dfs.namenode.audit.log.async.buffer.size";
- @Deprecated
- public static final int DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_DEFAULT = 128;
public static final String DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST = "dfs.namenode.audit.log.debug.cmdlist";
public static final String DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY =
"dfs.namenode.metrics.logger.period.seconds";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 4e5e1234716..ec8bbf82498 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3987,17 +3987,11 @@ public class BlockManager implements BlockStatsMXBean {
}
if (shouldProcessExtraRedundancy(num, expectedRedundancy)) {
- if (num.replicasOnStaleNodes() > 0) {
- // If any of the replicas of this block are on nodes that are
- // considered "stale", then these replicas may in fact have
- // already been deleted. So, we cannot safely act on the
- // over-replication until a later point in time, when
- // the "stale" nodes have block reported.
+ // extra redundancy block
+ if (!processExtraRedundancyBlockWithoutPostpone(block, expectedRedundancy,
+ null, null)) {
return MisReplicationResult.POSTPONE;
}
-
- // extra redundancy block
- processExtraRedundancyBlock(block, expectedRedundancy, null, null);
return MisReplicationResult.OVER_REPLICATED;
}
@@ -4020,12 +4014,26 @@ public class BlockManager implements BlockStatsMXBean {
}
}
+ /**
+ * Process blocks with redundant replicas. If there are replicas in
+ * stale storages, mark them in the postponedMisreplicatedBlocks.
+ */
+ private void processExtraRedundancyBlock(final BlockInfo block,
+ final short replication, final DatanodeDescriptor addedNode,
+ DatanodeDescriptor delNodeHint) {
+ if (!processExtraRedundancyBlockWithoutPostpone(block, replication,
+ addedNode, delNodeHint)) {
+ postponeBlock(block);
+ }
+ }
+
/**
* Find how many of the containing nodes are "extra", if any.
* If there are any extras, call chooseExcessRedundancies() to
* mark them in the excessRedundancyMap.
+ * @return true if all redundancy replicas are removed.
*/
- private void processExtraRedundancyBlock(final BlockInfo block,
+ private boolean processExtraRedundancyBlockWithoutPostpone(final BlockInfo block,
final short replication, final DatanodeDescriptor addedNode,
DatanodeDescriptor delNodeHint) {
assert namesystem.hasWriteLock();
@@ -4035,17 +4043,17 @@ public class BlockManager implements BlockStatsMXBean {
CollectionDistributedCache is a facility provided by the Map-Reduce
+ framework to cache files (text, archives, jars etc.) needed by applications.
+
Applications specify the files, via urls (hdfs:// or http://) to be cached
+ via the {@link org.apache.hadoop.mapred.JobConf}. The
+ DistributedCache assumes that the files specified via urls are
+ already present on the {@link FileSystem} at the path specified by the url
+ and are accessible by every machine in the cluster.
The framework will copy the necessary files on to the worker node before + any tasks for the job are executed on that node. Its efficiency stems from + the fact that the files are only copied once per job and the ability to + cache archives which are un-archived on the workers.
+ +DistributedCache can be used to distribute simple, read-only
+ data/text files and/or more complex types such as archives, jars etc.
+ Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
+ Jars may be optionally added to the classpath of the tasks, a rudimentary
+ software distribution mechanism. Files have execution permissions.
+ In older version of Hadoop Map/Reduce users could optionally ask for symlinks
+ to be created in the working directory of the child task. In the current
+ version symlinks are always created. If the URL does not have a fragment
+ the name of the file or directory will be used. If multiple files or
+ directories map to the same link name, the last one added, will be used. All
+ others will not even be downloaded.
DistributedCache tracks modification timestamps of the cache
+ files. Clearly the cache files should not be modified by the application
+ or externally while the job is executing.
Here is an illustrative example on how to use the
+ DistributedCache:
+ + It is also very common to use the DistributedCache by using + {@link org.apache.hadoop.util.GenericOptionsParser}. + + This class includes methods that should be used by users + (specifically those mentioned in the example above, as well + as {@link DistributedCache#addArchiveToClassPath(Path, Configuration)}), + as well as methods intended for use by the MapReduce framework + (e.g., {@link org.apache.hadoop.mapred.JobClient}). + + @see org.apache.hadoop.mapred.JobConf + @see org.apache.hadoop.mapred.JobClient + @see org.apache.hadoop.mapreduce.Job]]> + + + + ++ // Setting up the cache for the application + + 1. Copy the requisite files to theFileSystem: + + $ bin/hadoop fs -copyFromLocal lookup.dat /myapp/lookup.dat + $ bin/hadoop fs -copyFromLocal map.zip /myapp/map.zip + $ bin/hadoop fs -copyFromLocal mylib.jar /myapp/mylib.jar + $ bin/hadoop fs -copyFromLocal mytar.tar /myapp/mytar.tar + $ bin/hadoop fs -copyFromLocal mytgz.tgz /myapp/mytgz.tgz + $ bin/hadoop fs -copyFromLocal mytargz.tar.gz /myapp/mytargz.tar.gz + + 2. Setup the application'sJobConf: + + JobConf job = new JobConf(); + DistributedCache.addCacheFile(new URI("/myapp/lookup.dat#lookup.dat"), + job); + DistributedCache.addCacheArchive(new URI("/myapp/map.zip"), job); + DistributedCache.addFileToClassPath(new Path("/myapp/mylib.jar"), job); + DistributedCache.addCacheArchive(new URI("/myapp/mytar.tar"), job); + DistributedCache.addCacheArchive(new URI("/myapp/mytgz.tgz"), job); + DistributedCache.addCacheArchive(new URI("/myapp/mytargz.tar.gz"), job); + + 3. Use the cached files in the {@link org.apache.hadoop.mapred.Mapper} + or {@link org.apache.hadoop.mapred.Reducer}: + + public static class MapClass extends MapReduceBase + implements Mapper<K, V, K, V> { + + private Path[] localArchives; + private Path[] localFiles; + + public void configure(JobConf job) { + // Get the cached archives/files + File f = new File("./map.zip/some/file/in/zip.txt"); + } + + public void map(K key, V value, + OutputCollector<K, V> output, Reporter reporter) + throws IOException { + // Use data from the cached archives/files here + // ... + // ... + output.collect(k, v); + } + } + +
JobTracker.]]>
+ ClusterStatus provides clients with information such as:
+ JobTracker.
+ Clients can query for the latest ClusterStatus, via
+ {@link JobClient#getClusterStatus()}.
Counters represent global counters, defined either by the
+ Map-Reduce framework or applications. Each Counter can be of
+ any {@link Enum} type.
+
+ Counters are bunched into {@link Group}s, each comprising of
+ counters from a particular Enum class.]]>
+
Grouphandles localization of the class name and the
+ counter names.
FileInputFormat implementations can override this and return
+ false to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+
+ @param fs the file system that the file is on
+ @param filename the file name to check
+ @return is this file splitable?]]>
+ FileInputFormat is the base class for all file-based
+ InputFormats. This provides a generic implementation of
+ {@link #getSplits(JobConf, int)}.
+
+ Implementations of FileInputFormat can also override the
+ {@link #isSplitable(FileSystem, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files must override this method, since
+ the default implementation assumes splitting is always possible.]]>
+ false otherwise]]>
+ Note: The following is valid only if the {@link OutputCommitter}
+ is {@link FileOutputCommitter}. If OutputCommitter is not
+ a FileOutputCommitter, the task's temporary output
+ directory is same as {@link #getOutputPath(JobConf)} i.e.
+ ${mapreduce.output.fileoutputformat.outputdir}$
Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +
In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.
+ +To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.
+ +The application-writer can take advantage of this by creating any + side-files required in ${mapreduce.task.output.dir} during execution + of his reduce-task i.e. via {@link #getWorkOutputPath(JobConf)}, and the + framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.
+ +Note: the value of ${mapreduce.task.output.dir} during + execution of a particular task-attempt is actually + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}, and this value is + set by the map-reduce framework. So, just create any side-files in the + path returned by {@link #getWorkOutputPath(JobConf)} from map/reduce + task to take advantage of this feature.
+ +The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.
+ + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +The given name is postfixed with the task type, 'm' for maps, 'r' for + reduces and the task partition number. For example, give a name 'test' + running on the first map o the job the generated name will be + 'test-m-00000'.
+ + @param conf the configuration for the job. + @param name the name to make unique. + @return a unique name accross all tasks of the job.]]> +This method uses the {@link #getUniqueName} method to make the file name + unique for the task.
+ + @param conf the configuration for the job. + @param name the name for the file. + @return a unique path accross all tasks of the job.]]> +Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. + + @param job job configuration. + @param numSplits the desired number of splits, a hint. + @return an array of {@link InputSplit}s for the job.]]> +
RecordReader to respect
+ record boundaries while processing the logical split to present a
+ record-oriented view to the individual task.
+
+ @param split the {@link InputSplit}
+ @param job the job that this split belongs to
+ @return a {@link RecordReader}]]>
+ The Map-Reduce framework relies on the InputFormat of the
+ job to:
+
InputSplit for processing by
+ the {@link Mapper}.
+ The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.
+ +Clearly, logical splits based on input-size is insufficient for many
+ applications since record boundaries are to be respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibilty to respect record-boundaries and present a record-oriented
+ view of the logical InputSplit to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see JobClient
+ @see FileInputFormat]]>
+
Strings.
+ @throws IOException]]>
+ Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> +
JobClient provides facilities to submit jobs, track their
+ progress, access component-tasks' reports/logs, get the Map-Reduce cluster
+ status information etc.
+
+ The job submission process involves: +
JobClient to submit
+ the job and monitor its progress.
+
+ Here is an example on how to use JobClient:
+ // Create a new JobConf
+ JobConf job = new JobConf(new Configuration(), MyJob.class);
+
+ // Specify various job-specific parameters
+ job.setJobName("myjob");
+
+ job.setInputPath(new Path("in"));
+ job.setOutputPath(new Path("out"));
+
+ job.setMapperClass(MyJob.MyMapper.class);
+ job.setReducerClass(MyJob.MyReducer.class);
+
+ // Submit the job, then poll for progress until the job is complete
+ JobClient.runJob(job);
+ At times clients would chain map-reduce jobs to accomplish complex tasks + which cannot be done via a single map-reduce job. This is fairly easy since + the output of the job, typically, goes to distributed file-system and that + can be used as the input for the next job.
+ +However, this also means that the onus on ensuring jobs are complete + (success/failure) lies squarely on the clients. In such situations the + various job-control options are: +
false otherwise.]]>
+ false otherwise.]]>
+ For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.
+ +Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.
+ +Note: This is not a guarantee of the combiner sort being + stable in any sense. (In any case, with the order of available + map-outputs to the combiner being non-deterministic, it wouldn't make + that much sense.)
+ + @param theClass the comparator class to be used for grouping keys for the + combiner. It should implementRawComparator.
+ @see #setOutputKeyComparatorClass(Class)]]>
+ For key-value pairs (K1,V1) and (K2,V2), the values (V1, V2) are passed + in a single call to the reduce function if K1 and K2 compare as equal.
+ +Since {@link #setOutputKeyComparatorClass(Class)} can be used to control + how keys are sorted, this can be used in conjunction to simulate + secondary sort on values.
+ +Note: This is not a guarantee of the reduce sort being + stable in any sense. (In any case, with the order of available + map-outputs to the reduce being non-deterministic, it wouldn't make + that much sense.)
+ + @param theClass the comparator class to be used for grouping keys. + It should implementRawComparator.
+ @see #setOutputKeyComparatorClass(Class)
+ @see #setCombinerKeyGroupingComparator(Class)]]>
+ The combiner is an application-specified aggregation operation, which + can help cut down the amount of data transferred between the + {@link Mapper} and the {@link Reducer}, leading to better performance.
+ +The framework may invoke the combiner 0, 1, or multiple times, in both + the mapper and reducer tasks. In general, the combiner is called as the + sort/merge result is written to disk. The combiner must: +
Typically the combiner is same as the Reducer for the
+ job i.e. {@link #setReducerClass(Class)}.
true if speculative execution be used for this job,
+ false otherwise.]]>
+ false.]]>
+ true if speculative execution be
+ used for this job for map tasks,
+ false otherwise.]]>
+ false.]]>
+ true if speculative execution be used
+ for reduce tasks for this job,
+ false otherwise.]]>
+ false.]]>
+ The number of maps is usually driven by the total size of the inputs + i.e. total number of blocks of the input files.
+ +The right level of parallelism for maps seems to be around 10-100 maps + per-node, although it has been set up to 300 or so for very cpu-light map + tasks. Task setup takes awhile, so it is best if the maps take at least a + minute to execute.
+ +The default behavior of file-based {@link InputFormat}s is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of input files. However, the {@link FileSystem} blocksize of the + input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.
+ +Thus, if you expect 10TB of input data and have a blocksize of 128MB, + you'll end up with 82,000 maps, unless {@link #setNumMapTasks(int)} is + used to set it even higher.
+ + @param n the number of map tasks for this job. + @see InputFormat#getSplits(JobConf, int) + @see FileInputFormat + @see FileSystem#getDefaultBlockSize() + @see FileStatus#getBlockSize()]]> +The right number of reduces seems to be 0.95 or
+ 1.75 multiplied by (
+ available memory for reduce tasks
+ (The value of this should be smaller than
+ numNodes * yarn.nodemanager.resource.memory-mb
+ since the resource of memory is shared by map tasks and other
+ applications) /
+
+ mapreduce.reduce.memory.mb).
+
With 0.95 all of the reduces can launch immediately and
+ start transfering map outputs as the maps finish. With 1.75
+ the faster nodes will finish their first round of reduces and launch a
+ second wave of reduces doing a much better job of load balancing.
Increasing the number of reduces increases the framework overhead, but + increases load balancing and lowers the cost of failures.
+ +The scaling factors above are slightly less than whole numbers to + reserve a few reduce slots in the framework for speculative-tasks, failures + etc.
+ + Reducer NONE + +It is legal to set the number of reduce-tasks to zero.
In this case the output of the map-tasks directly go to distributed + file-system, to the path set by + {@link FileOutputFormat#setOutputPath(JobConf, Path)}. Also, the + framework doesn't sort the map-outputs before writing it out to HDFS.
+ + @param n the number of reduce tasks for this job.]]> +zero, i.e. any failed map-task results in
+ the job being declared as {@link JobStatus#FAILED}.
+
+ @return the maximum percentage of map tasks that can fail without
+ the job being aborted.]]>
+ zero, i.e. any failed reduce-task results
+ in the job being declared as {@link JobStatus#FAILED}.
+
+ @return the maximum percentage of reduce tasks that can fail without
+ the job being aborted.]]>
+ The debug command, run on the node where the map failed, is:
++ ++ $script $stdout $stderr $syslog $jobconf. +
The script file is distributed through {@link DistributedCache} + APIs. The script needs to be symlinked.
+ +Here is an example on how to submit a script +
+ job.setMapDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ The debug command, run on the node where the map failed, is:
++ ++ $script $stdout $stderr $syslog $jobconf. +
The script file is distributed through {@link DistributedCache} + APIs. The script file needs to be symlinked
+ +Here is an example on how to submit a script +
+ job.setReduceDebugScript("./myscript");
+ DistributedCache.createSymlink(job);
+ DistributedCache.addCacheFile("/debug/scripts/myscript#myscript");
+ This is typically used by application-writers to implement chaining of + Map-Reduce jobs in an asynchronous manner.
+ + @param uri the job end notification uri + @see JobStatus]]> +
+ ${mapreduce.cluster.local.dir}/taskTracker/$user/jobcache/$jobid/work/ .
+ This directory is exposed to the users through
+ mapreduce.job.local.dir .
+ So, the tasks can use this space
+ as scratch space and share files among them.
+ This value is available as System property also.
+
+ @return The localized job specific shared directory]]>
+ + For backward compatibility, if the job configuration sets the + key {@link #MAPRED_TASK_MAXVMEM_PROPERTY}, that value is returned. + Otherwise, this method will return the larger of the values returned by + {@link #getMemoryForMapTask()} and {@link #getMemoryForReduceTask()} + after converting them into bytes. + + @return Memory required to run a task of this job, in bytes. + @see #setMaxVirtualMemoryForTask(long) + @deprecated Use {@link #getMemoryForMapTask()} and + {@link #getMemoryForReduceTask()}]]> +
$key on
+ Linux or %key% on Windows.
+
+ Example:
+ $key on
+ Linux or %key% on Windows.
+
+ Example:
+ .VARNAME to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ $key on
+ Linux or %key% on Windows.
+
+ Example:
+ .VARNAME to this configuration key, where VARNAME is
+ the name of the environment variable.
+
+ Example:
+ JobConf is the primary interface for a user to describe a
+ map-reduce job to the Hadoop framework for execution. The framework tries to
+ faithfully execute the job as-is described by JobConf, however:
+ JobConf typically specifies the {@link Mapper}, combiner
+ (if any), {@link Partitioner}, {@link Reducer}, {@link InputFormat} and
+ {@link OutputFormat} implementations to be used etc.
+
+
Optionally JobConf is used to specify other advanced facets
+ of the job such as Comparators to be used, files to be put in
+ the {@link DistributedCache}, whether or not intermediate and/or job outputs
+ are to be compressed (and how), debugability via user-provided scripts
+ ( {@link #setMapDebugScript(String)}/{@link #setReduceDebugScript(String)}),
+ for doing post-processing on task logs, task's stdout, stderr, syslog.
+ and etc.
Here is an example on how to configure a job via JobConf:
+ // Create a new JobConf
+ JobConf job = new JobConf(new Configuration(), MyJob.class);
+
+ // Specify various job-specific parameters
+ job.setJobName("myjob");
+
+ FileInputFormat.setInputPaths(job, new Path("in"));
+ FileOutputFormat.setOutputPath(job, new Path("out"));
+
+ job.setMapperClass(MyJob.MyMapper.class);
+ job.setCombinerClass(MyJob.MyReducer.class);
+ job.setReducerClass(MyJob.MyReducer.class);
+
+ job.setInputFormat(SequenceFileInputFormat.class);
+ job.setOutputFormat(SequenceFileOutputFormat.class);
+
+ JobID.getTaskIDsPattern("200707121733", null);
+
+ which will return :
+ "job_200707121733_[0-9]*"+ @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @return a regex pattern matching JobIDs]]> +
job_200707121733_0003 , which represents the third job
+ running at the jobtracker started at 200707121733.
+ + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> +
Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).
+ + @param key the input key. + @param value the input value. + @param output collects mapped keys and values. + @param reporter facility to report progress.]]> +The Hadoop Map-Reduce framework spawns one map task for each
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ Mapper implementations can access the {@link JobConf} for the
+ job via the {@link JobConfigurable#configure(JobConf)} and initialize
+ themselves. Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.
The framework then calls
+ {@link #map(Object, Object, OutputCollector, Reporter)}
+ for each key/value pair in the InputSplit for that task.
All intermediate values associated with a given output key are
+ subsequently grouped by the framework, and passed to a {@link Reducer} to
+ determine the final output. Users can control the grouping by specifying
+ a Comparator via
+ {@link JobConf#setOutputKeyComparatorClass(Class)}.
The grouped Mapper outputs are partitioned per
+ Reducer. Users can control which keys (and hence records) go to
+ which Reducer by implementing a custom {@link Partitioner}.
+
+
Users can optionally specify a combiner, via
+ {@link JobConf#setCombinerClass(Class)}, to perform local aggregation of the
+ intermediate outputs, which helps to cut down the amount of data transferred
+ from the Mapper to the Reducer.
+
+
The intermediate, grouped outputs are always stored in
+ {@link SequenceFile}s. Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the JobConf.
If the job has
+ zero
+ reduces then the output of the Mapper is directly written
+ to the {@link FileSystem} without grouping by keys.
Example:
+
+ public class MyMapper<K extends WritableComparable, V extends Writable>
+ extends MapReduceBase implements Mapper<K, V, K, V> {
+
+ static enum MyCounters { NUM_RECORDS }
+
+ private String mapTaskId;
+ private String inputFile;
+ private int noRecords = 0;
+
+ public void configure(JobConf job) {
+ mapTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+ inputFile = job.get(JobContext.MAP_INPUT_FILE);
+ }
+
+ public void map(K key, V val,
+ OutputCollector<K, V> output, Reporter reporter)
+ throws IOException {
+ // Process the <key, value> pair (assume this takes a while)
+ // ...
+ // ...
+
+ // Let the framework know that we are alive, and kicking!
+ // reporter.progress();
+
+ // Process some more
+ // ...
+ // ...
+
+ // Increment the no. of <key, value> pairs processed
+ ++noRecords;
+
+ // Increment counters
+ reporter.incrCounter(NUM_RECORDS, 1);
+
+ // Every 100 records update application-level status
+ if ((noRecords%100) == 0) {
+ reporter.setStatus(mapTaskId + " processed " + noRecords +
+ " from input-file: " + inputFile);
+ }
+
+ // Output the result
+ output.collect(key, val);
+ }
+ }
+ Applications may write a custom {@link MapRunnable} to exert greater
+ control on map processing e.g. multi-threaded Mappers etc.
Mapping of input records to output records is complete when this method + returns.
+ + @param input the {@link RecordReader} to read the input records. + @param output the {@link OutputCollector} to collect the outputrecords. + @param reporter {@link Reporter} to report progress, status-updates etc. + @throws IOException]]> +MapRunnable can exert greater
+ control on map processing e.g. multi-threaded, asynchronous mappers etc.
+
+ @see Mapper]]>
+ RecordReader's for MultiFileSplit's.
+ @see MultiFileSplit]]>
+ OutputCollector is the generalization of the facility
+ provided by the Map-Reduce framework to collect data output by either the
+ Mapper or the Reducer i.e. intermediate outputs
+ or the output of the job.
false otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+ false otherwise
+ @throws IOException]]>
+ The Map-Reduce framework relies on the OutputCommitter of
+ the job to:
+
The Map-Reduce framework relies on the OutputFormat of the
+ job to:
+
key.]]>
+ Partitioner controls the partitioning of the keys of the
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the m reduce tasks the intermediate key (and hence the
+ record) is sent for reduction.
+
+ Note: A Partitioner is created only when there are multiple
+ reducers.
1.0.
+ @throws IOException]]>
+ RecordReader, typically, converts the byte-oriented view of
+ the input, provided by the InputSplit, and presents a
+ record-oriented view for the {@link Mapper} and {@link Reducer} tasks for
+ processing. It thus assumes the responsibility of processing record
+ boundaries and presenting the tasks with keys and values.
RecordWriter implementations write the job outputs to the
+ {@link FileSystem}.
+
+ @see OutputFormat]]>
+
The framework calls this method for each
+ <key, (list of values)> pair in the grouped inputs.
+ Output values must be of the same type as input values. Input keys must
+ not be altered. The framework will reuse the key and value objects
+ that are passed into the reduce, therefore the application should clone
+ the objects they want to keep a copy of. In many cases, all values are
+ combined into zero or one value.
+
Output pairs are collected with calls to + {@link OutputCollector#collect(Object,Object)}.
+ +Applications can use the {@link Reporter} provided to report progress + or just indicate that they are alive. In scenarios where the application + takes a significant amount of time to process individual key/value + pairs, this is crucial since the framework might assume that the task has + timed-out and kill that task. The other way of avoiding this is to set + + mapreduce.task.timeout to a high-enough value (or even zero for no + time-outs).
+ + @param key the key. + @param values the list of values to reduce. + @param output to collect keys and combined values. + @param reporter facility to report progress.]]> +Reducers for the job is set by the user via
+ {@link JobConf#setNumReduceTasks(int)}. Reducer implementations
+ can access the {@link JobConf} for the job via the
+ {@link JobConfigurable#configure(JobConf)} method and initialize themselves.
+ Similarly they can use the {@link Closeable#close()} method for
+ de-initialization.
+
+ Reducer has 3 primary phases:
Reducer is input the grouped output of a {@link Mapper}.
+ In the phase the framework, for each Reducer, fetches the
+ relevant partition of the output of all the Mappers, via HTTP.
+
The framework groups Reducer inputs by keys
+ (since different Mappers may have output the same key) in this
+ stage.
The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.
+ + SecondarySort + +If equivalence rules for keys while grouping the intermediates are
+ different from those for grouping keys before reduction, then one may
+ specify a Comparator via
+ {@link JobConf#setOutputValueGroupingComparator(Class)}.Since
+ {@link JobConf#setOutputKeyComparatorClass(Class)} can be used to
+ control how intermediate keys are grouped, these can be used in conjunction
+ to simulate secondary sort on values.
In this phase the
+ {@link #reduce(Object, Iterator, OutputCollector, Reporter)}
+ method is called for each <key, (list of values)> pair in
+ the grouped inputs.
The output of the reduce task is typically written to the + {@link FileSystem} via + {@link OutputCollector#collect(Object, Object)}.
+The output of the Reducer is not re-sorted.
Example:
+
+ public class MyReducer<K extends WritableComparable, V extends Writable>
+ extends MapReduceBase implements Reducer<K, V, K, V> {
+
+ static enum MyCounters { NUM_RECORDS }
+
+ private String reduceTaskId;
+ private int noKeys = 0;
+
+ public void configure(JobConf job) {
+ reduceTaskId = job.get(JobContext.TASK_ATTEMPT_ID);
+ }
+
+ public void reduce(K key, Iterator<V> values,
+ OutputCollector<K, V> output,
+ Reporter reporter)
+ throws IOException {
+
+ // Process
+ int noValues = 0;
+ while (values.hasNext()) {
+ V value = values.next();
+
+ // Increment the no. of values for this key
+ ++noValues;
+
+ // Process the <key, value> pair (assume this takes a while)
+ // ...
+ // ...
+
+ // Let the framework know that we are alive, and kicking!
+ if ((noValues%10) == 0) {
+ reporter.progress();
+ }
+
+ // Process some more
+ // ...
+ // ...
+
+ // Output the <key, value>
+ output.collect(key, value);
+ }
+
+ // Increment the no. of <key, list of values> pairs processed
+ ++noKeys;
+
+ // Increment counters
+ reporter.incrCounter(NUM_RECORDS, 1);
+
+ // Every 100 keys update application-level status
+ if ((noKeys%100) == 0) {
+ reporter.setStatus(reduceTaskId + " processed " + noKeys);
+ }
+ }
+ }
+ Reporter
+ provided to report progress or just indicate that they are alive. In
+ scenarios where the application takes significant amount of time to
+ process individual key/value pairs, this is crucial since the framework
+ might assume that the task has timed-out and kill that task.
+
+ Applications can also update {@link Counters} via the provided
+ Reporter .
false.
+ @throws IOException]]>
+ false.
+ @throws IOException]]>
+ false.
+ @throws IOException]]>
+ Clients can get hold of RunningJob via the {@link JobClient}
+ and then query the running-job for details such as name, configuration,
+ progress etc.
false otherwise.]]>
+ false otherwise.]]>
+ This feature can be used when map/reduce tasks crashes deterministically on + certain input. This happens due to bugs in the map/reduce function. The usual + course would be to fix these bugs. But sometimes this is not possible; + perhaps the bug is in third party libraries for which the source code is + not available. Due to this, the task never reaches to completion even with + multiple attempts and complete data for that task is lost.
+ +With this feature, only a small portion of data is lost surrounding + the bad record, which may be acceptable for some user applications. + see {@link SkipBadRecords#setMapperMaxSkipRecords(Configuration, long)}
+ +The skipping mode gets kicked off after certain no of failures + see {@link SkipBadRecords#setAttemptsToStartSkipping(Configuration, int)}
+ +In the skipping mode, the map/reduce task maintains the record range which + is getting processed at all times. Before giving the input to the + map/reduce function, it sends this record range to the Task tracker. + If task crashes, the Task tracker knows which one was the last reported + range. On further attempts that range get skipped.
]]> ++ TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null); ++ which will return : +
"attempt_[^_]*_[0-9]*_m_000001_[0-9]*"+ @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +
+ TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null); ++ which will return : +
"attempt_[^_]*_[0-9]*_m_000001_[0-9]*"+ @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType} + @param taskId taskId number, or null + @param attemptId the task attempt number, or null + @return a regex pattern matching TaskAttemptIDs]]> +
attempt_200707121733_0003_m_000005_0 , which represents the
+ zeroth task attempt for the fifth map task in the third job
+ running at the jobtracker started at 200707121733.
+ + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> +
+ TaskID.getTaskIDsPattern(null, null, true, 1); ++ which will return : +
"task_[^_]*_[0-9]*_m_000001*"+ @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param isMap whether the tip is a map, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs + @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType, + Integer)}]]> +
+ TaskID.getTaskIDsPattern(null, null, true, 1); ++ which will return : +
"task_[^_]*_[0-9]*_m_000001*"+ @param jtIdentifier jobTracker identifier, or null + @param jobId job number, or null + @param type the {@link TaskType}, or null + @param taskId taskId number, or null + @return a regex pattern matching TaskIDs]]> +
task_200707121733_0003_m_000005 , which represents the
+ fifth map task in the third job running at the jobtracker
+ started at 200707121733.
+ + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> +
) }]]> +
+ For the added Mapper the configuration given for it,
+ mapperConf, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +
+
+ @param job job's JobConf to add the Mapper class.
+ @param klass the Mapper class to add.
+ @param inputKeyClass mapper input key class.
+ @param inputValueClass mapper input value class.
+ @param outputKeyClass mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ JobConf(boolean loadDefaults) constructor with FALSE.]]>
+
super.configure(...) should be
+ invoked at the beginning of the overwriter method.]]>
+ super.close() should be
+ invoked at the end of the overwriter method.]]>
+ + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +
+ Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +
+ ChainMapper usage pattern: +
+
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+ Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+ LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+ Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+ LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+ LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ ]]>
+
+ For the added Reducer the configuration given for it,
+ reducerConf, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainReducer, this is done by the setReducer or the addMapper for the last
+ element in the chain.
+
+ @param job job's JobConf to add the Reducer class.
+ @param klass the Reducer class to add.
+ @param inputKeyClass reducer input key class.
+ @param inputValueClass reducer input value class.
+ @param outputKeyClass reducer output key class.
+ @param outputValueClass reducer output value class.
+ @param byValue indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param reducerConf a JobConf with the configuration for the Reducer
+ class. It is recommended to use a JobConf without default values using the
+ JobConf(boolean loadDefaults) constructor with FALSE.]]>
+
+ For the added Mapper the configuration given for it,
+ mapperConf, have precedence over the job's JobConf. This
+ precedence is in effect when the task is running.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the
+ ChainMapper, this is done by the addMapper for the last mapper in the chain
+ .
+
+ @param job chain job's JobConf to add the Mapper class.
+ @param klass the Mapper class to add.
+ @param inputKeyClass mapper input key class.
+ @param inputValueClass mapper input value class.
+ @param outputKeyClass mapper output key class.
+ @param outputValueClass mapper output value class.
+ @param byValue indicates if key/values should be passed by value
+ to the next Mapper in the chain, if any.
+ @param mapperConf a JobConf with the configuration for the Mapper
+ class. It is recommended to use a JobConf without default values using the
+ JobConf(boolean loadDefaults) constructor with FALSE.]]>
+
super.configure(...) should be
+ invoked at the beginning of the overwriter method.]]>
+ map(...) methods of the Mappers in the chain.]]>
+ super.close() should be
+ invoked at the end of the overwriter method.]]>
+ + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. + This enables having reusable specialized Mappers that can be combined to + perform composite operations within a single task. +
+ Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use maching output and input key and + value classes as no conversion is done by the chaining code. +
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +
+ ChainReducer usage pattern: +
+
+ ...
+ conf.setJobName("chain");
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+
+ JobConf mapAConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, AMap.class, LongWritable.class, Text.class,
+ Text.class, Text.class, true, mapAConf);
+
+ JobConf mapBConf = new JobConf(false);
+ ...
+ ChainMapper.addMapper(conf, BMap.class, Text.class, Text.class,
+ LongWritable.class, Text.class, false, mapBConf);
+
+ JobConf reduceConf = new JobConf(false);
+ ...
+ ChainReducer.setReducer(conf, XReduce.class, LongWritable.class, Text.class,
+ Text.class, Text.class, true, reduceConf);
+
+ ChainReducer.addMapper(conf, CMap.class, Text.class, Text.class,
+ LongWritable.class, Text.class, false, null);
+
+ ChainReducer.addMapper(conf, DMap.class, LongWritable.class, Text.class,
+ LongWritable.class, LongWritable.class, true, null);
+
+ FileInputFormat.setInputPaths(conf, inDir);
+ FileOutputFormat.setOutputPath(conf, outDir);
+ ...
+
+ JobClient jc = new JobClient(conf);
+ RunningJob job = jc.submitJob(conf);
+ ...
+ ]]>
+ CombineFileSplit's.
+ @see CombineFileSplit]]>
+ SequenceFileInputFormat.
+
+ @see CombineFileInputFormat]]>
+ TextInputFormat.
+
+ @see CombineFileInputFormat]]>
+ false
+ if it is single. If the name output is not defined it returns
+ false]]>
+ + MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +
+ The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + @param conf job conf to enableadd the named output. + @param enabled indicates if the counters will be enabled or not.]]> ++ MultipleOutputs supports counters, by default the are disabled. + The counters group is the {@link MultipleOutputs} class name. +
+ The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. + + + @param conf job conf to enableadd the named output. + @return TRUE if the counters are enabled, FALSE if they are disabled.]]> +super.close() at the
+ end of their close()
+
+ @throws java.io.IOException thrown if any of the MultipleOutput files
+ could not be closed properly.]]>
+ map() and reduce() methods of the
+ Mapper and Reducer implementations.
+
+ Each additional output, or named output, may be configured with its own
+ OutputFormat, with its own key class and with its own value
+ class.
+
+ A named output can be a single file or a multi file. The later is referred as + a multi named output. +
+ A multi named output is an unbound set of files all sharing the same
+ OutputFormat, key class and value class configuration.
+
+ When named outputs are used within a Mapper implementation,
+ key/values written to a name output are not part of the reduce phase, only
+ key/values written to the job OutputCollector are part of the
+ reduce phase.
+
+ MultipleOutputs supports counters, by default the are disabled. The counters + group is the {@link MultipleOutputs} class name. +
+ The names of the counters are the same as the named outputs. For multi + named outputs the name of the counter is the concatenation of the named + output, and underscore '_' and the multiname. ++ Job configuration usage pattern is: +
+ + JobConf conf = new JobConf(); + + conf.setInputPath(inDir); + FileOutputFormat.setOutputPath(conf, outDir); + + conf.setMapperClass(MOMap.class); + conf.setReducerClass(MOReduce.class); + ... + + // Defines additional single text based output 'text' for the job + MultipleOutputs.addNamedOutput(conf, "text", TextOutputFormat.class, + LongWritable.class, Text.class); + + // Defines additional multi sequencefile based output 'sequence' for the + // job + MultipleOutputs.addMultiNamedOutput(conf, "seq", + SequenceFileOutputFormat.class, + LongWritable.class, Text.class); + ... + + JobClient jc = new JobClient(); + RunningJob job = jc.submitJob(conf); + + ... ++
+ Job configuration usage pattern is: +
+
+ public class MOReduce implements
+ Reducer<WritableComparable, Writable> {
+ private MultipleOutputs mos;
+
+ public void configure(JobConf conf) {
+ ...
+ mos = new MultipleOutputs(conf);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ OutputCollector output, Reporter reporter)
+ throws IOException {
+ ...
+ mos.getCollector("text", reporter).collect(key, new Text("Hello"));
+ mos.getCollector("seq", "A", reporter).collect(key, new Text("Bye"));
+ mos.getCollector("seq", "B", reporter).collect(key, new Text("Chau"));
+ ...
+ }
+
+ public void close() throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+ ]]>
+ + Map implementations using this MapRunnable must be thread-safe. +
+ The Map-Reduce job has to be configured to use this MapRunnable class (using
+ the JobConf.setMapRunnerClass method) and
+ the number of threads the thread-pool can use with the
+ mapred.map.multithreadedrunner.threads property, its default
+ value is 10 threads.
+
]]> +
ClusterMetrics provides clients with information such as:
+ Clients can query for the latest ClusterMetrics, via
+ {@link Cluster#getClusterStatus()}.
Counters represent global counters, defined either by the
+ Map-Reduce framework or applications. Each Counter is named by
+ an {@link Enum} and has a long for the value.
+
+ Counters are bunched into Groups, each comprising of
+ counters from a particular Enum class.]]>
+
Counters holds per job/task counters, defined either by the
+ Map-Reduce framework or applications. Each Counter can be of
+ any {@link Enum} type.
+
+ Counters are bunched into {@link CounterGroup}s, each
+ comprising of counters from a particular Enum class.]]>
+
Note: The split is a logical split of the inputs and the + input files are not physically split into chunks. For e.g. a split could + be <input-file-path, start, offset> tuple. The InputFormat + also creates the {@link RecordReader} to read the {@link InputSplit}. + + @param context job configuration. + @return an array of {@link InputSplit}s for the job.]]> +
The Map-Reduce framework relies on the InputFormat of the
+ job to:
+
InputSplit for processing by
+ the {@link Mapper}.
+ The default behavior of file-based {@link InputFormat}s, typically + sub-classes of {@link FileInputFormat}, is to split the + input into logical {@link InputSplit}s based on the total size, in + bytes, of the input files. However, the {@link FileSystem} blocksize of + the input files is treated as an upper bound for input splits. A lower bound + on the split size can be set via + + mapreduce.input.fileinputformat.split.minsize.
+ +Clearly, logical splits based on input-size is insufficient for many
+ applications since record boundaries are to respected. In such cases, the
+ application has to also implement a {@link RecordReader} on whom lies the
+ responsibility to respect record-boundaries and present a record-oriented
+ view of the logical InputSplit to the individual task.
+
+ @see InputSplit
+ @see RecordReader
+ @see FileInputFormat]]>
+
Typically, it presents a byte-oriented view on the input and is the + responsibility of {@link RecordReader} of the job to process this and present + a record-oriented view. + + @see InputFormat + @see RecordReader]]> +
Configuration so
+ that any necessary internal modifications do not reflect on the incoming
+ parameter.
+
+ A Cluster will be created from the conf parameter only when it's needed.
+
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+ Configuration so
+ that any necessary internal modifications do not reflect on the incoming
+ parameter.
+
+ @param conf the configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+ Configuration so
+ that any necessary internal modifications do not reflect on the incoming
+ parameter.
+
+ @param status job status
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException]]>
+ Configuration so
+ that any necessary internal modifications do not reflect on the incoming
+ parameter.
+
+ @param ignored
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance()}]]>
+ Configuration so
+ that any necessary internal modifications do not reflect on the incoming
+ parameter.
+
+ @param ignored
+ @param conf job configuration
+ @return the {@link Job} , with no connection to a cluster yet.
+ @throws IOException
+ @deprecated Use {@link #getInstance(Configuration)}]]>
+ false.
+ @throws IOException]]>
+ false.
+ @throws IOException]]>
+ false.]]>
+ false.]]>
+ false.]]>
+ + Normally the user creates the application, describes various facets of the + job via {@link Job} and then submits the job and monitor its progress.
+ +Here is an example on how to submit a job:
+
+ // Create a new Job
+ Job job = Job.getInstance();
+ job.setJarByClass(MyJob.class);
+
+ // Specify various job-specific parameters
+ job.setJobName("myjob");
+
+ job.setInputPath(new Path("in"));
+ job.setOutputPath(new Path("out"));
+
+ job.setMapperClass(MyJob.MyMapper.class);
+ job.setReducerClass(MyJob.MyReducer.class);
+
+ // Submit the job, then poll for progress until the job is complete
+ job.waitForCompletion(true);
+ job_200707121733_0003 , which represents the third job
+ running at the jobtracker started at 200707121733.
+ + Applications should never construct or parse JobID strings, but rather + use appropriate constructors or {@link #forName(String)} method. + + @see TaskID + @see TaskAttemptID]]> +
The Hadoop Map-Reduce framework spawns one map task for each
+ {@link InputSplit} generated by the {@link InputFormat} for the job.
+ Mapper implementations can access the {@link Configuration} for
+ the job via the {@link JobContext#getConfiguration()}.
+
+
The framework first calls
+ {@link #setup(org.apache.hadoop.mapreduce.Mapper.Context)}, followed by
+ {@link #map(Object, Object, org.apache.hadoop.mapreduce.Mapper.Context)}
+ for each key/value pair in the InputSplit. Finally
+ {@link #cleanup(org.apache.hadoop.mapreduce.Mapper.Context)} is called.
All intermediate values associated with a given output key are + subsequently grouped by the framework, and passed to a {@link Reducer} to + determine the final output. Users can control the sorting and grouping by + specifying two key {@link RawComparator} classes.
+ +The Mapper outputs are partitioned per
+ Reducer. Users can control which keys (and hence records) go to
+ which Reducer by implementing a custom {@link Partitioner}.
+
+
Users can optionally specify a combiner, via
+ {@link Job#setCombinerClass(Class)}, to perform local aggregation of the
+ intermediate outputs, which helps to cut down the amount of data transferred
+ from the Mapper to the Reducer.
+
+
Applications can specify if and how the intermediate
+ outputs are to be compressed and which {@link CompressionCodec}s are to be
+ used via the Configuration.
If the job has zero
+ reduces then the output of the Mapper is directly written
+ to the {@link OutputFormat} without sorting by keys.
Example:
+
+ public class TokenCounterMapper
+ extends Mapper<Object, Text, Text, IntWritable>{
+
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
+ }
+ }
+ Applications may override the
+ {@link #run(org.apache.hadoop.mapreduce.Mapper.Context)} method to exert
+ greater control on map processing e.g. multi-threaded Mappers
+ etc.
false otherwise
+ @see #recoverTask(TaskAttemptContext)
+ @deprecated Use {@link #isRecoverySupported(JobContext)} instead.]]>
+ false otherwise
+ @throws IOException]]>
+ false otherwise
+ @throws IOException
+ @see #recoverTask(TaskAttemptContext)]]>
+ The Map-Reduce framework relies on the OutputCommitter of
+ the job to:
+
The Map-Reduce framework relies on the OutputFormat of the
+ job to:
+
key.]]>
+ Partitioner controls the partitioning of the keys of the
+ intermediate map-outputs. The key (or a subset of the key) is used to derive
+ the partition, typically by a hash function. The total number of partitions
+ is the same as the number of reduce tasks for the job. Hence this controls
+ which of the m reduce tasks the intermediate key (and hence the
+ record) is sent for reduction.
+
+ Note: A Partitioner is created only when there are multiple
+ reducers.
Note: If you require your Partitioner class to obtain the Job's + configuration object, implement the {@link Configurable} interface.
+ + @see Reducer]]> +RecordWriter implementations write the job outputs to the
+ {@link FileSystem}.
+
+ @see OutputFormat]]>
+
Reducer implementations
+ can access the {@link Configuration} for the job via the
+ {@link JobContext#getConfiguration()} method.
+
+ Reducer has 3 primary phases:
The Reducer copies the sorted output from each
+ {@link Mapper} using HTTP across the network.
The framework merge sorts Reducer inputs by
+ keys
+ (since different Mappers may have output the same key).
The shuffle and sort phases occur simultaneously i.e. while outputs are + being fetched they are merged.
+ + SecondarySort + +To achieve a secondary sort on the values returned by the value + iterator, the application should extend the key with the secondary + key and define a grouping comparator. The keys will be sorted using the + entire key, but will be grouped using the grouping comparator to decide + which keys and values are sent in the same call to reduce.The grouping + comparator is specified via + {@link Job#setGroupingComparatorClass(Class)}. The sort order is + controlled by + {@link Job#setSortComparatorClass(Class)}.
+ + + For example, say that you want to find duplicate web pages and tag them + all with the url of the "best" known example. You would set up the job + like: +In this phase the
+ {@link #reduce(Object, Iterable, org.apache.hadoop.mapreduce.Reducer.Context)}
+ method is called for each <key, (collection of values)> in
+ the sorted inputs.
The output of the reduce task is typically written to a + {@link RecordWriter} via + {@link Context#write(Object, Object)}.
+The output of the Reducer is not re-sorted.
Example:
+
+ public class IntSumReducer<Key> extends Reducer<Key,IntWritable,
+ Key,IntWritable> {
+ private IntWritable result = new IntWritable();
+
+ public void reduce(Key key, Iterable<IntWritable> values,
+ Context context) throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(key, result);
+ }
+ }
+ Counter for the given counterName]]>
+ counterName.
+ @param counterName counter name
+ @return the Counter for the given groupName and
+ counterName]]>
+ attempt_200707121733_0003_m_000005_0 , which represents the
+ zeroth task attempt for the fifth map task in the third job
+ running at the jobtracker started at 200707121733.
+ + Applications should never construct or parse TaskAttemptID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskID]]> +
task_200707121733_0003_m_000005 , which represents the
+ fifth map task in the third job running at the jobtracker
+ started at 200707121733.
+ + Applications should never construct or parse TaskID strings + , but rather use appropriate constructors or {@link #forName(String)} + method. + + @see JobID + @see TaskAttemptID]]> +
mapperConf, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+
+ + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain +
+ + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the +Configuration(boolean loadDefaults) constructor with
+ FALSE.]]>
+ + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed in a chain. This enables having + reusable specialized Mappers that can be combined to perform composite + operations within a single task. +
++ Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +
+
+ Using the ChainMapper and the ChainReducer classes is possible to compose
+ Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+
+ IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the chain. +
+ ChainMapper usage pattern: ++ +
+ ... + Job = new Job(conf); + + Configuration mapAConf = new Configuration(false); + ... + ChainMapper.addMapper(job, AMap.class, LongWritable.class, Text.class, + Text.class, Text.class, true, mapAConf); + + Configuration mapBConf = new Configuration(false); + ... + ChainMapper.addMapper(job, BMap.class, Text.class, Text.class, + LongWritable.class, Text.class, false, mapBConf); + + ... + + job.waitForComplettion(true); + ... +]]> +
reducerConf, have precedence over the job's Configuration.
+ This precedence is in effect when the task is running.
+
+ + IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +
+ + @param job + the job + @param klass + the Reducer class to add. + @param inputKeyClass + reducer input key class. + @param inputValueClass + reducer input value class. + @param outputKeyClass + reducer output key class. + @param outputValueClass + reducer output value class. + @param reducerConf + a configuration for the Reducer class. It is recommended to use a + Configuration without default values using the +Configuration(boolean loadDefaults) constructor with
+ FALSE.]]>
+ mapperConf, have precedence over the job's Configuration. This
+ precedence is in effect when the task is running.
+
+ + IMPORTANT: There is no need to specify the output key/value classes for the + ChainMapper, this is done by the addMapper for the last mapper in the + chain. +
+ + @param job + The job. + @param klass + the Mapper class to add. + @param inputKeyClass + mapper input key class. + @param inputValueClass + mapper input value class. + @param outputKeyClass + mapper output key class. + @param outputValueClass + mapper output value class. + @param mapperConf + a configuration for the Mapper class. It is recommended to use a + Configuration without default values using the +Configuration(boolean loadDefaults) constructor with
+ FALSE.]]>
+ + The key functionality of this feature is that the Mappers in the chain do not + need to be aware that they are executed after the Reducer or in a chain. This + enables having reusable specialized Mappers that can be combined to perform + composite operations within a single task. +
++ Special care has to be taken when creating chains that the key/values output + by a Mapper are valid for the following Mapper in the chain. It is assumed + all Mappers and the Reduce in the chain use matching output and input key and + value classes as no conversion is done by the chaining code. +
+ Using the ChainMapper and the ChainReducer classes is possible to
+ compose Map/Reduce jobs that look like [MAP+ / REDUCE MAP*]. And
+ immediate benefit of this pattern is a dramatic reduction in disk IO.
+ IMPORTANT: There is no need to specify the output key/value classes for the + ChainReducer, this is done by the setReducer or the addMapper for the last + element in the chain. +
+ ChainReducer usage pattern: ++ +
+ ... + Job = new Job(conf); + .... + + Configuration reduceConf = new Configuration(false); + ... + ChainReducer.setReducer(job, XReduce.class, LongWritable.class, Text.class, + Text.class, Text.class, true, reduceConf); + + ChainReducer.addMapper(job, CMap.class, Text.class, Text.class, + LongWritable.class, Text.class, false, null); + + ChainReducer.addMapper(job, DMap.class, LongWritable.class, Text.class, + LongWritable.class, LongWritable.class, true, null); + + ... + + job.waitForCompletion(true); + ... +]]> +
+ Implementations are responsible for writing the fields of the object + to PreparedStatement, and reading the fields of the object from the + ResultSet. + +
Example:
+ If we have the following table in the database : ++ CREATE TABLE MyTable ( + counter INTEGER NOT NULL, + timestamp BIGINT NOT NULL, + ); ++ then we can read/write the tuples from/to the table with : +
+ public class MyWritable implements Writable, DBWritable {
+ // Some data
+ private int counter;
+ private long timestamp;
+
+ //Writable#write() implementation
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(counter);
+ out.writeLong(timestamp);
+ }
+
+ //Writable#readFields() implementation
+ public void readFields(DataInput in) throws IOException {
+ counter = in.readInt();
+ timestamp = in.readLong();
+ }
+
+ public void write(PreparedStatement statement) throws SQLException {
+ statement.setInt(1, counter);
+ statement.setLong(2, timestamp);
+ }
+
+ public void readFields(ResultSet resultSet) throws SQLException {
+ counter = resultSet.getInt(1);
+ timestamp = resultSet.getLong(2);
+ }
+ }
+ ]]>
+ CombineFileSplit's.
+
+ @see CombineFileSplit]]>
+ SequenceFileInputFormat.
+
+ @see CombineFileInputFormat]]>
+ TextInputFormat.
+
+ @see CombineFileInputFormat]]>
+ FileInputFormat implementations can override this and return
+ false to ensure that individual input files are never split-up
+ so that {@link Mapper}s process entire files.
+
+ @param context the job context
+ @param filename the file name to check
+ @return is this file splitable?]]>
+ FileInputFormat is the base class for all file-based
+ InputFormats. This provides a generic implementation of
+ {@link #getSplits(JobContext)}.
+
+ Implementations of FileInputFormat can also override the
+ {@link #isSplitable(JobContext, Path)} method to prevent input files
+ from being split-up in certain situations. Implementations that may
+ deal with non-splittable files must override this method, since
+ the default implementation assumes splitting is always possible.]]>
+ ) }]]> +
+ Mapper implementations using this MapRunnable must be thread-safe. +
+ The Map-Reduce job has to be configured with the mapper to use via + {@link #setMapperClass(Job, Class)} and + the number of thread the thread-pool can use with the + {@link #getNumberOfThreads(JobContext)} method. The default + value is 10 threads. +
]]> +
Mapper.Context for custom implementations]]>
+ false otherwise]]>
+ Some applications need to create/write-to side-files, which differ from + the actual job-outputs. + +
In such cases there could be issues with 2 instances of the same TIP + (running simultaneously e.g. speculative tasks) trying to open/write-to the + same file (path) on HDFS. Hence the application-writer will have to pick + unique names per task-attempt (e.g. using the attemptid, say + attempt_200709221812_0001_m_000000_0), not just per TIP.
+ +To get around this the Map-Reduce framework helps the application-writer + out by maintaining a special + ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} + sub-directory for each task-attempt on HDFS where the output of the + task-attempt goes. On successful completion of the task-attempt the files + in the ${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid} (only) + are promoted to ${mapreduce.output.fileoutputformat.outputdir}. Of course, the + framework discards the sub-directory of unsuccessful task-attempts. This + is completely transparent to the application.
+ +The application-writer can take advantage of this by creating any + side-files required in a work directory during execution + of his task i.e. via + {@link #getWorkOutputPath(TaskInputOutputContext)}, and + the framework will move them out similarly - thus she doesn't have to pick + unique paths per task-attempt.
+ +The entire discussion holds true for maps of jobs with + reducer=NONE (i.e. 0 reduces) since output of the map, in that case, + goes directly to HDFS.
+ + @return the {@link Path} to the task's temporary output directory + for the map-reduce job.]]> +This method uses the {@link #getUniqueFile} method to make the file name + unique for the task.
+ + @param context the context for the task. + @param name the name for the file. + @param extension the extension for the file + @return a unique path accross all tasks of the job.]]> +close()]]>
+ OutputFormat, with its own key class and with its own value
+ class.
+
+
+ + Case two: to write data to different files provided by user +
+ ++ MultipleOutputs supports counters, by default they are disabled. The + counters group is the {@link MultipleOutputs} class name. The names of the + counters are the same as the output name. These count the number records + written to each output name. +
+ + Usage pattern for job submission: ++ + Job job = new Job(); + + FileInputFormat.setInputPath(job, inDir); + FileOutputFormat.setOutputPath(job, outDir); + + job.setMapperClass(MOMap.class); + job.setReducerClass(MOReduce.class); + ... + + // Defines additional single text based output 'text' for the job + MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class, + LongWritable.class, Text.class); + + // Defines additional sequence-file based output 'sequence' for the job + MultipleOutputs.addNamedOutput(job, "seq", + SequenceFileOutputFormat.class, + LongWritable.class, Text.class); + ... + + job.waitForCompletion(true); + ... ++
+ Usage in Reducer: +
+ <K, V> String generateFileName(K k, V v) {
+ return k.toString() + "_" + v.toString();
+ }
+
+ public class MOReduce extends
+ Reducer<WritableComparable, Writable,WritableComparable, Writable> {
+ private MultipleOutputs mos;
+ public void setup(Context context) {
+ ...
+ mos = new MultipleOutputs(context);
+ }
+
+ public void reduce(WritableComparable key, Iterator<Writable> values,
+ Context context)
+ throws IOException {
+ ...
+ mos.write("text", , key, new Text("Hello"));
+ mos.write("seq", LongWritable(1), new Text("Bye"), "seq_a");
+ mos.write("seq", LongWritable(2), key, new Text("Chau"), "seq_b");
+ mos.write(key, new Text("value"), generateFileName(key, new Text("value")));
+ ...
+ }
+
+ public void cleanup(Context) throws IOException {
+ mos.close();
+ ...
+ }
+
+ }
+
+
+ + When used in conjuction with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat, + MultipleOutputs can mimic the behaviour of MultipleTextOutputFormat and MultipleSequenceFileOutputFormat + from the old Hadoop API - ie, output can be written from the Reducer to more than one location. +
+ +
+ Use MultipleOutputs.write(KEYOUT key, VALUEOUT value, String baseOutputPath) to write key and
+ value to a path specified by baseOutputPath, with no need to specify a named output.
+ Warning: when the baseOutputPath passed to MultipleOutputs.write
+ is a path that resolves outside of the final job output directory, the
+ directory is created immediately and then persists through subsequent
+ task retries, breaking the concept of output committing:
+
+ private MultipleOutputs<Text, Text> out;
+
+ public void setup(Context context) {
+ out = new MultipleOutputs<Text, Text>(context);
+ ...
+ }
+
+ public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
+ for (Text t : values) {
+ out.write(key, t, generateFileName(<parameter list...>));
+ }
+ }
+
+ protected void cleanup(Context context) throws IOException, InterruptedException {
+ out.close();
+ }
+
+
+
+ Use your own code in generateFileName() to create a custom path to your results.
+ '/' characters in baseOutputPath will be translated into directory levels in your file system.
+ Also, append your custom-generated path with "part" or similar, otherwise your output will be -00000, -00001 etc.
+ No call to context.write() is necessary. See example generateFileName() code below.
+
+ private String generateFileName(Text k) {
+ // expect Text k in format "Surname|Forename"
+ String[] kStr = k.toString().split("\\|");
+
+ String sName = kStr[0];
+ String fName = kStr[1];
+
+ // example for k = Smith|John
+ // output written to /user/hadoop/path/to/output/Smith/John-r-00000 (etc)
+ return sName + "/" + fName;
+ }
+
+
+
+ Using MultipleOutputs in this way will still create zero-sized default output, eg part-00000.
+ To prevent this use LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);
+ instead of job.setOutputFormatClass(TextOutputFormat.class); in your Hadoop job configuration.
+
The subarray to be used for the partitioning can be defined by means + of the following properties: +
+ +---+---+---+---+---+
+ | B | B | B | B | B |
+ +---+---+---+---+---+
+ 0 1 2 3 4
+ -5 -4 -3 -2 -1
+ i
+ and j as first and last element, repectively, when
+ i and j are the left and right offset.
+
+ For Hadoop programs written in Java, it is advisable to use one of + the following static convenience methods for setting the offsets: +
Reducer.Context for custom implementations]]>
+
+ * This has the problem: it can consume a lot of heap space
+ * proportional to the mismatch between writes to the stream and
+ * the JVM-wide upload bandwidth to the OSS endpoint.
+ * The memory consumption can be limited by tuning the filesystem settings
+ * to restrict the number of queued/active uploads.
+ */
+
+ class ByteArrayBlock extends DataBlock {
+ private OSSByteArrayOutputStream buffer;
+ private final int limit;
+ // cache data size so that it is consistent after the buffer is reset.
+ private Integer dataSize;
+
+ ByteArrayBlock(long index,
+ int limit,
+ BlockOutputStreamStatistics statistics) throws MemoryLimitException {
+ super(index, statistics);
+ this.limit = limit;
+ allocateMemory(limit);
+ buffer = new OSSByteArrayOutputStream(limit);
+ blockAllocated();
+ bytesAllocated(limit);
+ }
+
+ /**
+ * Get the amount of data; if there is no buffer then the size is 0.
+ *
+ * @return the amount of data available to upload.
+ */
+ @Override
+ int dataSize() {
+ return dataSize != null ? dataSize : buffer.size();
+ }
+
+ @Override
+ BlockUploadData startUpload() throws IOException {
+ super.startUpload();
+ dataSize = buffer.size();
+ ByteArrayInputStream bufferData = buffer.getInputStream();
+ buffer = null;
+ return new BlockUploadData(bufferData);
+ }
+
+ @Override
+ boolean hasCapacity(long bytes) {
+ return dataSize() + bytes <= limit;
+ }
+
+ @Override
+ int remainingCapacity() {
+ return limit - dataSize();
+ }
+
+ @Override
+ int write(byte[] b, int offset, int len) throws IOException {
+ super.write(b, offset, len);
+ int written = Math.min(remainingCapacity(), len);
+ buffer.write(b, offset, written);
+ return written;
+ }
+
+ @Override
+ protected void innerClose() {
+ buffer = null;
+ releaseMemory(limit);
+ blockReleased();
+ bytesReleased(limit);
+ }
+
+ @Override
+ public String toString() {
+ return "ByteArrayBlock{"
+ + "index=" + getIndex() +
+ ", state=" + getState() +
+ ", limit=" + limit +
+ ", dataSize=" + dataSize +
+ '}';
+ }
+ }
+ }
+
+ // ====================================================================
+
+ /**
+ * Stream via Direct ByteBuffers; these are allocated off heap
+ * via {@link DirectBufferPool}.
+ */
+ static class ByteBufferBlockFactory extends MemoryBlockFactory {
+ private final DirectBufferPool bufferPool = new DirectBufferPool();
+ private final AtomicInteger buffersOutstanding = new AtomicInteger(0);
+
+ ByteBufferBlockFactory(AliyunOSSFileSystem owner) {
+ super(owner);
+ }
+
+ @Override
+ ByteBufferBlock create(long index, int limit,
+ BlockOutputStreamStatistics statistics)
+ throws IOException {
+ try {
+ return new ByteBufferBlock(index, limit, statistics);
+ } catch (MemoryLimitException e) {
+ LOG.debug(e.getMessage() + ", index " + index);
+ return null;
+ }
+ }
+
+ private ByteBuffer requestBuffer(int limit) {
+ LOG.debug("Requesting buffer of size {}", limit);
+ buffersOutstanding.incrementAndGet();
+ return bufferPool.getBuffer(limit);
+ }
+
+ private void releaseBuffer(ByteBuffer buffer) {
+ LOG.debug("Releasing buffer");
+ bufferPool.returnBuffer(buffer);
+ buffersOutstanding.decrementAndGet();
+ }
+
+ /**
+ * Get count of outstanding buffers.
+ * @return the current buffer count
+ */
+ public int getOutstandingBufferCount() {
+ return buffersOutstanding.get();
+ }
+
+ @Override
+ public String toString() {
+ return "ByteBufferBlockFactory{"
+ + "buffersOutstanding=" + buffersOutstanding +
+ '}';
+ }
+
+ /**
+ * A DataBlock which requests a buffer from pool on creation; returns
+ * it when it is closed.
+ */
+ class ByteBufferBlock extends DataBlock {
+ private ByteBuffer blockBuffer;
+ private final int bufferSize;
+ // cache data size so that it is consistent after the buffer is reset.
+ private Integer dataSize;
+
+ /**
+ * Instantiate. This will request a ByteBuffer of the desired size.
+ * @param index block index
+ * @param bufferSize buffer size
+ */
+ ByteBufferBlock(long index, int bufferSize,
+ BlockOutputStreamStatistics statistics) throws MemoryLimitException {
+ super(index, statistics);
+ this.bufferSize = bufferSize;
+ allocateMemory(bufferSize);
+ blockBuffer = requestBuffer(bufferSize);
+ blockAllocated();
+ bytesAllocated(bufferSize);
+ }
+
+ /**
+ * Get the amount of data; if there is no buffer then the size is 0.
+ * @return the amount of data available to upload.
+ */
+ @Override
+ int dataSize() {
+ return dataSize != null ? dataSize : bufferCapacityUsed();
+ }
+
+ @Override
+ BlockUploadData startUpload() throws IOException {
+ super.startUpload();
+ dataSize = bufferCapacityUsed();
+ // set the buffer up from reading from the beginning
+ blockBuffer.limit(blockBuffer.position());
+ blockBuffer.position(0);
+ return new BlockUploadData(
+ new ByteBufferInputStream(dataSize, blockBuffer));
+ }
+
+ @Override
+ public boolean hasCapacity(long bytes) {
+ return bytes <= remainingCapacity();
+ }
+
+ @Override
+ public int remainingCapacity() {
+ return blockBuffer != null ? blockBuffer.remaining() : 0;
+ }
+
+ private int bufferCapacityUsed() {
+ return blockBuffer.capacity() - blockBuffer.remaining();
+ }
+
+ @Override
+ int write(byte[] b, int offset, int len) throws IOException {
+ super.write(b, offset, len);
+ int written = Math.min(remainingCapacity(), len);
+ blockBuffer.put(b, offset, written);
+ return written;
+ }
+
+ /**
+ * Closing the block will release the buffer.
+ */
+ @Override
+ protected void innerClose() {
+ if (blockBuffer != null) {
+ releaseMemory(bufferSize);
+ blockReleased();
+ bytesReleased(bufferSize);
+ releaseBuffer(blockBuffer);
+ blockBuffer = null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "ByteBufferBlock{"
+ + "index=" + getIndex() +
+ ", state=" + getState() +
+ ", dataSize=" + dataSize() +
+ ", limit=" + bufferSize +
+ ", remainingCapacity=" + remainingCapacity() +
+ '}';
+ }
+
+ /**
+ * Provide an input stream from a byte buffer; supporting
+ * {@link #mark(int)}, which is required to enable replay of failed
+ * PUT attempts.
+ */
+ class ByteBufferInputStream extends InputStream {
+
+ private final int size;
+ private ByteBuffer byteBuffer;
+
+ ByteBufferInputStream(int size,
+ ByteBuffer byteBuffer) {
+ LOG.debug("Creating ByteBufferInputStream of size {}", size);
+ this.size = size;
+ this.byteBuffer = byteBuffer;
+ }
+
+ /**
+ * After the stream is closed, set the local reference to the byte
+ * buffer to null; this guarantees that future attempts to use
+ * stream methods will fail.
+ */
+ @Override
+ public synchronized void close() {
+ LOG.debug("ByteBufferInputStream.close() for {}",
+ ByteBufferBlock.super.toString());
+ byteBuffer = null;
+ }
+
+ /**
+ * Verify that the stream is open.
+ * @throws IOException if the stream is closed
+ */
+ private void verifyOpen() throws IOException {
+ if (byteBuffer == null) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ }
+
+ public synchronized int read() throws IOException {
+ if (available() > 0) {
+ return byteBuffer.get() & 0xFF;
+ } else {
+ return -1;
+ }
+ }
+
+ @Override
+ public synchronized long skip(long offset) throws IOException {
+ verifyOpen();
+ long newPos = position() + offset;
+ if (newPos < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ if (newPos > size) {
+ throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+ }
+ byteBuffer.position((int) newPos);
+ return newPos;
+ }
+
+ @Override
+ public synchronized int available() {
+ Preconditions.checkState(byteBuffer != null,
+ FSExceptionMessages.STREAM_IS_CLOSED);
+ return byteBuffer.remaining();
+ }
+
+ /**
+ * Get the current buffer position.
+ * @return the buffer position
+ */
+ public synchronized int position() {
+ return byteBuffer.position();
+ }
+
+ /**
+ * Check if there is data left.
+ * @return true if there is data remaining in the buffer.
+ */
+ public synchronized boolean hasRemaining() {
+ return byteBuffer.hasRemaining();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ LOG.debug("mark at {}", position());
+ byteBuffer.mark();
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ LOG.debug("reset");
+ byteBuffer.reset();
+ }
+
+ @Override
+ public boolean markSupported() {
+ return true;
+ }
+
+ /**
+ * Read in data.
+ * @param b destination buffer
+ * @param offset offset within the buffer
+ * @param length length of bytes to read
+ * @throws EOFException if the position is negative
+ * @throws IndexOutOfBoundsException if there isn't space for the
+ * amount of data requested.
+ * @throws IllegalArgumentException other arguments are invalid.
+ */
+ @SuppressWarnings("NullableProblems")
+ public synchronized int read(byte[] b, int offset, int length)
+ throws IOException {
+ Preconditions.checkArgument(length >= 0, "length is negative");
+ Preconditions.checkArgument(b != null, "Null buffer");
+ if (b.length - offset < length) {
+ throw new IndexOutOfBoundsException(
+ FSExceptionMessages.TOO_MANY_BYTES_FOR_DEST_BUFFER
+ + ": request length =" + length
+ + ", with offset =" + offset
+ + "; buffer capacity =" + (b.length - offset));
+ }
+ verifyOpen();
+ if (!hasRemaining()) {
+ return -1;
+ }
+
+ int toRead = Math.min(length, available());
+ byteBuffer.get(b, offset, toRead);
+ return toRead;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ByteBufferInputStream{");
+ sb.append("size=").append(size);
+ ByteBuffer buf = this.byteBuffer;
+ if (buf != null) {
+ sb.append(", available=").append(buf.remaining());
+ }
+ sb.append(", ").append(ByteBufferBlock.super.toString());
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+ }
+ }
+
+ // ====================================================================
+
+ /**
+ * Buffer blocks to disk.
+ */
+ static class DiskBlockFactory extends BlockFactory {
+
+ DiskBlockFactory(AliyunOSSFileSystem owner) {
+ super(owner);
+ }
+
+ /**
+ * Create a temp file and a {@link DiskBlock} instance to manage it.
+ *
+ * @param index block index
+ * @param limit limit of the block.
+ * @return the new block
+ * @throws IOException IO problems
+ */
+ @Override
+ DataBlock create(long index, int limit,
+ BlockOutputStreamStatistics statistics)
+ throws IOException {
+ File destFile = AliyunOSSUtils.createTmpFileForWrite(
+ String.format("oss-block-%04d-", index), limit, getOwner().getConf());
+ return new DiskBlock(destFile, limit, index, statistics);
+ }
+ }
+
+ /**
+ * Stream to a file.
+ * This will stop at the limit; the caller is expected to create a new block.
+ */
+ static class DiskBlock extends DataBlock {
+
+ private int bytesWritten = 0;
+ private final File bufferFile;
+ private final int limit;
+ private BufferedOutputStream out;
+ private final AtomicBoolean closed = new AtomicBoolean(false);
+
+ DiskBlock(File bufferFile,
+ int limit,
+ long index,
+ BlockOutputStreamStatistics statistics)
+ throws FileNotFoundException {
+ super(index, statistics);
+ this.limit = limit;
+ this.bufferFile = bufferFile;
+ blockAllocated();
+ diskBlockAllocated();
+ out = new BufferedOutputStream(new FileOutputStream(bufferFile));
+ }
+
+ @Override
+ int dataSize() {
+ return bytesWritten;
+ }
+
+ @Override
+ boolean hasCapacity(long bytes) {
+ return dataSize() + bytes <= limit;
+ }
+
+ @Override
+ int remainingCapacity() {
+ return limit - bytesWritten;
+ }
+
+ @Override
+ int write(byte[] b, int offset, int len) throws IOException {
+ super.write(b, offset, len);
+ int written = Math.min(remainingCapacity(), len);
+ out.write(b, offset, written);
+ bytesWritten += written;
+ return written;
+ }
+
+ @Override
+ BlockUploadData startUpload() throws IOException {
+ super.startUpload();
+ try {
+ out.flush();
+ } finally {
+ out.close();
+ out = null;
+ }
+ return new BlockUploadData(bufferFile);
+ }
+
+ /**
+ * The close operation will delete the destination file if it still
+ * exists.
+ * @throws IOException IO problems
+ */
+ @SuppressWarnings("UnnecessaryDefault")
+ @Override
+ protected void innerClose() throws IOException {
+ final DestState state = getState();
+ LOG.debug("Closing {}", this);
+ switch (state) {
+ case Writing:
+ if (bufferFile.exists()) {
+ // file was not uploaded
+ LOG.debug("Block[{}]: Deleting buffer file as upload did not start",
+ getIndex());
+ closeBlock();
+ }
+ break;
+
+ case Upload:
+ LOG.debug("Block[{}]: Buffer file {} exists —close upload stream",
+ getIndex(), bufferFile);
+ break;
+
+ case Closed:
+ closeBlock();
+ break;
+
+ default:
+ // this state can never be reached, but checkstyle complains, so
+ // it is here.
+ }
+ }
+
+ /**
+ * Flush operation will flush to disk.
+ * @throws IOException IOE raised on FileOutputStream
+ */
+ @Override
+ void flush() throws IOException {
+ super.flush();
+ out.flush();
+ }
+
+ @Override
+ public String toString() {
+ String sb = "FileBlock{"
+ + "index=" + getIndex()
+ + ", destFile=" + bufferFile +
+ ", state=" + getState() +
+ ", dataSize=" + dataSize() +
+ ", limit=" + limit +
+ '}';
+ return sb;
+ }
+
+ /**
+ * Close the block.
+ * This will delete the block's buffer file if the block has
+ * not previously been closed.
+ */
+ void closeBlock() {
+ LOG.debug("block[{}]: closeBlock()", getIndex());
+ if (!closed.getAndSet(true)) {
+ blockReleased();
+ diskBlockReleased();
+ if (!bufferFile.delete() && bufferFile.exists()) {
+ LOG.warn("delete({}) returned false",
+ bufferFile.getAbsoluteFile());
+ }
+ } else {
+ LOG.debug("block[{}]: skipping re-entrant closeBlock()", getIndex());
+ }
+ }
+ }
+
+ /**
+ * Buffer blocks to memory and fallback to disk if
+ * used memory exceed the quota.
+ */
+ static class MemoryAndDiskBlockFactory extends BlockFactory {
+ private BlockFactory memoryFactory;
+ private BlockFactory diskFactory;
+
+ MemoryAndDiskBlockFactory(AliyunOSSFileSystem owner,
+ BlockFactory memoryFactory) {
+ super(owner);
+ this.memoryFactory = memoryFactory;
+ diskFactory = new DiskBlockFactory(owner);
+
+ long memoryLimit = owner.getConf().getLong(
+ Constants.FAST_UPLOAD_BUFFER_MEMORY_LIMIT,
+ Constants.FAST_UPLOAD_BUFFER_MEMORY_LIMIT_DEFAULT);
+ ((MemoryBlockFactory)this.memoryFactory).setMemoryLimit(memoryLimit);
+ }
+
+ /**
+ * Create a temp file and a {@link DataBlock} instance to manage it.
+ *
+ * @param index block index
+ * @param limit limit of the block.
+ * @return the new block
+ * @throws IOException IO problems
+ */
+ @Override
+ DataBlock create(long index, int limit,
+ BlockOutputStreamStatistics statistics)
+ throws IOException {
+ DataBlock block = memoryFactory.create(index, limit, statistics);
+ if (block != null) {
+ return block;
+ } else {
+ return diskFactory.create(index, limit, statistics);
+ }
+ }
+
+ @VisibleForTesting
+ MemoryBlockFactory getMemoryFactory() {
+ return (MemoryBlockFactory)memoryFactory;
+ }
+ }
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/BlockOutputStreamStatistics.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/BlockOutputStreamStatistics.java
new file mode 100644
index 00000000000..51940b71831
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/BlockOutputStreamStatistics.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.statistics;
+
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Block output stream statistics.
+ */
+@InterfaceStability.Unstable
+public interface BlockOutputStreamStatistics {
+
+ /**
+ * A block has been allocated.
+ */
+ void blockAllocated();
+
+ /**
+ * A block has been released.
+ */
+ void blockReleased();
+
+ /**
+ * A disk block has been allocated.
+ */
+ void diskBlockAllocated();
+
+ /**
+ * A disk block has been released.
+ */
+ void diskBlockReleased();
+
+ /**
+ * Memory bytes has been allocated.
+ * @param size allocated size.
+ */
+ void bytesAllocated(long size);
+
+ /**
+ * Memory bytes has been released.
+ * @param size released size.
+ */
+ void bytesReleased(long size);
+
+ int getBlocksAllocated();
+
+ int getBlocksReleased();
+
+ int getDiskBlocksAllocated();
+
+ int getDiskBlocksReleased();
+
+ long getBytesAllocated();
+
+ long getBytesReleased();
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/OutputStreamStatistics.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/OutputStreamStatistics.java
new file mode 100644
index 00000000000..011a2eecd08
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/OutputStreamStatistics.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.statistics.impl;
+
+import org.apache.hadoop.fs.aliyun.oss.statistics.BlockOutputStreamStatistics;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Implementation of {@link BlockOutputStreamStatistics}.
+ */
+public class OutputStreamStatistics implements BlockOutputStreamStatistics {
+ private final AtomicInteger blocksAllocated = new AtomicInteger(0);
+ private final AtomicInteger blocksReleased = new AtomicInteger(0);
+
+ private final AtomicInteger diskBlocksAllocated = new AtomicInteger(0);
+ private final AtomicInteger diskBlocksReleased = new AtomicInteger(0);
+
+ private final AtomicLong bytesAllocated = new AtomicLong(0);
+ private final AtomicLong bytesReleased = new AtomicLong(0);
+
+ @Override
+ public void blockAllocated() {
+ blocksAllocated.incrementAndGet();
+ }
+
+ @Override
+ public void blockReleased() {
+ blocksReleased.incrementAndGet();
+ }
+
+ @Override
+ public void diskBlockAllocated() {
+ diskBlocksAllocated.incrementAndGet();
+ }
+
+ @Override
+ public void diskBlockReleased() {
+ diskBlocksReleased.incrementAndGet();
+ }
+
+ @Override
+ public int getBlocksAllocated() {
+ return blocksAllocated.get();
+ }
+
+ @Override
+ public int getBlocksReleased() {
+ return blocksReleased.get();
+ }
+
+ @Override
+ public int getDiskBlocksAllocated() {
+ return diskBlocksAllocated.get();
+ }
+
+ @Override
+ public int getDiskBlocksReleased() {
+ return diskBlocksReleased.get();
+ }
+
+ @Override
+ public void bytesAllocated(long size) {
+ bytesAllocated.getAndAdd(size);
+ }
+
+ @Override
+ public void bytesReleased(long size) {
+ bytesReleased.getAndAdd(size);
+ }
+
+ @Override
+ public long getBytesAllocated() {
+ return bytesAllocated.get();
+ }
+
+ @Override
+ public long getBytesReleased() {
+ return bytesReleased.get();
+ }
+}
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/package-info.java
new file mode 100644
index 00000000000..2f044173bc3
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/impl/package-info.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Statistics collection for the OSS connector: implementation.
+ * Not for use by anything outside the hadoop-aliyun source tree.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.aliyun.oss.statistics.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/package-info.java
new file mode 100644
index 00000000000..49abca73d5d
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/statistics/package-info.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Statistics collection for the OSS connector: interfaces.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.aliyun.oss.statistics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index d48bf1c6b02..578b9595d1d 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -164,7 +164,7 @@ please raise your issues with them.
+ * Default is {@link #DEFAULT_MULTIPART_UPLOAD_ENABLED}.
+ */
+ public static final String MULTIPART_UPLOADS_ENABLED = "fs.s3a.multipart.uploads.enabled";
+
+ /**
+ * Default value for multipart uploads.
+ * {@value}
+ */
+ public static final boolean DEFAULT_MULTIPART_UPLOAD_ENABLED = true;
+
+ /**
+ * Stream supports multipart uploads to the given path.
+ */
+ public static final String STORE_CAPABILITY_DIRECTORY_MARKER_MULTIPART_UPLOAD_ENABLED =
+ "fs.s3a.capability.multipart.uploads.enabled";
+
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
index ec433fa95c2..34779996963 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java
@@ -18,14 +18,16 @@
package org.apache.hadoop.fs.s3a;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DelegateToFileSystem;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
+import static org.apache.hadoop.fs.s3a.Constants.FS_S3A;
/**
* S3A implementation of AbstractFileSystem.
@@ -37,7 +39,8 @@ public class S3A extends DelegateToFileSystem {
public S3A(URI theUri, Configuration conf)
throws IOException, URISyntaxException {
- super(theUri, new S3AFileSystem(), conf, "s3a", false);
+ super(theUri, new S3AFileSystem(), conf,
+ theUri.getScheme().isEmpty() ? FS_S3A : theUri.getScheme(), false);
}
@Override
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index 19943ff2f70..df3c9315ba8 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -101,7 +101,7 @@ class S3ABlockOutputStream extends OutputStream implements
private final String key;
/** Size of all blocks. */
- private final int blockSize;
+ private final long blockSize;
/** IO Statistics. */
private final IOStatistics iostatistics;
@@ -169,6 +169,9 @@ class S3ABlockOutputStream extends OutputStream implements
/** Thread level IOStatistics Aggregator. */
private final IOStatisticsAggregator threadIOStatisticsAggregator;
+ /** Is multipart upload enabled? */
+ private final boolean isMultipartUploadEnabled;
+
/**
* An S3A output stream which uploads partitions in a separate pool of
* threads; different {@link S3ADataBlocks.BlockFactory}
@@ -181,7 +184,6 @@ class S3ABlockOutputStream extends OutputStream implements
this.builder = builder;
this.key = builder.key;
this.blockFactory = builder.blockFactory;
- this.blockSize = (int) builder.blockSize;
this.statistics = builder.statistics;
// test instantiations may not provide statistics;
this.iostatistics = statistics.getIOStatistics();
@@ -195,17 +197,26 @@ class S3ABlockOutputStream extends OutputStream implements
(ProgressListener) progress
: new ProgressableListener(progress);
downgradeSyncableExceptions = builder.downgradeSyncableExceptions;
- // create that first block. This guarantees that an open + close sequence
- // writes a 0-byte entry.
- createBlockIfNeeded();
- LOG.debug("Initialized S3ABlockOutputStream for {}" +
- " output to {}", key, activeBlock);
+
+ // look for multipart support.
+ this.isMultipartUploadEnabled = builder.isMultipartUploadEnabled;
+ // block size is infinite if multipart is disabled, so ignore
+ // what was passed in from the builder.
+ this.blockSize = isMultipartUploadEnabled
+ ? builder.blockSize
+ : -1;
+
if (putTracker.initialize()) {
LOG.debug("Put tracker requests multipart upload");
initMultipartUpload();
}
this.isCSEEnabled = builder.isCSEEnabled;
this.threadIOStatisticsAggregator = builder.ioStatisticsAggregator;
+ // create that first block. This guarantees that an open + close sequence
+ // writes a 0-byte entry.
+ createBlockIfNeeded();
+ LOG.debug("Initialized S3ABlockOutputStream for {}" +
+ " output to {}", key, activeBlock);
}
/**
@@ -318,7 +329,15 @@ class S3ABlockOutputStream extends OutputStream implements
statistics.writeBytes(len);
S3ADataBlocks.DataBlock block = createBlockIfNeeded();
int written = block.write(source, offset, len);
- int remainingCapacity = block.remainingCapacity();
+ if (!isMultipartUploadEnabled) {
+ // no need to check for space as multipart uploads
+ // are not available...everything is saved to a single
+ // (disk) block.
+ return;
+ }
+ // look to see if another block is needed to complete
+ // the upload or exactly a block was written.
+ int remainingCapacity = (int) block.remainingCapacity();
if (written < len) {
// not everything was written —the block has run out
// of capacity
@@ -369,6 +388,8 @@ class S3ABlockOutputStream extends OutputStream implements
*/
@Retries.RetryTranslated
private void initMultipartUpload() throws IOException {
+ Preconditions.checkState(isMultipartUploadEnabled,
+ "multipart upload is disabled");
if (multiPartUpload == null) {
LOG.debug("Initiating Multipart upload");
multiPartUpload = new MultiPartUpload(key);
@@ -558,19 +579,20 @@ class S3ABlockOutputStream extends OutputStream implements
}
/**
- * Upload the current block as a single PUT request; if the buffer
- * is empty a 0-byte PUT will be invoked, as it is needed to create an
- * entry at the far end.
- * @throws IOException any problem.
- * @return number of bytes uploaded. If thread was interrupted while
- * waiting for upload to complete, returns zero with interrupted flag set
- * on this thread.
+ * Upload the current block as a single PUT request; if the buffer is empty a
+ * 0-byte PUT will be invoked, as it is needed to create an entry at the far
+ * end.
+ * @return number of bytes uploaded. If thread was interrupted while waiting
+ * for upload to complete, returns zero with interrupted flag set on this
+ * thread.
+ * @throws IOException
+ * any problem.
*/
- private int putObject() throws IOException {
+ private long putObject() throws IOException {
LOG.debug("Executing regular upload for {}", writeOperationHelper);
final S3ADataBlocks.DataBlock block = getActiveBlock();
- int size = block.dataSize();
+ long size = block.dataSize();
final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
writeOperationHelper.createPutObjectRequest(
@@ -617,6 +639,7 @@ class S3ABlockOutputStream extends OutputStream implements
"S3ABlockOutputStream{");
sb.append(writeOperationHelper.toString());
sb.append(", blockSize=").append(blockSize);
+ sb.append(", isMultipartUploadEnabled=").append(isMultipartUploadEnabled);
// unsynced access; risks consistency in exchange for no risk of deadlock.
S3ADataBlocks.DataBlock block = activeBlock;
if (block != null) {
@@ -835,7 +858,7 @@ class S3ABlockOutputStream extends OutputStream implements
Preconditions.checkNotNull(uploadId, "Null uploadId");
maybeRethrowUploadFailure();
partsSubmitted++;
- final int size = block.dataSize();
+ final long size = block.dataSize();
bytesSubmitted += size;
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request;
@@ -1011,7 +1034,7 @@ class S3ABlockOutputStream extends OutputStream implements
ProgressEventType eventType = progressEvent.getEventType();
long bytesTransferred = progressEvent.getBytesTransferred();
- int size = block.dataSize();
+ long size = block.dataSize();
switch (eventType) {
case REQUEST_BYTE_TRANSFER_EVENT:
@@ -1126,6 +1149,11 @@ class S3ABlockOutputStream extends OutputStream implements
*/
private IOStatisticsAggregator ioStatisticsAggregator;
+ /**
+ * Is Multipart Uploads enabled for the given upload.
+ */
+ private boolean isMultipartUploadEnabled;
+
private BlockOutputStreamBuilder() {
}
@@ -1276,5 +1304,11 @@ class S3ABlockOutputStream extends OutputStream implements
ioStatisticsAggregator = value;
return this;
}
+
+ public BlockOutputStreamBuilder withMultipartEnabled(
+ final boolean value) {
+ isMultipartUploadEnabled = value;
+ return this;
+ }
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 03b5bd96162..b20d8e859aa 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -180,7 +180,7 @@ final class S3ADataBlocks {
* @param statistics stats to work with
* @return a new block.
*/
- abstract DataBlock create(long index, int limit,
+ abstract DataBlock create(long index, long limit,
BlockOutputStreamStatistics statistics)
throws IOException;
@@ -258,7 +258,7 @@ final class S3ADataBlocks {
* Return the current data size.
* @return the size of the data
*/
- abstract int dataSize();
+ abstract long dataSize();
/**
* Predicate to verify that the block has the capacity to write
@@ -280,7 +280,7 @@ final class S3ADataBlocks {
* The remaining capacity in the block before it is full.
* @return the number of bytes remaining.
*/
- abstract int remainingCapacity();
+ abstract long remainingCapacity();
/**
* Write a series of bytes from the buffer, from the offset.
@@ -391,9 +391,11 @@ final class S3ADataBlocks {
}
@Override
- DataBlock create(long index, int limit,
+ DataBlock create(long index, long limit,
BlockOutputStreamStatistics statistics)
throws IOException {
+ Preconditions.checkArgument(limit > 0,
+ "Invalid block size: %d", limit);
return new ByteArrayBlock(0, limit, statistics);
}
@@ -436,11 +438,11 @@ final class S3ADataBlocks {
private Integer dataSize;
ByteArrayBlock(long index,
- int limit,
+ long limit,
BlockOutputStreamStatistics statistics) {
super(index, statistics);
- this.limit = limit;
- buffer = new S3AByteArrayOutputStream(limit);
+ this.limit = (limit > Integer.MAX_VALUE) ? Integer.MAX_VALUE : (int) limit;
+ buffer = new S3AByteArrayOutputStream(this.limit);
blockAllocated();
}
@@ -449,7 +451,7 @@ final class S3ADataBlocks {
* @return the amount of data available to upload.
*/
@Override
- int dataSize() {
+ long dataSize() {
return dataSize != null ? dataSize : buffer.size();
}
@@ -468,14 +470,14 @@ final class S3ADataBlocks {
}
@Override
- int remainingCapacity() {
+ long remainingCapacity() {
return limit - dataSize();
}
@Override
int write(byte[] b, int offset, int len) throws IOException {
super.write(b, offset, len);
- int written = Math.min(remainingCapacity(), len);
+ int written = (int) Math.min(remainingCapacity(), len);
buffer.write(b, offset, written);
return written;
}
@@ -514,9 +516,11 @@ final class S3ADataBlocks {
}
@Override
- ByteBufferBlock create(long index, int limit,
+ ByteBufferBlock create(long index, long limit,
BlockOutputStreamStatistics statistics)
throws IOException {
+ Preconditions.checkArgument(limit > 0,
+ "Invalid block size: %d", limit);
return new ByteBufferBlock(index, limit, statistics);
}
@@ -564,11 +568,12 @@ final class S3ADataBlocks {
* @param statistics statistics to update
*/
ByteBufferBlock(long index,
- int bufferSize,
+ long bufferSize,
BlockOutputStreamStatistics statistics) {
super(index, statistics);
- this.bufferSize = bufferSize;
- blockBuffer = requestBuffer(bufferSize);
+ this.bufferSize = bufferSize > Integer.MAX_VALUE ?
+ Integer.MAX_VALUE : (int) bufferSize;
+ blockBuffer = requestBuffer(this.bufferSize);
blockAllocated();
}
@@ -577,7 +582,7 @@ final class S3ADataBlocks {
* @return the amount of data available to upload.
*/
@Override
- int dataSize() {
+ long dataSize() {
return dataSize != null ? dataSize : bufferCapacityUsed();
}
@@ -598,7 +603,7 @@ final class S3ADataBlocks {
}
@Override
- public int remainingCapacity() {
+ public long remainingCapacity() {
return blockBuffer != null ? blockBuffer.remaining() : 0;
}
@@ -609,7 +614,7 @@ final class S3ADataBlocks {
@Override
int write(byte[] b, int offset, int len) throws IOException {
super.write(b, offset, len);
- int written = Math.min(remainingCapacity(), len);
+ int written = (int) Math.min(remainingCapacity(), len);
blockBuffer.put(b, offset, written);
return written;
}
@@ -802,16 +807,18 @@ final class S3ADataBlocks {
* Create a temp file and a {@link DiskBlock} instance to manage it.
*
* @param index block index
- * @param limit limit of the block.
+ * @param limit limit of the block. -1 means "no limit"
* @param statistics statistics to update
* @return the new block
* @throws IOException IO problems
*/
@Override
DataBlock create(long index,
- int limit,
+ long limit,
BlockOutputStreamStatistics statistics)
throws IOException {
+ Preconditions.checkArgument(limit != 0,
+ "Invalid block size: %d", limit);
File destFile = getOwner()
.createTmpFileForWrite(String.format("s3ablock-%04d-", index),
limit, getOwner().getConf());
@@ -825,14 +832,14 @@ final class S3ADataBlocks {
*/
static class DiskBlock extends DataBlock {
- private int bytesWritten;
+ private long bytesWritten;
private final File bufferFile;
- private final int limit;
+ private final long limit;
private BufferedOutputStream out;
private final AtomicBoolean closed = new AtomicBoolean(false);
DiskBlock(File bufferFile,
- int limit,
+ long limit,
long index,
BlockOutputStreamStatistics statistics)
throws FileNotFoundException {
@@ -844,24 +851,39 @@ final class S3ADataBlocks {
}
@Override
- int dataSize() {
+ long dataSize() {
return bytesWritten;
}
+ /**
+ * Does this block have unlimited space?
+ * @return true if a block with no size limit was created.
+ */
+ private boolean unlimited() {
+ return limit < 0;
+ }
+
@Override
boolean hasCapacity(long bytes) {
- return dataSize() + bytes <= limit;
+ return unlimited() || dataSize() + bytes <= limit;
}
+ /**
+ * {@inheritDoc}.
+ * If there is no limit to capacity, return MAX_VALUE.
+ * @return capacity in the block.
+ */
@Override
- int remainingCapacity() {
- return limit - bytesWritten;
+ long remainingCapacity() {
+ return unlimited()
+ ? Integer.MAX_VALUE
+ : limit - bytesWritten;
}
@Override
int write(byte[] b, int offset, int len) throws IOException {
super.write(b, offset, len);
- int written = Math.min(remainingCapacity(), len);
+ int written = (int) Math.min(remainingCapacity(), len);
out.write(b, offset, written);
bytesWritten += written;
return written;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index cb17b80fb6a..30b2813caf7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -413,12 +413,22 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
*/
private ArnResource accessPoint;
+ /**
+ * Does this S3A FS instance have multipart upload enabled?
+ */
+ private boolean isMultipartUploadEnabled = DEFAULT_MULTIPART_UPLOAD_ENABLED;
+
/**
* A cache of files that should be deleted when the FileSystem is closed
* or the JVM is exited.
*/
private final Set
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ClientErrorRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionResetRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ConnectionTimeoutRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ReadTimeoutRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.RetryReasonCategory;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.ServerErrorRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownHostRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownIOExceptionRetryReason;
+import org.apache.hadoop.fs.azurebfs.services.retryReasonCategories.UnknownSocketExceptionRetryReason;
+
+
+/**
+ * This utility class exposes methods to convert a server response-error to a
+ * category of error.
+ */
+final class RetryReason {
+
+ /**
+ * Linked-list of the implementations of RetryReasonCategory. The objects in the
+ * list are arranged by the rank of their significance.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+public final class RetryReasonConstants {
+
+ private RetryReasonConstants() {
+
+ }
+ public static final String CONNECTION_TIMEOUT_JDK_MESSAGE = "connect timed out";
+ public static final String READ_TIMEOUT_JDK_MESSAGE = "Read timed out";
+ public static final String CONNECTION_RESET_MESSAGE = "Connection reset";
+ public static final String OPERATION_BREACH_MESSAGE = "Operations per second is over the account limit.";
+ public static final String CONNECTION_RESET_ABBREVIATION = "CR";
+ public static final String CONNECTION_TIMEOUT_ABBREVIATION = "CT";
+ public static final String READ_TIMEOUT_ABBREVIATION = "RT";
+ public static final String INGRESS_LIMIT_BREACH_ABBREVIATION = "ING";
+ public static final String EGRESS_LIMIT_BREACH_ABBREVIATION = "EGR";
+ public static final String OPERATION_LIMIT_BREACH_ABBREVIATION = "OPR";
+ public static final String UNKNOWN_HOST_EXCEPTION_ABBREVIATION = "UH";
+ public static final String IO_EXCEPTION_ABBREVIATION = "IOE";
+ public static final String SOCKET_EXCEPTION_ABBREVIATION = "SE";
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java
new file mode 100644
index 00000000000..cf1c47e3eb0
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ClientErrorRetryReason.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
+
+/**
+ * Category that can capture server-response errors for 4XX status-code.
+ */
+public class ClientErrorRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 4) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return statusCode + "";
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java
new file mode 100644
index 00000000000..702f8875646
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionResetRetryReason.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for connection-reset exception.
+ */
+public class ConnectionResetRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, CONNECTION_RESET_MESSAGE);
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return CONNECTION_RESET_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java
new file mode 100644
index 00000000000..28f35dcc805
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ConnectionTimeoutRetryReason.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for connection-timeout.
+ */
+public class ConnectionTimeoutRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return CONNECTION_TIMEOUT_ABBREVIATION;
+ }
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, CONNECTION_TIMEOUT_JDK_MESSAGE);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java
new file mode 100644
index 00000000000..4663d9a52bb
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ReadTimeoutRetryReason.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
+
+/**
+ * Category that can capture server-response errors for read-timeout.
+ */
+public class ReadTimeoutRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ return checkExceptionMessage(ex, READ_TIMEOUT_JDK_MESSAGE);
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return READ_TIMEOUT_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java
new file mode 100644
index 00000000000..893451b496f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/RetryReasonCategory.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.util.Locale;
+
+/**
+ * Provides methods to define if given exception can be categorised to certain category.
+ * Each category has a different implementation of the abstract class.
+ */
+public abstract class RetryReasonCategory {
+
+ /**
+ * Returns if given server response error can be categorised by the implementation.
+ *
+ * @param ex exception captured in the server response.
+ * @param statusCode statusCode on the server response
+ * @param serverErrorMessage serverErrorMessage on the server response.
+ *
+ * @return
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_STATUS_CATEGORY_QUOTIENT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for 5XX status-code.
+ */
+public class ServerErrorRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == null || statusCode / HTTP_STATUS_CATEGORY_QUOTIENT != 5) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ if (statusCode == HTTP_UNAVAILABLE && serverErrorMessage != null) {
+ String splitedServerErrorMessage = serverErrorMessage.split(System.lineSeparator(),
+ 2)[0];
+ if (INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return INGRESS_LIMIT_BREACH_ABBREVIATION;
+ }
+ if (EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage().equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return EGRESS_LIMIT_BREACH_ABBREVIATION;
+ }
+ if (OPERATION_BREACH_MESSAGE.equalsIgnoreCase(
+ splitedServerErrorMessage)) {
+ return OPERATION_LIMIT_BREACH_ABBREVIATION;
+ }
+ return HTTP_UNAVAILABLE + "";
+ }
+ return statusCode + "";
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java
new file mode 100644
index 00000000000..c329348d81f
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownHostRetryReason.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.net.UnknownHostException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for {@link UnknownHostException}.
+ */
+public class UnknownHostRetryReason extends RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof UnknownHostException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java
new file mode 100644
index 00000000000..8a69ebb928d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownIOExceptionRetryReason.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
+
+
+/**
+ * Category that can capture server-response errors for {@link IOException}.
+ */
+public class UnknownIOExceptionRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof IOException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return IO_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java
new file mode 100644
index 00000000000..18e9f115fea
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/UnknownSocketExceptionRetryReason.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import java.net.SocketException;
+
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
+
+/**
+ * Category that can capture server-response errors for {@link SocketException}.
+ */
+public class UnknownSocketExceptionRetryReason extends
+ RetryReasonCategory {
+
+ @Override
+ Boolean canCapture(final Exception ex,
+ final Integer statusCode,
+ final String serverErrorMessage) {
+ if (ex instanceof SocketException) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ String getAbbreviation(final Integer statusCode,
+ final String serverErrorMessage) {
+ return SOCKET_EXCEPTION_ABBREVIATION;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java
new file mode 100644
index 00000000000..7d8078620af
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A retryReasonCategory defines methods applicable on server-response errors.
+ */
+@Private
+@Evolving
+package org.apache.hadoop.fs.azurebfs.services.retryReasonCategories;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
index 5a115451df1..97864e61e0b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java
@@ -63,6 +63,16 @@ public class TracingContext {
//final concatenated ID list set into x-ms-client-request-id header
private String header = EMPTY_STRING;
+ /**
+ * If {@link #primaryRequestId} is null, this field shall be set equal
+ * to the last part of the {@link #clientRequestId}'s UUID
+ * in {@link #constructHeader(AbfsHttpOperation, String)} only on the
+ * first API call for an operation. Subsequent retries for that operation
+ * will not change this field. In case {@link #primaryRequestId} is non-null,
+ * this field shall not be set.
+ */
+ private String primaryRequestIdForRetry;
+
private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class);
public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72;
public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*";
@@ -139,6 +149,10 @@ public class TracingContext {
this.opType = operation;
}
+ public int getRetryCount() {
+ return retryCount;
+ }
+
public void setRetryCount(int retryCount) {
this.retryCount = retryCount;
}
@@ -152,15 +166,18 @@ public class TracingContext {
* X_MS_CLIENT_REQUEST_ID header of the http operation
* @param httpOperation AbfsHttpOperation instance to set header into
* connection
+ * @param previousFailure Failure seen before this API trigger on same operation
+ * from AbfsClient.
*/
- public void constructHeader(AbfsHttpOperation httpOperation) {
+ public void constructHeader(AbfsHttpOperation httpOperation, String previousFailure) {
clientRequestId = UUID.randomUUID().toString();
switch (format) {
case ALL_ID_FORMAT: // Optional IDs (e.g. streamId) may be empty
header =
clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":"
- + primaryRequestId + ":" + streamID + ":" + opType + ":"
- + retryCount;
+ + getPrimaryRequestIdForHeader(retryCount > 0) + ":" + streamID
+ + ":" + opType + ":" + retryCount;
+ header = addFailureReasons(header, previousFailure);
break;
case TWO_ID_FORMAT:
header = clientCorrelationID + ":" + clientRequestId;
@@ -172,6 +189,39 @@ public class TracingContext {
listener.callTracingHeaderValidator(header, format);
}
httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header);
+ /*
+ * In case the primaryRequestId is an empty-string and if it is the first try to
+ * API call (previousFailure shall be null), maintain the last part of clientRequestId's
+ * UUID in primaryRequestIdForRetry. This field shall be used as primaryRequestId part
+ * of the x-ms-client-request-id header in case of retry of the same API-request.
+ */
+ if (primaryRequestId.isEmpty() && previousFailure == null) {
+ String[] clientRequestIdParts = clientRequestId.split("-");
+ primaryRequestIdForRetry = clientRequestIdParts[
+ clientRequestIdParts.length - 1];
+ }
+ }
+
+ /**
+ * Provide value to be used as primaryRequestId part of x-ms-client-request-id header.
+ * @param isRetry define if it's for a retry case.
+ * @return {@link #primaryRequestIdForRetry}:If the {@link #primaryRequestId}
+ * is an empty-string, and it's a retry iteration.
+ * {@link #primaryRequestId} for other cases.
+ */
+ private String getPrimaryRequestIdForHeader(final Boolean isRetry) {
+ if (!primaryRequestId.isEmpty() || !isRetry) {
+ return primaryRequestId;
+ }
+ return primaryRequestIdForRetry;
+ }
+
+ private String addFailureReasons(final String header,
+ final String previousFailure) {
+ if (previousFailure == null) {
+ return header;
+ }
+ return String.format("%s_%s", header, previousFailure);
}
/**
diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
index 31498df1790..aff1e32b83f 100644
--- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
+++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md
@@ -767,6 +767,17 @@ Hflush() being the only documented API that can provide persistent data
transfer, Flush() also attempting to persist buffered data will lead to
performance issues.
+### Hundred Continue Options
+
+`fs.azure.account.expect.header.enabled`: This configuration parameter is used
+to specify whether you wish to send a expect 100 continue header with each
+append request or not. It is configured to true by default. This flag configures
+the client to check with the Azure store before uploading a block of data from
+an output stream. This allows the client to throttle back gracefully -before
+actually attempting to upload the block. In experiments this provides
+significant throughput improvements under heavy load. For more information :
+- https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Expect
+
### Account level throttling Options
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
index c5bf85a4f81..74655fd5736 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -254,7 +254,7 @@ public abstract class AbstractAbfsIntegrationTest extends
}
public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
- return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
+ return ITestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
}
public void loadConfiguredFileSystem() throws Exception {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index 2f23ac5c5c7..d9a3cea089f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConcurrentWriteOperati
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator;
@@ -362,7 +362,7 @@ public class ITestAzureBlobFileSystemCreate extends
// Get mock AbfsClient with current config
AbfsClient
mockClient
- = TestAbfsClient.getMockAbfsClient(
+ = ITestAbfsClient.getMockAbfsClient(
fs.getAbfsStore().getClient(),
fs.getAbfsStore().getAbfsConfiguration());
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
index b164689ef80..5735423aaf9 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
@@ -70,6 +70,8 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
private static final Logger LOG =
LoggerFactory.getLogger(ITestAzureBlobFileSystemDelegationSAS.class);
+ private boolean isHNSEnabled;
+
public ITestAzureBlobFileSystemDelegationSAS() throws Exception {
// These tests rely on specific settings in azure-auth-keys.xml:
String sasProvider = getRawConfiguration().get(FS_AZURE_SAS_TOKEN_PROVIDER_TYPE);
@@ -85,7 +87,7 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
@Override
public void setup() throws Exception {
- boolean isHNSEnabled = this.getConfiguration().getBoolean(
+ isHNSEnabled = this.getConfiguration().getBoolean(
TestConfigurationKeys.FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
Assume.assumeTrue(isHNSEnabled);
createFilesystemForSASTests();
@@ -401,7 +403,7 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
fs.create(new Path(src)).close();
AbfsRestOperation abfsHttpRestOperation = fs.getAbfsClient()
.renamePath(src, "/testABC" + "/abc.txt", null,
- getTestTracingContext(fs, false), null, false)
+ getTestTracingContext(fs, false), null, false, isHNSEnabled)
.getOp();
AbfsHttpOperation result = abfsHttpRestOperation.getResult();
String url = result.getMaskedUrl();
@@ -419,7 +421,7 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
intercept(IOException.class, "sig=XXXX",
() -> getFileSystem().getAbfsClient()
.renamePath("testABC/test.xt", "testABC/abc.txt", null,
- getTestTracingContext(getFileSystem(), false), null, false));
+ getTestTracingContext(getFileSystem(), false), null, false, isHNSEnabled));
}
@Test
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
index db181fb5dd6..1f0ff667522 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationExcep
import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
-import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
+import org.apache.hadoop.fs.azurebfs.services.ITestAbfsClient;
import org.apache.hadoop.fs.azurebfs.services.TestAbfsPerfTracker;
import org.apache.hadoop.fs.azurebfs.utils.TestMockHelpers;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
@@ -176,7 +176,7 @@ public class ITestAzureBlobFileSystemDelete extends
final AzureBlobFileSystem fs = getFileSystem();
AbfsClient abfsClient = fs.getAbfsStore().getClient();
- AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext(
+ AbfsClient testClient = ITestAbfsClient.createTestClientFromCurrentContext(
abfsClient,
abfsConfig);
@@ -223,7 +223,7 @@ public class ITestAzureBlobFileSystemDelete extends
public void testDeleteIdempotencyTriggerHttp404() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
- AbfsClient client = TestAbfsClient.createTestClientFromCurrentContext(
+ AbfsClient client = ITestAbfsClient.createTestClientFromCurrentContext(
fs.getAbfsStore().getClient(),
this.getConfiguration());
@@ -242,7 +242,7 @@ public class ITestAzureBlobFileSystemDelete extends
getTestTracingContext(fs, true)));
// mock idempotency check to mimic retried case
- AbfsClient mockClient = TestAbfsClient.getMockAbfsClient(
+ AbfsClient mockClient = ITestAbfsClient.getMockAbfsClient(
fs.getAbfsStore().getClient(),
this.getConfiguration());
AzureBlobFileSystemStore mockStore = mock(AzureBlobFileSystemStore.class);
@@ -257,10 +257,10 @@ public class ITestAzureBlobFileSystemDelete extends
// Case 2: Mimic retried case
// Idempotency check on Delete always returns success
- AbfsRestOperation idempotencyRetOp = TestAbfsClient.getRestOp(
+ AbfsRestOperation idempotencyRetOp = ITestAbfsClient.getRestOp(
DeletePath, mockClient, HTTP_METHOD_DELETE,
- TestAbfsClient.getTestUrl(mockClient, "/NonExistingPath"),
- TestAbfsClient.getTestRequestHeaders(mockClient));
+ ITestAbfsClient.getTestUrl(mockClient, "/NonExistingPath"),
+ ITestAbfsClient.getTestRequestHeaders(mockClient));
idempotencyRetOp.hardSetResult(HTTP_OK);
doReturn(idempotencyRetOp).when(mockClient).deleteIdempotencyCheckOp(any());
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
index 58c3be66883..76b8a77fffc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestCustomerProvidedKey.java
@@ -99,10 +99,14 @@ public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest {
private static final int FILE_SIZE = 10 * ONE_MB;
private static final int FILE_SIZE_FOR_COPY_BETWEEN_ACCOUNTS = 24 * ONE_MB;
+ private boolean isNamespaceEnabled;
+
public ITestCustomerProvidedKey() throws Exception {
boolean isCPKTestsEnabled = getConfiguration()
.getBoolean(FS_AZURE_TEST_CPK_ENABLED, false);
Assume.assumeTrue(isCPKTestsEnabled);
+ isNamespaceEnabled = getConfiguration()
+ .getBoolean(FS_AZURE_TEST_NAMESPACE_ENABLED_ACCOUNT, false);
}
@Test
@@ -203,7 +207,7 @@ public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest {
// Trying to append with correct CPK headers
AppendRequestParameters appendRequestParameters =
new AppendRequestParameters(
- 0, 0, 5, Mode.APPEND_MODE, false, null);
+ 0, 0, 5, Mode.APPEND_MODE, false, null, true);
byte[] buffer = getRandomBytesArray(5);
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
@@ -248,7 +252,7 @@ public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest {
// Trying to append without CPK headers
AppendRequestParameters appendRequestParameters =
new AppendRequestParameters(
- 0, 0, 5, Mode.APPEND_MODE, false, null);
+ 0, 0, 5, Mode.APPEND_MODE, false, null, true);
byte[] buffer = getRandomBytesArray(5);
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
@@ -526,7 +530,7 @@ public class ITestCustomerProvidedKey extends AbstractAbfsIntegrationTest {
AbfsClient abfsClient = fs.getAbfsClient();
AbfsRestOperation abfsRestOperation = abfsClient
.renamePath(testFileName, newName, null,
- getTestTracingContext(fs, false), null, false)
+ getTestTracingContext(fs, false), null, false, isNamespaceEnabled)
.getOp();
assertCPKHeaders(abfsRestOperation, false);
assertNoCPKResponseHeadersPresent(abfsRestOperation);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
index b91a3e2208b..23e65ed2dd2 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java
@@ -31,12 +31,14 @@ import org.junit.Assume;
import org.junit.AssumptionViolatedException;
import org.junit.Ignore;
import org.junit.Test;
+import org.mockito.Mockito;
import org.apache.hadoop.fs.CommonPathCapabilities;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.enums.Trilean;
+import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation;
import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
@@ -198,4 +200,74 @@ public class TestTracingContext extends AbstractAbfsIntegrationTest {
fs.getAbfsStore().setNamespaceEnabled(Trilean.TRUE);
fs.access(new Path("/"), FsAction.READ);
}
+
+ @Test
+ public void testRetryPrimaryRequestIdWhenInitiallySuppliedEmpty() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String fileSystemId = fs.getFileSystemId();
+ final String clientCorrelationId = fs.getClientCorrelationId();
+ final TracingHeaderFormat tracingHeaderFormat = TracingHeaderFormat.ALL_ID_FORMAT;
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 0));
+ AbfsHttpOperation abfsHttpOperation = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doNothing().when(abfsHttpOperation).setRequestProperty(Mockito.anyString(), Mockito.anyString());
+ tracingContext.constructHeader(abfsHttpOperation, null);
+ String header = tracingContext.getHeader();
+ String clientRequestIdUsed = header.split(":")[1];
+ String[] clientRequestIdUsedParts = clientRequestIdUsed.split("-");
+ String assertionPrimaryId = clientRequestIdUsedParts[clientRequestIdUsedParts.length - 1];
+
+ tracingContext.setRetryCount(1);
+ tracingContext.setListener(new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 1));
+
+ tracingContext.constructHeader(abfsHttpOperation, "RT");
+ header = tracingContext.getHeader();
+ String primaryRequestId = header.split(":")[3];
+
+ Assertions.assertThat(primaryRequestId)
+ .describedAs("PrimaryRequestId in a retried request's "
+ + "tracingContext should be equal to last part of original "
+ + "request's clientRequestId UUID")
+ .isEqualTo(assertionPrimaryId);
+ }
+
+ @Test
+ public void testRetryPrimaryRequestIdWhenInitiallySuppliedNonEmpty() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String fileSystemId = fs.getFileSystemId();
+ final String clientCorrelationId = fs.getClientCorrelationId();
+ final TracingHeaderFormat tracingHeaderFormat = TracingHeaderFormat.ALL_ID_FORMAT;
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat, new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 0));
+ tracingContext.setPrimaryRequestID();
+ AbfsHttpOperation abfsHttpOperation = Mockito.mock(AbfsHttpOperation.class);
+ Mockito.doNothing().when(abfsHttpOperation).setRequestProperty(Mockito.anyString(), Mockito.anyString());
+ tracingContext.constructHeader(abfsHttpOperation, null);
+ String header = tracingContext.getHeader();
+ String assertionPrimaryId = header.split(":")[3];
+
+ tracingContext.setRetryCount(1);
+ tracingContext.setListener(new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 1));
+
+ tracingContext.constructHeader(abfsHttpOperation, "RT");
+ header = tracingContext.getHeader();
+ String primaryRequestId = header.split(":")[3];
+
+ Assertions.assertThat(primaryRequestId)
+ .describedAs("PrimaryRequestId in a retried request's tracingContext "
+ + "should be equal to PrimaryRequestId in the original request.")
+ .isEqualTo(assertionPrimaryId);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
similarity index 64%
rename from hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
index 08eb3adc926..c031e5daa6c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClient.java
@@ -20,20 +20,43 @@ package org.apache.hadoop.fs.azurebfs.services;
import java.io.IOException;
import java.lang.reflect.Field;
+import java.net.HttpURLConnection;
+import java.net.ProtocolException;
import java.net.URL;
import java.util.List;
+import java.util.Random;
import java.util.regex.Pattern;
+import org.assertj.core.api.Assertions;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+import org.apache.hadoop.fs.azurebfs.TestAbfsConfigurationFieldsValidation;
+import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.contracts.services.AppendRequestParameters;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat;
import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory;
-import static org.assertj.core.api.Assertions.assertThat;
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APPEND_ACTION;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PATCH;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_PUT;
+import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HUNDRED_CONTINUE;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.EXPECT;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_HTTP_METHOD_OVERRIDE;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_ACTION;
+import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_POSITION;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ABFS_ACCOUNT_NAME;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -59,14 +82,19 @@ import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST
* Test useragent of abfs client.
*
*/
-public final class TestAbfsClient {
+public final class ITestAbfsClient extends AbstractAbfsIntegrationTest {
private static final String ACCOUNT_NAME = "bogusAccountName.dfs.core.windows.net";
private static final String FS_AZURE_USER_AGENT_PREFIX = "Partner Service";
+ private static final String TEST_PATH = "/testfile";
+ public static final int REDUCED_RETRY_COUNT = 2;
+ public static final int REDUCED_BACKOFF_INTERVAL = 100;
+ public static final int BUFFER_LENGTH = 5;
+ public static final int BUFFER_OFFSET = 0;
private final Pattern userAgentStringPattern;
- public TestAbfsClient(){
+ public ITestAbfsClient() throws Exception {
StringBuilder regEx = new StringBuilder();
regEx.append("^");
regEx.append(APN_VERSION);
@@ -124,7 +152,7 @@ public final class TestAbfsClient {
}
private void verifybBasicInfo(String userAgentStr) {
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string [" + userAgentStr
+ "] should be of the pattern: " + this.userAgentStringPattern.pattern())
.matches(this.userAgentStringPattern)
@@ -153,7 +181,7 @@ public final class TestAbfsClient {
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain " + FS_AZURE_USER_AGENT_PREFIX)
.contains(FS_AZURE_USER_AGENT_PREFIX);
@@ -163,7 +191,7 @@ public final class TestAbfsClient {
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain " + FS_AZURE_USER_AGENT_PREFIX)
.doesNotContain(FS_AZURE_USER_AGENT_PREFIX);
}
@@ -179,14 +207,14 @@ public final class TestAbfsClient {
String userAgentStr = getUserAgentString(abfsConfiguration, true);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain sslProvider")
.contains(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName());
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain sslProvider")
.doesNotContain(DelegatingSSLSocketFactory.getDefaultFactory().getProviderName());
}
@@ -202,7 +230,7 @@ public final class TestAbfsClient {
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain cluster name")
.contains(clusterName);
@@ -212,7 +240,7 @@ public final class TestAbfsClient {
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain cluster name")
.doesNotContain(clusterName)
.describedAs("User-Agent string should contain UNKNOWN as cluster name config is absent")
@@ -230,7 +258,7 @@ public final class TestAbfsClient {
String userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should contain cluster type")
.contains(clusterType);
@@ -240,7 +268,7 @@ public final class TestAbfsClient {
userAgentStr = getUserAgentString(abfsConfiguration, false);
verifybBasicInfo(userAgentStr);
- assertThat(userAgentStr)
+ Assertions.assertThat(userAgentStr)
.describedAs("User-Agent string should not contain cluster type")
.doesNotContain(clusterType)
.describedAs("User-Agent string should contain UNKNOWN as cluster type config is absent")
@@ -311,24 +339,23 @@ public final class TestAbfsClient {
AbfsThrottlingInterceptFactory.getInstance(
abfsConfig.getAccountName().substring(0,
abfsConfig.getAccountName().indexOf(DOT)), abfsConfig));
-
// override baseurl
- client = TestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
+ client = ITestAbfsClient.setAbfsClientField(client, "abfsConfiguration",
abfsConfig);
// override baseurl
- client = TestAbfsClient.setAbfsClientField(client, "baseUrl",
+ client = ITestAbfsClient.setAbfsClientField(client, "baseUrl",
baseAbfsClientInstance.getBaseUrl());
// override auth provider
if (currentAuthType == AuthType.SharedKey) {
- client = TestAbfsClient.setAbfsClientField(client, "sharedKeyCredentials",
+ client = ITestAbfsClient.setAbfsClientField(client, "sharedKeyCredentials",
new SharedKeyCredentials(
abfsConfig.getAccountName().substring(0,
abfsConfig.getAccountName().indexOf(DOT)),
abfsConfig.getStorageAccountKey()));
} else {
- client = TestAbfsClient.setAbfsClientField(client, "tokenProvider",
+ client = ITestAbfsClient.setAbfsClientField(client, "tokenProvider",
abfsConfig.getTokenProvider());
}
@@ -336,7 +363,7 @@ public final class TestAbfsClient {
String userAgent = "APN/1.0 Azure Blob FS/3.4.0-SNAPSHOT (PrivateBuild "
+ "JavaJRE 1.8.0_252; Linux 5.3.0-59-generic/amd64; openssl-1.0; "
+ "UNKNOWN/UNKNOWN) MSFT";
- client = TestAbfsClient.setAbfsClientField(client, "userAgent", userAgent);
+ client = ITestAbfsClient.setAbfsClientField(client, "userAgent", userAgent);
return client;
}
@@ -404,4 +431,156 @@ public final class TestAbfsClient {
public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
return client.getTokenProvider();
}
+
+ /**
+ * Test helper method to get random bytes array.
+ * @param length The length of byte buffer.
+ * @return byte buffer.
+ */
+ private byte[] getRandomBytesArray(int length) {
+ final byte[] b = new byte[length];
+ new Random().nextBytes(b);
+ return b;
+ }
+
+ /**
+ * Test to verify that client retries append request without
+ * expect header enabled if append with expect header enabled fails
+ * with 4xx kind of error.
+ * @throws Exception
+ */
+ @Test
+ public void testExpectHundredContinue() throws Exception {
+ // Get the filesystem.
+ final AzureBlobFileSystem fs = getFileSystem();
+
+ final Configuration configuration = new Configuration();
+ configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+ AbfsClient abfsClient = fs.getAbfsStore().getClient();
+
+ AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
+ configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME));
+
+ // Update the configuration with reduced retry count and reduced backoff interval.
+ AbfsConfiguration abfsConfig
+ = TestAbfsConfigurationFieldsValidation.updateRetryConfigs(
+ abfsConfiguration,
+ REDUCED_RETRY_COUNT, REDUCED_BACKOFF_INTERVAL);
+
+ // Gets the client.
+ AbfsClient testClient = Mockito.spy(
+ ITestAbfsClient.createTestClientFromCurrentContext(
+ abfsClient,
+ abfsConfig));
+
+ // Create the append request params with expect header enabled initially.
+ AppendRequestParameters appendRequestParameters
+ = new AppendRequestParameters(
+ BUFFER_OFFSET, BUFFER_OFFSET, BUFFER_LENGTH,
+ AppendRequestParameters.Mode.APPEND_MODE, false, null, true);
+
+ byte[] buffer = getRandomBytesArray(BUFFER_LENGTH);
+
+ // Create a test container to upload the data.
+ Path testPath = path(TEST_PATH);
+ fs.create(testPath);
+ String finalTestPath = testPath.toString()
+ .substring(testPath.toString().lastIndexOf("/"));
+
+ // Creates a list of request headers.
+ final List
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.HttpURLConnection;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.stubbing.Stubber;
+
+import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
+
+import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
+import static java.net.HttpURLConnection.HTTP_OK;
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.services.AuthType.OAuth;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.nullable;
+
+public class TestAbfsRestOperationMockFailures {
+
+ @Test
+ public void testClientRequestIdForConnectTimeoutRetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
+ abbreviations[0] = CONNECTION_TIMEOUT_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForConnectAndReadTimeoutRetry()
+ throws Exception {
+ Exception[] exceptions = new Exception[2];
+ String[] abbreviations = new String[2];
+ exceptions[0] = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
+ abbreviations[0] = CONNECTION_TIMEOUT_ABBREVIATION;
+ exceptions[1] = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
+ abbreviations[1] = READ_TIMEOUT_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForReadTimeoutRetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
+ abbreviations[0] = READ_TIMEOUT_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForUnknownHostRetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new UnknownHostException();
+ abbreviations[0] = UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForConnectionResetRetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new SocketTimeoutException(CONNECTION_RESET_MESSAGE + " by peer");
+ abbreviations[0] = CONNECTION_RESET_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForUnknownSocketExRetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new SocketException("unknown");
+ abbreviations[0] = SOCKET_EXCEPTION_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdForIOERetry() throws Exception {
+ Exception[] exceptions = new Exception[1];
+ String[] abbreviations = new String[1];
+ exceptions[0] = new InterruptedIOException();
+ abbreviations[0] = IO_EXCEPTION_ABBREVIATION;
+ testClientRequestIdForTimeoutRetry(exceptions, abbreviations, 1);
+ }
+
+ @Test
+ public void testClientRequestIdFor400Retry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_BAD_REQUEST, "", "400");
+ }
+
+ @Test
+ public void testClientRequestIdFor500Retry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_INTERNAL_ERROR, "", "500");
+ }
+
+ @Test
+ public void testClientRequestIdFor503INGRetry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
+ INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage(),
+ INGRESS_LIMIT_BREACH_ABBREVIATION);
+ }
+
+ @Test
+ public void testClientRequestIdFor503egrRetry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
+ EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage(),
+ EGRESS_LIMIT_BREACH_ABBREVIATION);
+ }
+
+ @Test
+ public void testClientRequestIdFor503OPRRetry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE,
+ OPERATION_BREACH_MESSAGE, OPERATION_LIMIT_BREACH_ABBREVIATION);
+ }
+
+ @Test
+ public void testClientRequestIdFor503OtherRetry() throws Exception {
+ testClientRequestIdForStatusRetry(HTTP_UNAVAILABLE, "Other.", "503");
+ }
+
+ private void testClientRequestIdForStatusRetry(int status,
+ String serverErrorMessage,
+ String keyExpected) throws Exception {
+
+ AbfsClient abfsClient = Mockito.mock(AbfsClient.class);
+ ExponentialRetryPolicy retryPolicy = Mockito.mock(
+ ExponentialRetryPolicy.class);
+ addMockBehaviourToAbfsClient(abfsClient, retryPolicy);
+
+
+ AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation(
+ AbfsRestOperationType.ReadFile,
+ abfsClient,
+ "PUT",
+ null,
+ new ArrayList<>()
+ ));
+
+ AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class);
+ addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation);
+
+ Mockito.doNothing()
+ .doNothing()
+ .when(httpOperation)
+ .processResponse(nullable(byte[].class), nullable(int.class),
+ nullable(int.class));
+
+ int[] statusCount = new int[1];
+ statusCount[0] = 0;
+ Mockito.doAnswer(answer -> {
+ if (statusCount[0] <= 5) {
+ statusCount[0]++;
+ return status;
+ }
+ return HTTP_OK;
+ }).when(httpOperation).getStatusCode();
+
+ Mockito.doReturn(serverErrorMessage)
+ .when(httpOperation)
+ .getStorageErrorMessage();
+
+ TracingContext tracingContext = Mockito.mock(TracingContext.class);
+ Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class));
+
+ int[] count = new int[1];
+ count[0] = 0;
+ Mockito.doAnswer(invocationOnMock -> {
+ if (count[0] == 1) {
+ Assertions.assertThat((String) invocationOnMock.getArgument(1))
+ .isEqualTo(keyExpected);
+ }
+ count[0]++;
+ return null;
+ }).when(tracingContext).constructHeader(any(), any());
+
+ abfsRestOperation.execute(tracingContext);
+ Assertions.assertThat(count[0]).isEqualTo(2);
+
+ }
+
+ private void testClientRequestIdForTimeoutRetry(Exception[] exceptions,
+ String[] abbreviationsExpected,
+ int len) throws Exception {
+ AbfsClient abfsClient = Mockito.mock(AbfsClient.class);
+ ExponentialRetryPolicy retryPolicy = Mockito.mock(
+ ExponentialRetryPolicy.class);
+ addMockBehaviourToAbfsClient(abfsClient, retryPolicy);
+
+
+ AbfsRestOperation abfsRestOperation = Mockito.spy(new AbfsRestOperation(
+ AbfsRestOperationType.ReadFile,
+ abfsClient,
+ "PUT",
+ null,
+ new ArrayList<>()
+ ));
+
+ AbfsHttpOperation httpOperation = Mockito.mock(AbfsHttpOperation.class);
+ addMockBehaviourToRestOpAndHttpOp(abfsRestOperation, httpOperation);
+
+ Stubber stubber = Mockito.doThrow(exceptions[0]);
+ for (int iteration = 1; iteration < len; iteration++) {
+ stubber.doThrow(exceptions[iteration]);
+ }
+ stubber
+ .doNothing()
+ .when(httpOperation)
+ .processResponse(nullable(byte[].class), nullable(int.class),
+ nullable(int.class));
+
+ Mockito.doReturn(HTTP_OK).when(httpOperation).getStatusCode();
+
+ TracingContext tracingContext = Mockito.mock(TracingContext.class);
+ Mockito.doNothing().when(tracingContext).setRetryCount(nullable(int.class));
+
+ int[] count = new int[1];
+ count[0] = 0;
+ Mockito.doAnswer(invocationOnMock -> {
+ if (count[0] > 0 && count[0] <= len) {
+ Assertions.assertThat((String) invocationOnMock.getArgument(1))
+ .isEqualTo(abbreviationsExpected[count[0] - 1]);
+ }
+ count[0]++;
+ return null;
+ }).when(tracingContext).constructHeader(any(), any());
+
+ abfsRestOperation.execute(tracingContext);
+ Assertions.assertThat(count[0]).isEqualTo(len + 1);
+ }
+
+ private void addMockBehaviourToRestOpAndHttpOp(final AbfsRestOperation abfsRestOperation,
+ final AbfsHttpOperation httpOperation) throws IOException {
+ HttpURLConnection httpURLConnection = Mockito.mock(HttpURLConnection.class);
+ Mockito.doNothing()
+ .when(httpURLConnection)
+ .setRequestProperty(nullable(String.class), nullable(String.class));
+ Mockito.doReturn(httpURLConnection).when(httpOperation).getConnection();
+ Mockito.doReturn("").when(abfsRestOperation).getClientLatency();
+ Mockito.doReturn(httpOperation).when(abfsRestOperation).createHttpOperation();
+ }
+
+ private void addMockBehaviourToAbfsClient(final AbfsClient abfsClient,
+ final ExponentialRetryPolicy retryPolicy) throws IOException {
+ Mockito.doReturn(OAuth).when(abfsClient).getAuthType();
+ Mockito.doReturn("").when(abfsClient).getAccessToken();
+ AbfsThrottlingIntercept intercept = Mockito.mock(
+ AbfsThrottlingIntercept.class);
+ Mockito.doReturn(intercept).when(abfsClient).getIntercept();
+ Mockito.doNothing()
+ .when(intercept)
+ .sendingRequest(any(), nullable(AbfsCounters.class));
+ Mockito.doNothing().when(intercept).updateMetrics(any(), any());
+
+ Mockito.doReturn(retryPolicy).when(abfsClient).getRetryPolicy();
+ Mockito.doReturn(true)
+ .when(retryPolicy)
+ .shouldRetry(nullable(Integer.class), nullable(Integer.class));
+ Mockito.doReturn(false).when(retryPolicy).shouldRetry(1, HTTP_OK);
+ Mockito.doReturn(false).when(retryPolicy).shouldRetry(2, HTTP_OK);
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
index a1fc4e138d6..12ab4e9ead6 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestExponentialRetryPolicy.java
@@ -103,7 +103,7 @@ public class TestExponentialRetryPolicy extends AbstractAbfsIntegrationTest {
AbfsConfiguration abfsConfiguration = new AbfsConfiguration(configuration,
"dummy.dfs.core.windows.net");
AbfsThrottlingIntercept intercept;
- AbfsClient abfsClient = TestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration);
+ AbfsClient abfsClient = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration);
intercept = abfsClient.getIntercept();
Assertions.assertThat(intercept)
.describedAs("AbfsNoOpThrottlingIntercept instance expected")
@@ -114,7 +114,7 @@ public class TestExponentialRetryPolicy extends AbstractAbfsIntegrationTest {
// On disabling throttling AbfsClientThrottlingIntercept object is returned
AbfsConfiguration abfsConfiguration1 = new AbfsConfiguration(configuration,
"dummy1.dfs.core.windows.net");
- AbfsClient abfsClient1 = TestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1);
+ AbfsClient abfsClient1 = ITestAbfsClient.createTestClientFromCurrentContext(fs.getAbfsStore().getClient(), abfsConfiguration1);
intercept = abfsClient1.getIntercept();
Assertions.assertThat(intercept)
.describedAs("AbfsClientThrottlingIntercept instance expected")
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestRetryReason.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestRetryReason.java
new file mode 100644
index 00000000000..76fcc6dc2c8
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestRetryReason.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.io.IOException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
+import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_JDK_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.EGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.INGRESS_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.IO_EXCEPTION_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_BREACH_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.OPERATION_LIMIT_BREACH_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.READ_TIMEOUT_JDK_MESSAGE;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.SOCKET_EXCEPTION_ABBREVIATION;
+import static org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.UNKNOWN_HOST_EXCEPTION_ABBREVIATION;
+
+public class TestRetryReason {
+
+ @Test
+ public void test4xxStatusRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_FORBIDDEN, null))
+ .describedAs("Abbreviation for 4xx should be equal to 4xx")
+ .isEqualTo(HTTP_FORBIDDEN + "");
+ }
+
+ @Test
+ public void testConnectionResetRetryReason() {
+ SocketException connReset = new SocketException(CONNECTION_RESET_MESSAGE.toUpperCase());
+ Assertions.assertThat(RetryReason.getAbbreviation(connReset, null, null)).isEqualTo(CONNECTION_RESET_ABBREVIATION);
+ }
+
+ @Test
+ public void testConnectionTimeoutRetryReason() {
+ SocketTimeoutException connectionTimeoutException = new SocketTimeoutException(CONNECTION_TIMEOUT_JDK_MESSAGE);
+ Assertions.assertThat(RetryReason.getAbbreviation(connectionTimeoutException, null, null)).isEqualTo(
+ CONNECTION_TIMEOUT_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testReadTimeoutRetryReason() {
+ SocketTimeoutException connectionTimeoutException = new SocketTimeoutException(READ_TIMEOUT_JDK_MESSAGE);
+ Assertions.assertThat(RetryReason.getAbbreviation(connectionTimeoutException, null, null)).isEqualTo(
+ READ_TIMEOUT_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testEgressLimitRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, EGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage())).isEqualTo(
+ EGRESS_LIMIT_BREACH_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testIngressLimitRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, INGRESS_OVER_ACCOUNT_LIMIT.getErrorMessage())).isEqualTo(
+ INGRESS_LIMIT_BREACH_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testOperationLimitRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, OPERATION_BREACH_MESSAGE)).isEqualTo(
+ OPERATION_LIMIT_BREACH_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void test503UnknownRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_UNAVAILABLE, null)).isEqualTo(
+ "503"
+ );
+ }
+
+ @Test
+ public void test500RetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(null, HTTP_INTERNAL_ERROR, null)).isEqualTo(
+ "500"
+ );
+ }
+
+ @Test
+ public void testUnknownHostRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(new UnknownHostException(), null, null)).isEqualTo(
+ UNKNOWN_HOST_EXCEPTION_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testUnknownIOExceptionRetryReason() {
+ Assertions.assertThat(RetryReason.getAbbreviation(new IOException(), null, null)).isEqualTo(
+ IO_EXCEPTION_ABBREVIATION
+ );
+ }
+
+ @Test
+ public void testUnknownSocketException() {
+ Assertions.assertThat(RetryReason.getAbbreviation(new SocketException(), null, null)).isEqualTo(
+ SOCKET_EXCEPTION_ABBREVIATION
+ );
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java
index e195f1c381a..7569c80d67c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java
@@ -130,6 +130,9 @@ public class TracingHeaderValidator implements Listener {
}
Assertions.assertThat(idList[5]).describedAs("Operation name incorrect")
.isEqualTo(operation.toString());
+ if (idList[6].contains("_")) {
+ idList[6] = idList[6].split("_")[0];
+ }
int retryCount = Integer.parseInt(idList[6]);
Assertions.assertThat(retryCount)
.describedAs("Retry was required due to issue on server side")
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
index 1cf2d97ec1f..dbc86fd0b47 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpSync.java
@@ -20,18 +20,19 @@ package org.apache.hadoop.tools;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonPathCapabilities;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.tools.CopyListing.InvalidInputException;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@@ -106,20 +107,7 @@ class DistCpSync {
final FileSystem snapshotDiffFs = isRdiff() ? tgtFs : srcFs;
final Path snapshotDiffDir = isRdiff() ? targetDir : sourceDir;
- // currently we require both the source and the target file system are
- // DistributedFileSystem or (S)WebHdfsFileSystem.
- if (!(srcFs instanceof DistributedFileSystem
- || srcFs instanceof WebHdfsFileSystem)) {
- throw new IllegalArgumentException("Unsupported source file system: "
- + srcFs.getScheme() + "://. " +
- "Supported file systems: hdfs://, webhdfs:// and swebhdfs://.");
- }
- if (!(tgtFs instanceof DistributedFileSystem
- || tgtFs instanceof WebHdfsFileSystem)) {
- throw new IllegalArgumentException("Unsupported target file system: "
- + tgtFs.getScheme() + "://. " +
- "Supported file systems: hdfs://, webhdfs:// and swebhdfs://.");
- }
+ checkFilesystemSupport(sourceDir,targetDir,srcFs, tgtFs);
// make sure targetFS has no change between from and the current states
if (!checkNoChange(tgtFs, targetDir)) {
@@ -165,6 +153,42 @@ class DistCpSync {
return true;
}
+ /**
+ * Check if the source and target filesystems support snapshots.
+ */
+ private void checkFilesystemSupport(Path sourceDir, Path targetDir,
+ FileSystem srcFs, FileSystem tgtFs) throws IOException {
+ if (!srcFs.hasPathCapability(sourceDir,
+ CommonPathCapabilities.FS_SNAPSHOTS)) {
+ throw new UnsupportedOperationException(
+ "The source file system " + srcFs.getScheme()
+ + " does not support snapshot.");
+ }
+ if (!tgtFs.hasPathCapability(targetDir,
+ CommonPathCapabilities.FS_SNAPSHOTS)) {
+ throw new UnsupportedOperationException(
+ "The target file system " + tgtFs.getScheme()
+ + " does not support snapshot.");
+ }
+ try {
+ getSnapshotDiffReportMethod(srcFs);
+ } catch (NoSuchMethodException e) {
+ throw new UnsupportedOperationException(
+ "The source file system " + srcFs.getScheme()
+ + " does not support getSnapshotDiffReport",
+ e);
+ }
+ try {
+ getSnapshotDiffReportMethod(tgtFs);
+ } catch (NoSuchMethodException e) {
+ throw new UnsupportedOperationException(
+ "The target file system " + tgtFs.getScheme()
+ + " does not support getSnapshotDiffReport",
+ e);
+ }
+
+ }
+
public boolean sync() throws IOException {
if (!preSyncCheck()) {
return false;
@@ -211,21 +235,10 @@ class DistCpSync {
context.getTargetPath() : context.getSourcePaths().get(0);
try {
- SnapshotDiffReport report = null;
- FileSystem fs = ssDir.getFileSystem(conf);
final String from = getSnapshotName(context.getFromSnapshot());
final String to = getSnapshotName(context.getToSnapshot());
- if (fs instanceof DistributedFileSystem) {
- DistributedFileSystem dfs = (DistributedFileSystem)fs;
- report = dfs.getSnapshotDiffReport(ssDir, from, to);
- } else if (fs instanceof WebHdfsFileSystem) {
- WebHdfsFileSystem webHdfs = (WebHdfsFileSystem)fs;
- report = webHdfs.getSnapshotDiffReport(ssDir, from, to);
- } else {
- throw new IllegalArgumentException("Unsupported file system: " +
- fs.getScheme() + "://. " +
- "Supported file systems: hdfs://, webhdfs:// and swebhdfs://.");
- }
+ SnapshotDiffReport report =
+ getSnapshotDiffReport(ssDir.getFileSystem(conf), ssDir, from, to);
this.diffMap = new EnumMap<>(SnapshotDiffReport.DiffType.class);
for (SnapshotDiffReport.DiffType type :
@@ -286,6 +299,36 @@ class DistCpSync {
return false;
}
+ /**
+ * Check if the filesystem implementation has a method named
+ * getSnapshotDiffReport.
+ */
+ private static Method getSnapshotDiffReportMethod(FileSystem fs)
+ throws NoSuchMethodException {
+ return fs.getClass().getMethod(
+ "getSnapshotDiffReport", Path.class, String.class, String.class);
+ }
+
+ /**
+ * Get the snapshotDiff b/w the fromSnapshot & toSnapshot for the given
+ * filesystem.
+ */
+ private static SnapshotDiffReport getSnapshotDiffReport(
+ final FileSystem fs,
+ final Path snapshotDir,
+ final String fromSnapshot,
+ final String toSnapshot) throws IOException {
+ try {
+ return (SnapshotDiffReport) getSnapshotDiffReportMethod(fs).invoke(
+ fs, snapshotDir, fromSnapshot, toSnapshot);
+ } catch (InvocationTargetException e) {
+ throw new IOException(e.getCause());
+ } catch (NoSuchMethodException|IllegalAccessException e) {
+ throw new IllegalArgumentException(
+ "Failed to invoke getSnapshotDiffReport.", e);
+ }
+ }
+
private String getSnapshotName(String name) {
return Path.CUR_DIR.equals(name) ? "" : name;
}
@@ -327,14 +370,7 @@ class DistCpSync {
private boolean checkNoChange(FileSystem fs, Path path) {
try {
final String from = getSnapshotName(context.getFromSnapshot());
- SnapshotDiffReport targetDiff = null;
- if (fs instanceof DistributedFileSystem) {
- DistributedFileSystem dfs = (DistributedFileSystem)fs;
- targetDiff = dfs.getSnapshotDiffReport(path, from, "");
- } else {
- WebHdfsFileSystem webHdfs = (WebHdfsFileSystem)fs;
- targetDiff = webHdfs.getSnapshotDiffReport(path, from, "");
- }
+ SnapshotDiffReport targetDiff = getSnapshotDiffReport(fs, path, from, "");
if (!targetDiff.getDiffList().isEmpty()) {
DistCp.LOG.warn("The target has been modified since snapshot "
+ context.getFromSnapshot());
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
index 93796e752eb..0fbcd6571c6 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSync.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.fs.CommonPathCapabilities;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -38,6 +40,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.tools.mapred.CopyMapper;
import org.junit.After;
import org.junit.Assert;
@@ -47,6 +50,7 @@ import org.junit.Test;
import java.io.IOException;
import java.io.FileWriter;
import java.io.BufferedWriter;
+import java.net.URI;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.ArrayList;
@@ -56,6 +60,9 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
+import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
public class TestDistCpSync {
private MiniDFSCluster cluster;
private final Configuration conf = new HdfsConfiguration();
@@ -89,6 +96,7 @@ public class TestDistCpSync {
conf.set(DistCpConstants.CONF_LABEL_TARGET_WORK_PATH, target.toString());
conf.set(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH, target.toString());
+ conf.setClass("fs.dummy.impl", DummyFs.class, FileSystem.class);
}
@After
@@ -1276,4 +1284,63 @@ public class TestDistCpSync {
verifyCopyByFs(sourceFS, targetFS, sourceFS.getFileStatus(sourceFSPath),
targetFS.getFileStatus(targetFSPath), false);
}
+
+ @Test
+ public void testSyncSnapshotDiffWithLocalFileSystem() throws Exception {
+ String[] args = new String[]{"-update", "-diff", "s1", "s2",
+ "file:///source", "file:///target"};
+ LambdaTestUtils.intercept(
+ UnsupportedOperationException.class,
+ "The source file system file does not support snapshot",
+ () -> new DistCp(conf, OptionsParser.parse(args)).execute());
+ }
+
+ @Test
+ public void testSyncSnapshotDiffWithDummyFileSystem() {
+ String[] args =
+ new String[] { "-update", "-diff", "s1", "s2", "dummy:///source",
+ "dummy:///target" };
+ try {
+ FileSystem dummyFs = FileSystem.get(URI.create("dummy:///"), conf);
+ assertThat(dummyFs).isInstanceOf(DummyFs.class);
+ new DistCp(conf, OptionsParser.parse(args)).execute();
+ } catch (UnsupportedOperationException e) {
+ throw e;
+ } catch (Exception e) {
+ // can expect other exceptions as source and target paths
+ // are not created.
+ }
+ }
+
+ public static class DummyFs extends RawLocalFileSystem {
+ public DummyFs() {
+ super();
+ }
+
+ public URI getUri() {
+ return URI.create("dummy:///");
+ }
+
+ @Override
+ public boolean hasPathCapability(Path path, String capability)
+ throws IOException {
+ switch (validatePathCapabilityArgs(makeQualified(path), capability)) {
+ case CommonPathCapabilities.FS_SNAPSHOTS:
+ return true;
+ default:
+ return super.hasPathCapability(path, capability);
+ }
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ return new FileStatus();
+ }
+
+ public SnapshotDiffReport getSnapshotDiffReport(final Path snapshotDir,
+ final String fromSnapshot, final String toSnapshot) {
+ return new SnapshotDiffReport(snapshotDir.getName(), fromSnapshot,
+ toSnapshot, new ArrayList The The The client is required to provide details such as queue,
+ {@link Resource} required to run the Currently the During the submission process, it checks whether the application
+ already exists. If the application exists, it will simply return
+ SubmitApplicationResponse In secure mode,the The client, via {@link FailApplicationAttemptRequest} provides the
+ {@link ApplicationAttemptId} of the attempt to be failed. In secure mode,the Currently, the
+ * 100 - 199 : Informational responses
+ * 200 - 299 : Successful responses
+ * 300 - 399 : Redirection messages
+ * 400 - 499 : Client error responses
+ * 500 - 599 : Server error responses
+ *
+ */
+ public static final Integer HTTP_STATUS_CATEGORY_QUOTIENT = 100;
private AbfsHttpConstants() {}
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
index a59f76b6d0f..872364a8e61 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java
@@ -35,6 +35,11 @@ public final class ConfigurationKeys {
* path to determine HNS status.
*/
public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled";
+ /**
+ * Enable or disable expect hundred continue header.
+ * Value: {@value}.
+ */
+ public static final String FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED = "fs.azure.account.expect.header.enabled";
public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key";
public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)";
public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode";
@@ -233,6 +238,9 @@ public final class ConfigurationKeys {
/** Key for rate limit capacity, as used by IO operations which try to throttle themselves. */
public static final String FS_AZURE_ABFS_IO_RATE_LIMIT = "fs.azure.io.rate.limit";
+ /** Add extra resilience to rename failures, at the expense of performance. */
+ public static final String FS_AZURE_ABFS_RENAME_RESILIENCE = "fs.azure.enable.rename.resilience";
+
public static String accountProperty(String property, String account) {
return property + "." + account;
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
index 9994d9f5207..32f9966e30a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java
@@ -32,7 +32,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_ST
public final class FileSystemConfigurations {
public static final String DEFAULT_FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "";
-
+ public static final boolean DEFAULT_FS_AZURE_ACCOUNT_IS_EXPECT_HEADER_ENABLED = true;
public static final String USER_HOME_DIRECTORY_PREFIX = "/user";
private static final int SIXTY_SECONDS = 60 * 1000;
@@ -118,6 +118,7 @@ public final class FileSystemConfigurations {
public static final int STREAM_ID_LEN = 12;
public static final boolean DEFAULT_ENABLE_ABFS_LIST_ITERATOR = true;
+ public static final boolean DEFAULT_ENABLE_ABFS_RENAME_RESILIENCE = true;
/**
* Limit of queued block upload operations before writes
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
index d4065ac2836..b123e90170e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/HttpHeaderConfigurations.java
@@ -70,6 +70,7 @@ public final class HttpHeaderConfigurations {
public static final String X_MS_LEASE_ID = "x-ms-lease-id";
public static final String X_MS_PROPOSED_LEASE_ID = "x-ms-proposed-lease-id";
public static final String X_MS_LEASE_BREAK_PERIOD = "x-ms-lease-break-period";
+ public static final String EXPECT = "Expect";
private HttpHeaderConfigurations() {}
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
index 19620212134..285297024c7 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/InvalidAbfsRestOperationException.java
@@ -30,6 +30,9 @@ import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class InvalidAbfsRestOperationException extends AbfsRestOperationException {
+
+ private static final String ERROR_MESSAGE = "InvalidAbfsRestOperationException";
+
public InvalidAbfsRestOperationException(
final Exception innerException) {
super(
@@ -37,7 +40,23 @@ public class InvalidAbfsRestOperationException extends AbfsRestOperationExceptio
AzureServiceErrorCode.UNKNOWN.getErrorCode(),
innerException != null
? innerException.toString()
- : "InvalidAbfsRestOperationException",
+ : ERROR_MESSAGE,
innerException);
}
+
+ /**
+ * Adds the retry count along with the exception.
+ * @param innerException The inner exception which is originally caught.
+ * @param retryCount The retry count when the exception was thrown.
+ */
+ public InvalidAbfsRestOperationException(
+ final Exception innerException, int retryCount) {
+ super(
+ AzureServiceErrorCode.UNKNOWN.getStatusCode(),
+ AzureServiceErrorCode.UNKNOWN.getErrorCode(),
+ innerException != null
+ ? innerException.toString()
+ : ERROR_MESSAGE + " RetryCount: " + retryCount,
+ innerException);
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
index 7369bfaf564..57e559a60ec 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AppendRequestParameters.java
@@ -34,19 +34,22 @@ public class AppendRequestParameters {
private final Mode mode;
private final boolean isAppendBlob;
private final String leaseId;
+ private boolean isExpectHeaderEnabled;
public AppendRequestParameters(final long position,
final int offset,
final int length,
final Mode mode,
final boolean isAppendBlob,
- final String leaseId) {
+ final String leaseId,
+ final boolean isExpectHeaderEnabled) {
this.position = position;
this.offset = offset;
this.length = length;
this.mode = mode;
this.isAppendBlob = isAppendBlob;
this.leaseId = leaseId;
+ this.isExpectHeaderEnabled = isExpectHeaderEnabled;
}
public long getPosition() {
@@ -72,4 +75,12 @@ public class AppendRequestParameters {
public String getLeaseId() {
return this.leaseId;
}
+
+ public boolean isExpectHeaderEnabled() {
+ return isExpectHeaderEnabled;
+ }
+
+ public void setExpectHeaderEnabled(boolean expectHeaderEnabled) {
+ isExpectHeaderEnabled = expectHeaderEnabled;
+ }
}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java
index 8bc31c4f92b..8a5e9db8553 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java
@@ -66,6 +66,10 @@ public enum AzureServiceErrorCode {
return this.errorCode;
}
+ public String getErrorMessage() {
+ return this.errorMessage;
+ }
+
public static List
+ *
+ */
+ private static List
+ */
+ abstract Boolean canCapture(Exception ex,
+ Integer statusCode,
+ String serverErrorMessage);
+
+ /**
+ * Returns the abbreviation corresponding to the server response error.
+ *
+ * @param statusCode statusCode on the server response
+ * @param serverErrorMessage serverErrorMessage on the server response.
+ *
+ * @return abbreviation on the basis of the statusCode and the serverErrorMessage
+ */
+ abstract String getAbbreviation(Integer statusCode, String serverErrorMessage);
+
+ /**
+ * Converts the server-error response to an abbreviation if the response can be
+ * categorised by the implementation.
+ *
+ * @param ex exception received while making API request
+ * @param statusCode statusCode received in the server-response
+ * @param serverErrorMessage error-message received in the server-response
+ *
+ * @return abbreviation if the server-response can be categorised by the implementation.
+ * null if the server-response can not be categorised by the implementation.
+ */
+ public String captureAndGetAbbreviation(Exception ex,
+ Integer statusCode,
+ String serverErrorMessage) {
+ if (canCapture(ex, statusCode, serverErrorMessage)) {
+ return getAbbreviation(statusCode, serverErrorMessage);
+ }
+ return null;
+ }
+
+ /**
+ * Checks if a required search-string is in the exception's message.
+ */
+ Boolean checkExceptionMessage(final Exception exceptionCaptured,
+ final String search) {
+ if (search == null) {
+ return false;
+ }
+ if (exceptionCaptured != null
+ && exceptionCaptured.getMessage() != null
+ && exceptionCaptured.getMessage()
+ .toLowerCase(Locale.US)
+ .contains(search.toLowerCase(Locale.US))) {
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java
new file mode 100644
index 00000000000..dd67a0cb8cb
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/retryReasonCategories/ServerErrorRetryReason.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * ResourceManager responds with a new, monotonically
+ increasing, {@link ApplicationId} which is used by the client to submit
+ a new application.ResourceManager also responds with details such
+ as maximum resource capabilities in the cluster as specified in
+ {@link GetNewApplicationResponse}.ApplicationId
+ @return response containing the new ApplicationId to be used
+ to submit an application
+ @throws YarnException
+ @throws IOException
+ @see #submitApplication(SubmitApplicationRequest)]]>
+
+
+ ResourceManager.ApplicationMaster,
+ the equivalent of {@link ContainerLaunchContext} for launching
+ the ApplicationMaster etc. via the
+ {@link SubmitApplicationRequest}.ResourceManager sends an immediate (empty)
+ {@link SubmitApplicationResponse} on accepting the submission and throws
+ an exception if it rejects the submission. However, this call needs to be
+ followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ to make sure that the application gets properly submitted - obtaining a
+ {@link SubmitApplicationResponse} from ResourceManager doesn't guarantee
+ that RM 'remembers' this application beyond failover or restart. If RM
+ failover or RM restart happens before ResourceManager saves the
+ application's state successfully, the subsequent
+ {@link #getApplicationReport(GetApplicationReportRequest)} will throw
+ a {@link ApplicationNotFoundException}. The Clients need to re-submit
+ the application with the same {@link ApplicationSubmissionContext} when
+ it encounters the {@link ApplicationNotFoundException} on the
+ {@link #getApplicationReport(GetApplicationReportRequest)} call.ResourceManager verifies access to
+ queues etc. before accepting the application submission.ResourceManager to fail an application attempt.ResourceManager verifies access to the
+ application, queue etc. before failing the attempt.ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request.ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+
+
+ ResourceManager to abort submitted application.
The client, via {@link KillApplicationRequest} provides the + {@link ApplicationId} of the application to be aborted.
+ + In secure mode,the ResourceManager verifies access to the
+ application, queue etc. before terminating the application.
Currently, the ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request.
ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException
+ @see #getQueueUserAcls(GetQueueUserAclsInfoRequest)]]>
+
+
+ ResourceManager.
+
+ The ResourceManager responds with a
+ {@link GetClusterMetricsResponse} which includes the
+ {@link YarnClusterMetrics} with details such as number of current
+ nodes in the cluster.
ResourceManager.
+
+ The ResourceManager responds with a
+ {@link GetClusterNodesResponse} which includes the
+ {@link NodeReport} for all the nodes in the cluster.
ResourceManager.
+
+ The client, via {@link GetQueueInfoRequest}, can ask for details such + as used/total resources, child queues, running applications etc.
+ + In secure mode,the ResourceManager verifies access before
+ providing the information.
ResourceManager.
+
+
+ The ResourceManager responds with queue acls for all
+ existing queues.
The ResourceManager responds with a new, unique,
+ {@link ReservationId} which is used by the client to submit
+ a new reservation.
ReservationId
+ @return response containing the new ReservationId to be used
+ to submit a new reservation
+ @throws YarnException if the reservation system is not enabled.
+ @throws IOException on IO failures.
+ @see #submitReservation(ReservationSubmissionRequest)]]>
+ + The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and concurrency needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +
+ ++ In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationSubmissionRequest is satisfiable the + {@code ResourceManager} answers with a + {@link ReservationSubmissionResponse} that include a non-null + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the reason. + + On application submission the client can use this {@link ReservationId} to + obtain access to the reserved resources. +
+ ++ The system guarantees that during the time-range specified by the user, the + reservationID will be corresponding to a valid reservation. The amount of + capacity dedicated to such queue can vary overtime, depending of the + allocation that has been determined. But it is guaranteed to satisfy all + the constraint expressed by the user in the + {@link ReservationSubmissionRequest}. +
+ + @param request the request to submit a new Reservation + @return response the {@link ReservationId} on accepting the submission + @throws YarnException if the request is invalid or reservation cannot be + created successfully + @throws IOException]]> ++ The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationUpdateRequest}. Upon success the previous allocation is + substituted by the new one, and on failure (i.e., if the system cannot find + a valid allocation for the updated request), the previous allocation + remains valid. + + The {@link ReservationId} is not changed, and applications currently + running within this reservation will automatically receive the resources + based on the new allocation. +
+ + @param request to update an existing Reservation (the ReservationRequest + should refer to an existing valid {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +ResourceManager to signal a container. For example,
+ the client can send command OUTPUT_THREAD_DUMP to dump threads of the
+ container.
+
+ The client, via {@link SignalContainerRequest} provides the + id of the container and the signal command.
+ + In secure mode,the ResourceManager verifies access to the
+ application before signaling the container.
+ The user needs to have MODIFY_APP permission.
Currently, the ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request.
ResourceManager returns an empty response
+ on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+ ResourceManager
+ to submit/abort jobs and to get information on applications, cluster metrics,
+ nodes, queues and ACLs.]]>
+ ApplicationHistoryServer to
+ get the information of completed applications etc.
+ ]]>
+ ApplicationMaster to register with
+ the ResourceManager.
+
+
+
+ The ApplicationMaster needs to provide details such as RPC
+ Port, HTTP tracking url etc. as specified in
+ {@link RegisterApplicationMasterRequest}.
+
+ The ResourceManager responds with critical details such as
+ maximum resource capabilities in the cluster as specified in
+ {@link RegisterApplicationMasterResponse}.
+
+ Re-register is only allowed for Unmanaged Application Master
+ (UAM) HA, with
+ {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#getKeepContainersAcrossApplicationAttempts()}
+ set to true.
+
ApplicationMaster to notify the
+ ResourceManager about its completion (success or failed).
+
+ The ApplicationMaster has to provide details such as
+ final state, diagnostics (in case of failures) etc. as specified in
+ {@link FinishApplicationMasterRequest}.
The ResourceManager responds with
+ {@link FinishApplicationMasterResponse}.
ApplicationMaster and the
+ ResourceManager.
+
+
+
+ The ApplicationMaster uses this interface to provide a list of
+ {@link ResourceRequest} and returns unused {@link Container} allocated to
+ it via {@link AllocateRequest}. Optionally, the
+ ApplicationMaster can also blacklist resources which
+ it doesn't want to use.
+
+ This also doubles up as a heartbeat to let the
+ ResourceManager know that the ApplicationMaster
+ is alive. Thus, applications should periodically make this call to be kept
+ alive. The frequency depends on
+ {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.
+
+ The ResourceManager responds with list of allocated
+ {@link Container}, status of completed containers and headroom information
+ for the application.
+
+ The ApplicationMaster can use the available headroom
+ (resources) to decide how to utilized allocated resources and make informed
+ decisions about future resource requests.
+
ApplicationMaster
+ and the ResourceManager.
+
+ This is used by the ApplicationMaster to register/unregister
+ and to request and obtain resources in the cluster from the
+ ResourceManager.
SharedCacheManager. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application will be
+ using the resource.
+
+
+
+ The SharedCacheManager responds with whether or not the
+ resource exists in the cache. If the resource exists, a Path
+ to the resource in the shared cache is returned. If the resource does not
+ exist, the response is empty.
+
SharedCacheManager. This method is called once an application
+ is no longer using a claimed resource in the shared cache. The client uses
+ a checksum to identify the resource and an {@link ApplicationId} to
+ identify which application is releasing the resource.
+
+
+ + Note: This method is an optimization and the client is not required to call + it for correctness. +
+ +
+ Currently the SharedCacheManager sends an empty response.
+
SharedCacheManager to claim
+ and release resources in the shared cache.
+ ]]>
+ ApplicationMaster provides a list of
+ {@link StartContainerRequest}s to a NodeManager to
+ start {@link Container}s allocated to it using this interface.
+
+
+
+ The ApplicationMaster has to provide details such as allocated
+ resource capability, security tokens (if enabled), command to be executed
+ to start the container, environment for the process, necessary
+ binaries/jar/shared-objects etc. via the {@link ContainerLaunchContext} in
+ the {@link StartContainerRequest}.
+
+ The NodeManager sends a response via
+ {@link StartContainersResponse} which includes a list of
+ {@link Container}s of successfully launched {@link Container}s, a
+ containerId-to-exception map for each failed {@link StartContainerRequest} in
+ which the exception indicates errors from per container and a
+ allServicesMetaData map between the names of auxiliary services and their
+ corresponding meta-data. Note: None-container-specific exceptions will
+ still be thrown by the API method itself.
+
+ The ApplicationMaster can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the to-be-launched or launched containers.
+
ApplicationMaster requests a NodeManager to
+ stop a list of {@link Container}s allocated to it using this
+ interface.
+
+
+
+ The ApplicationMaster sends a {@link StopContainersRequest}
+ which includes the {@link ContainerId}s of the containers to be stopped.
+
+ The NodeManager sends a response via
+ {@link StopContainersResponse} which includes a list of {@link ContainerId}
+ s of successfully stopped containers, a containerId-to-exception map for
+ each failed request in which the exception indicates errors from per
+ container. Note: None-container-specific exceptions will still be thrown by
+ the API method itself. ApplicationMaster can use
+ {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated
+ statuses of the containers.
+
ApplicationMaster to request for current
+ statuses of Containers from the NodeManager.
+
+
+
+ The ApplicationMaster sends a
+ {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s
+ of all containers whose statuses are needed.
+
+ The NodeManager responds with
+ {@link GetContainerStatusesResponse} which includes a list of
+ {@link ContainerStatus} of the successfully queried containers and a
+ containerId-to-exception map for each failed request in which the exception
+ indicates errors from per container. Note: None-container-specific
+ exceptions will still be thrown by the API method itself.
+
ContainerStatuses of containers with
+ the specified ContainerIds
+ @return response containing the list of ContainerStatus of the
+ successfully queried containers and a containerId-to-exception map
+ for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+ ApplicationMaster to request for
+ resource increase of running containers on the NodeManager.
+
+
+ @param request
+ request to increase resource of a list of containers
+ @return response which includes a list of containerIds of containers
+ whose resource has been successfully increased and a
+ containerId-to-exception map for failed requests.
+
+ @throws YarnException
+ @throws IOException]]>
+ ApplicationMaster to request for
+ resource update of running containers on the NodeManager.
+
+
+ @param request
+ request to update resource of a list of containers
+ @return response which includes a list of containerIds of containers
+ whose resource has been successfully updated and a
+ containerId-to-exception map for failed requests.
+
+ @throws YarnException Exception specific to YARN
+ @throws IOException IOException thrown from NodeManager]]>
+ ApplicationMaster and a
+ NodeManager to start/stop and increase resource of containers
+ and to get status of running containers.
+
+ If security is enabled the NodeManager verifies that the
+ ApplicationMaster has truly been allocated the container
+ by the ResourceManager and also verifies all interactions such
+ as stopping the container or obtaining status information for the container.
+
ResourceManager about the application's resource requirements.
+ @return the list of ResourceRequest
+ @see ResourceRequest]]>
+ ResourceManager about the application's resource requirements.
+ @param resourceRequests list of ResourceRequest to update the
+ ResourceManager about the application's
+ resource requirements
+ @see ResourceRequest]]>
+ ApplicationMaster.
+ @return list of ContainerId of containers being
+ released by the ApplicationMaster]]>
+ ApplicationMaster
+ @param releaseContainers list of ContainerId of
+ containers being released by the
+ ApplicationMaster]]>
+ ApplicationMaster.
+ @return the ResourceBlacklistRequest being sent by the
+ ApplicationMaster
+ @see ResourceBlacklistRequest]]>
+ ResourceManager about the blacklist additions and removals
+ per the ApplicationMaster.
+
+ @param resourceBlacklistRequest the ResourceBlacklistRequest
+ to inform the ResourceManager about
+ the blacklist additions and removals
+ per the ApplicationMaster
+ @see ResourceBlacklistRequest]]>
+ ApplicationMaster.]]>
+ UpdateContainerRequest for
+ containers to be updated]]>
+ ApplicationMaster.]]>
+ ResourceManager about the application's resource
+ requirements.]]>
+ ApplicationMaster to the
+ ResourceManager to obtain resources in the cluster.
+
+ The request includes: +
ResourceManager about the application's
+ resource requirements.
+ ResourceManager about the change in
+ requirements of running containers.
+ responseId of the request
+ @return {@link AllocateRequestBuilder}]]>
+ progress of the request
+ @return {@link AllocateRequestBuilder}]]>
+ askList of the request
+ @return {@link AllocateRequestBuilder}]]>
+ releaseList of the request
+ @return {@link AllocateRequestBuilder}]]>
+ resourceBlacklistRequest of the request
+ @return {@link AllocateRequestBuilder}]]>
+ updateRequests of the request
+ @return {@link AllocateRequestBuilder}]]>
+ SchedulingRequest of the request
+ @return {@link AllocateRequestBuilder}]]>
+ ApplicationMaster to take some action then it will send an
+ AMCommand to the ApplicationMaster. See AMCommand
+ for details on commands and actions for them.
+ @return AMCommand if the ApplicationMaster should
+ take action, null otherwise
+ @see AMCommand]]>
+ Container by the
+ ResourceManager.
+ @return list of newly allocated Container]]>
+ NodeReports. Updates could
+ be changes in health, availability etc of the nodes.
+ @return The delta of updated nodes since the last response]]>
+ + AM will receive one NMToken per NM irrespective of the number of containers + issued on same NM. AM is expected to store these tokens until issued a + new token for the same NM. + @return list of NMTokens required for communicating with NM]]> +
ResourceManager from previous application attempts.]]>
+ ApplicationMaster during resource negotiation.
+ + The response, includes: +
Application Master to the
+ Node Manager to change the resource quota of a container.
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+ NodeManager to the
+ ApplicationMaster when asked to update container resource.
+
+
+ @see ContainerManagementProtocol#updateContainer(ContainerUpdateRequest)]]>
+ ApplicationAttemptId of the attempt.]]>
+ ResourceManager
+ to fail an application attempt.
+
+ The request includes the {@link ApplicationAttemptId} of the attempt to + be failed.
+ + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +ResourceManager to the client
+ failing an application attempt.
+
+ Currently it's empty.
+ + @see ApplicationClientProtocol#failApplicationAttempt(FailApplicationAttemptRequest)]]> +ApplicationMaster.
+ @return final state of the ApplicationMaster]]>
+ ApplicationMaster
+ @param finalState final state of the ApplicationMaster]]>
+ ApplicationMaster.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return tracking URLfor the ApplicationMaster]]>
+ ApplicationMaster.
+ This is the web-URL to which ResourceManager or web-application proxy will
+ redirect client/users once the application is finished and the
+ ApplicationMaster is gone.
+ + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +
++ Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +
+
+ @param url
+ tracking URLfor the ApplicationMaster]]>
+
ApplicationMaster on it's completion.
+ + The response, includes: +
+ Note: The flag indicates whether the application has successfully + unregistered and is safe to stop. The application may stop after the flag is + true. If the application stops before the flag is true then the RM may retry + the application. + + @see ApplicationMasterProtocol#finishApplicationMaster(FinishApplicationMasterRequest)]]> +
ApplicationAttemptId of an application attempt]]>
+ ApplicationAttemptId of an application attempt]]>
+ ResourceManager to get an
+ {@link ApplicationAttemptReport} for an application attempt.
+
+
+ + The request should include the {@link ApplicationAttemptId} of the + application attempt. +
+ + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +ApplicationAttemptReport for the application attempt]]>
+ ApplicationAttemptReport for the application attempt]]>
+ ResourceManager to a client requesting
+ an application attempt report.
+
+
+ + The response includes an {@link ApplicationAttemptReport} which has the + details about the particular application attempt +
+ + @see ApplicationAttemptReport + @see ApplicationHistoryProtocol#getApplicationAttemptReport(GetApplicationAttemptReportRequest)]]> +ApplicationId of an application]]>
+ ApplicationId of an application]]>
+ ResourceManager.
+
+
+ @see ApplicationHistoryProtocol#getApplicationAttempts(GetApplicationAttemptsRequest)]]>
+ ApplicationReport of an application]]>
+ ApplicationReport of an application]]>
+ ResourceManager to a client requesting
+ a list of {@link ApplicationAttemptReport} for application attempts.
+
+
+
+ The ApplicationAttemptReport for each application includes the
+ details of an application attempt.
+
ApplicationId of the application]]>
+ ApplicationId of the application]]>
+ ResourceManager to
+ get an {@link ApplicationReport} for an application.
+
+ The request should include the {@link ApplicationId} of the + application.
+ + @see ApplicationClientProtocol#getApplicationReport(GetApplicationReportRequest) + @see ApplicationReport]]> +ApplicationReport for the application]]>
+ ResourceManager to a client
+ requesting an application report.
+
+ The response includes an {@link ApplicationReport} which has details such
+ as user, queue, name, host on which the ApplicationMaster is
+ running, RPC port, tracking URL, diagnostics, start time etc.
ResourceManager.
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+
+ Setting any of the parameters to null, would just disable that + filter
+ + @param scope {@link ApplicationsRequestScope} to filter by + @param users list of users to filter by + @param queues list of scheduler queues to filter by + @param applicationTypes types of applications + @param applicationTags application tags to filter by + @param applicationStates application states to filter by + @param startRange range of application start times to filter by + @param finishRange range of application finish times to filter by + @param limit number of applications to limit to + @return {@link GetApplicationsRequest} to be used with + {@link ApplicationClientProtocol#getApplications(GetApplicationsRequest)}]]> +ResourceManager.
+
+
+ @param scope {@link ApplicationsRequestScope} to filter by
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+ ResourceManager.
+
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+ ResourceManager.
+
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in {@link GetApplicationsRequest}]]>
+ ResourceManager.
+
+
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ @return a report of Applications in GetApplicationsRequest]]>
+ ResourceManager.
+
+ @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)]]>
+ ApplicationReport for applications]]>
+ ResourceManager to a client
+ requesting an {@link ApplicationReport} for applications.
+
+ The ApplicationReport for each application includes details
+ such as user, queue, name, host on which the ApplicationMaster
+ is running, RPC port, tracking URL, diagnostics, start time etc.
ResourceManager.
+
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+ (GetAttributesToNodesRequest)]]>
+ ResourceManager to a client requesting
+ node to attribute value mapping for all or given set of Node AttributeKey's.
+
+
+ @see ApplicationClientProtocol#getAttributesToNodes
+ (GetAttributesToNodesRequest)]]>
+ ResourceManager.
+
+ Currently, this is empty.
+ + @see ApplicationClientProtocol#getClusterMetrics(GetClusterMetricsRequest)]]> +YarnClusterMetrics for the cluster]]>
+ ResourceManager.
+
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+ ResourceManager to a client requesting
+ a node attributes in cluster.
+
+
+ @see ApplicationClientProtocol#getClusterNodeAttributes
+ (GetClusterNodeAttributesRequest)]]>
+ ResourceManager.
+
+ The request will ask for all nodes in the given {@link NodeState}s.
+
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+ NodeReport for all nodes in the cluster]]>
+ ResourceManager to a client
+ requesting a {@link NodeReport} for all nodes.
+
+ The NodeReport contains per-node information such as
+ available resources, number of containers, tracking url, rack name, health
+ status etc.
+
+ @see NodeReport
+ @see ApplicationClientProtocol#getClusterNodes(GetClusterNodesRequest)]]>
+
ContainerId of the Container]]>
+ ContainerId of the container]]>
+ ResourceManager to get an
+ {@link ContainerReport} for a container.
+ ]]>
+ ContainerReport for the container]]>
+ ResourceManager to a client requesting
+ a container report.
+
+
+ + The response includes a {@link ContainerReport} which has details of a + container. +
]]> +ApplicationAttemptId of an application attempt]]>
+ ApplicationAttemptId of an application attempt]]>
+ ResourceManager.
+
+
+ @see ApplicationHistoryProtocol#getContainers(GetContainersRequest)]]>
+ ContainerReport for all the containers of an
+ application attempt]]>
+ ContainerReport for all the containers of
+ an application attempt]]>
+ ResourceManager to a client requesting
+ a list of {@link ContainerReport} for containers.
+
+
+
+ The ContainerReport for each container includes the container
+ details.
+
ContainerStatus.
+
+ @return the list of ContainerIds of containers for which to
+ obtain the ContainerStatus.]]>
+ ContainerStatus
+
+ @param containerIds
+ a list of ContainerIds of containers for which to
+ obtain the ContainerStatus]]>
+ NodeManager to get {@link ContainerStatus} of requested
+ containers.
+
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+ ContainerStatuses of the requested containers.]]>
+ ApplicationMaster when asked to obtain the
+ ContainerStatus of requested containers.
+
+ @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest)]]>
+ Currently, this is empty.
+ + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +ApplicationId allocated by the
+ ResourceManager.
+ @return new ApplicationId allocated by the
+ ResourceManager]]>
+ ResourceManager to the client for
+ a request to get a new {@link ApplicationId} for submitting applications.
+
+ Clients can submit an application with the returned + {@link ApplicationId}.
+ + @see ApplicationClientProtocol#getNewApplication(GetNewApplicationRequest)]]> +ResourceManager to the client for
+ a request to get a new {@link ReservationId} for submitting reservations.
+
+ Clients can submit an reservation with the returned + {@link ReservationId}.
+ + {@code ApplicationClientProtocol#getNewReservation(GetNewReservationRequest)}]]> +ResourceManager.
+
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+ ResourceManager to a client requesting
+ nodes to attributes mapping.
+
+
+ @see ApplicationClientProtocol#getNodesToAttributes
+ (GetNodesToAttributesRequest)]]>
+ true if applications' information is to be included,
+ else false]]>
+ true if information about child queues is required,
+ else false]]>
+ true if information about entire hierarchy is
+ required, false otherwise]]>
+ ResourceManager.
+
+ @see ApplicationClientProtocol#getQueueInfo(GetQueueInfoRequest)]]>
+ QueueInfo for the specified queue]]>
+ ResourceManager to
+ get queue acls for the current user.
+
+ Currently, this is empty.
+ + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +QueueUserACLInfo per queue for the user]]>
+ ResourceManager to clients
+ seeking queue acls for the user.
+
+ The response contains a list of {@link QueueUserACLInfo} which + provides information about {@link QueueACL} per queue.
+ + @see QueueACL + @see QueueUserACLInfo + @see ApplicationClientProtocol#getQueueUserAcls(GetQueueUserAclsInfoRequest)]]> +Application Master to the
+ Node Manager to change the resource quota of a container.
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+ NodeManager to the
+ ApplicationMaster when asked to increase container resource.
+
+
+ @see ContainerManagementProtocol#increaseContainersResource(IncreaseContainersResourceRequest)]]>
+ ApplicationId of the application to be aborted]]>
+ ResourceManager
+ to abort a submitted application.
+
+ The request includes the {@link ApplicationId} of the application to be + aborted.
+ + @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]> ++ The response, includes: +
ResourceManager crashes before the process of killing the
+ application is completed, the ResourceManager may retry this
+ application on recovery.
+
+ @see ApplicationClientProtocol#forceKillApplication(KillApplicationRequest)]]>
+ ApplicationId of the application to be moved]]>
+ ApplicationId of the application to be moved]]>
+ ResourceManager
+ to move a submitted application to a different queue.
+
+ The request includes the {@link ApplicationId} of the application to be + moved and the queue to place it in.
+ + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +ResourceManager to the client moving
+ a submitted application to a different queue.
+
+ + A response without exception means that the move has completed successfully. +
+ + @see ApplicationClientProtocol#moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest)]]> +RegisterApplicationMasterRequest]]>
+ ApplicationMaster is
+ running.
+ @return host on which the ApplicationMaster is running]]>
+ ApplicationMaster is
+ running.
+ @param host host on which the ApplicationMaster
+ is running]]>
+ ApplicationMaster.
+ This url if contains scheme then that will be used by resource manager
+ web application proxy otherwise it will default to http.
+ @return tracking URL for the ApplicationMaster]]>
+ ApplicationMaster while
+ it is running. This is the web-URL to which ResourceManager or
+ web-application proxy will redirect client/users while the application and
+ the ApplicationMaster are still running.
+ + If the passed url has a scheme then that will be used by the + ResourceManager and web-application proxy, otherwise the scheme will + default to http. +
++ Empty, null, "N/A" strings are all valid besides a real URL. In case an url + isn't explicitly passed, it defaults to "N/A" on the ResourceManager. +
+
+ @param trackingUrl
+ tracking URLfor the ApplicationMaster]]>
+
ApplicationACLs]]>
+ The ClientToAMToken master key is sent to ApplicationMaster
+ by ResourceManager via {@link RegisterApplicationMasterResponse}
+ , used to verify corresponding ClientToAMToken.
+ @return the queue that the application was placed in.]]> +
]]> +
ResourceManager from previous application attempts.
+
+
+ @return the list of running containers as viewed by
+ ResourceManager from previous application attempts
+ @see RegisterApplicationMasterResponse#getNMTokensFromPreviousAttempts()]]>
+ ContainerId of the container to re-initialize.]]>
+ ContainerLaunchContext of to re-initialize the
+ container with.]]>
+ ApplicationId]]>
+ ApplicationId]]>
+ key]]>
+ SharedCacheManager when
+ releasing a resource in the shared cache.
+
+
+ + Currently, this is empty. +
]]> +ResourceManager to a client on
+ reservation submission.
+
+ Currently, this is empty.
+ + {@code ApplicationClientProtocol#submitReservation( + ReservationSubmissionRequest)}]]> +ContainerId of the container to localize resources.]]>
+ LocalResource required by the container]]>
+ ContainerId of the container to signal.]]>
+ SignalContainerCommand of the signal request.]]>
+ ResourceManager
+ or by the ApplicationMaster to the NodeManager
+ to signal a container.
+ @see SignalContainerCommand ]]>
+ ResourceManager to the client
+ signalling a container.
+
+ Currently it's empty.
+ + @see ApplicationClientProtocol#signalToContainer(SignalContainerRequest)]]> +NodeManager.
+
+ @return ContainerLaunchContext for the container to be started
+ by the NodeManager]]>
+ NodeManager
+ @param context ContainerLaunchContext for the container to be
+ started by the NodeManager]]>
+ ApplicationMaster to the
+ NodeManager to start a container.
+
+ The ApplicationMaster has to provide details such as
+ allocated resource capability, security tokens (if enabled), command
+ to be executed to start the container, environment for the process,
+ necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
ApplicationMaster to the NodeManager to
+ start containers.
+
+
+
+ In each {@link StartContainerRequest}, the ApplicationMaster has
+ to provide details such as allocated resource capability, security tokens (if
+ enabled), command to be executed to start the container, environment for the
+ process, necessary binaries/jar/shared-objects etc. via the
+ {@link ContainerLaunchContext}.
+
ContainerId s of the containers that are
+ started successfully.
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+ NodeManager.
+
+
+ The meta-data is returned as a Map between the auxiliary service names and
+ their corresponding per service meta-data as an opaque blob
+ ByteBuffer
+
+ To be able to interpret the per-service meta-data, you should consult the + documentation for the Auxiliary-service configured on the NodeManager +
+ + @return a Map between the names of auxiliary services and their + corresponding meta-data]]> +NodeManager to the
+ ApplicationMaster when asked to start an allocated
+ container.
+
+
+ @see ContainerManagementProtocol#startContainers(StartContainersRequest)]]>
+ ContainerIds of containers to be stopped]]>
+ ContainerIds of the containers to be stopped]]>
+ ApplicationMaster to the
+ NodeManager to stop containers.
+
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+ NodeManager to the
+ ApplicationMaster when asked to stop allocated
+ containers.
+
+
+ @see ContainerManagementProtocol#stopContainers(StopContainersRequest)]]>
+ ApplicationSubmissionContext for the application]]>
+ ApplicationSubmissionContext for the
+ application]]>
+ ResourceManager.
+
+ The request, via {@link ApplicationSubmissionContext}, contains
+ details such as queue, {@link Resource} required to run the
+ ApplicationMaster, the equivalent of
+ {@link ContainerLaunchContext} for launching the
+ ApplicationMaster etc.
+
+ @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]>
+
ResourceManager to a client on
+ application submission.
+
+ Currently, this is empty.
+ + @see ApplicationClientProtocol#submitApplication(SubmitApplicationRequest)]]> +ApplicationId of the application]]>
+ ApplicationId of the application]]>
+ Priority of the application to be set.]]>
+ Priority of the application]]>
+ ResourceManager to set or
+ update the application priority.
+
+ + The request includes the {@link ApplicationId} of the application and + {@link Priority} to be set for an application +
+ + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +Priority of the application.]]>
+ Priority of the application]]>
+ ResourceManager to the client on update
+ the application priority.
+
+ + A response without exception means that the move has completed successfully. +
+ + @see ApplicationClientProtocol#updateApplicationPriority(UpdateApplicationPriorityRequest)]]> +ApplicationId of the application]]>
+ ApplicationId of the application]]>
+ ApplicationTimeouts of the application.]]>
+ ApplicationTimeoutss for the
+ application]]>
+ ResourceManager to set or
+ update the application timeout.
+
+ + The request includes the {@link ApplicationId} of the application and timeout + to be set for an application +
]]> +ApplicationTimeouts of the application.]]>
+ ApplicationTimeoutss for the
+ application]]>
+ ResourceManager to the client on update
+ application timeout.
+
+ + A response without exception means that the update has completed + successfully. +
]]> +ApplicationId]]>
+ ApplicationId]]>
+ key]]>
+ SharedCacheManager that claims a
+ resource in the shared cache.
+ ]]>
+ Path if the resource exists in the shared
+ cache, null otherwise]]>
+ Path corresponding to a resource in the shared
+ cache]]>
+ ApplicationAttempId.
+ @return ApplicationId of the ApplicationAttempId]]>
+ Application.
+ @return attempt id of the Application]]>
+ ApplicationAttemptId denotes the particular attempt
+ of an ApplicationMaster for a given {@link ApplicationId}.
+
+ Multiple attempts might be needed to run an application to completion due
+ to temporal failures of the ApplicationMaster such as hardware
+ failures, connectivity issues etc. on the node on which it was scheduled.
ApplicationMaster.
+
+ @return RPC port of this attempt ApplicationMaster]]>
+ ApplicationMaster is running.
+
+ @return host on which this attempt of
+ ApplicationMaster is running]]>
+ ApplicationAttemptId of the attempt]]>
+ ContainerId of the attempt]]>
+ ApplicationMaster of this attempt is
+ running.ApplicationMaster of this attempt.ResourceManager.
+ @return short integer identifier of the ApplicationId]]>
+ ResourceManager which is
+ used to generate globally unique ApplicationId.
+ @return start time of the ResourceManager]]>
+ ApplicationId represents the globally unique
+ identifier for an application.
+
+ The globally unique nature of the identifier is achieved by using the
+ cluster timestamp i.e. start-time of the
+ ResourceManager along with a monotonically increasing counter
+ for the application.
ApplicationId of the application]]>
+ ApplicationAttemptId of the attempt]]>
+ ApplicationMaster
+ is running.
+ @return host on which the ApplicationMaster
+ is running]]>
+ ApplicationMaster.
+ @return RPC port of the ApplicationMaster]]>
+ ApplicationMaster.
+
+ ClientToAMToken is the security token used by the AMs to verify
+ authenticity of any client.
+
+ The ResourceManager, provides a secure token (via
+ {@link ApplicationReport#getClientToAMToken()}) which is verified by the
+ ApplicationMaster when the client directly talks to an AM.
+
ApplicationMaster]]>
+ YarnApplicationState of the application]]>
+ + The AMRM token will be returned only if all the following conditions are + met: +
ApplicationMaster is running.ApplicationMaster.Resource]]>
+ Resource]]>
+ Resource]]>
+ ApplicationId of the submitted application]]>
+ ApplicationId of the submitted
+ application]]>
+ Priority of the application]]>
+ Container with which the ApplicationMaster is
+ launched.
+ @return ContainerLaunchContext for the
+ ApplicationMaster container]]>
+ Container with which the ApplicationMaster is
+ launched.
+ @param amContainer ContainerLaunchContext for the
+ ApplicationMaster container]]>
+ ApplicationMaster for
+ this application.]]>
+ ApplicationMaster
+ for this application.]]>
+ + For unmanaged AM, if the flag is true, RM allows re-register and returns + the running containers in the same attempt back to the UAM for HA. +
+ + @param keepContainers the flag which indicates whether to keep containers + across application attempts.]]> +LogAggregationContext of the application]]>
+ ApplicationTimeouts of the application.]]>
+ + Note: If application timeout value is less than or equal to zero + then application submission will throw an exception. +
+ @param applicationTimeoutsApplicationTimeoutss for the
+ application]]>
+ ApplicationMaster is executed.
+ Resource allocated to the container]]>
+ Container was
+ allocated.
+ @return Priority at which the Container was
+ allocated]]>
+ ContainerToken is the security token used by the framework
+ to verify authenticity of any Container.
The ResourceManager, on container allocation provides a
+ secure token which is verified by the NodeManager on
+ container launch.
Applications do not need to care about ContainerToken, they
+ are transparently handled by the framework - the allocated
+ Container includes the ContainerToken.
ContainerToken for the container]]>
+ + The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}s. + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +
+ + @return the ID corresponding to the original allocation request + which is satisfied by this allocation.]]> +
+ It includes details such as: +
Container was assigned.
+
+ Note: If containers are kept alive across application attempts via
+ {@link ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ the ContainerId does not necessarily contain the current
+ running application attempt's ApplicationAttemptId This
+ container can be allocated by previously exited application attempt and
+ managed by the current running attempt thus have the previous application
+ attempt's ApplicationAttemptId.
+
ApplicationAttemptId of the application to which the
+ Container was assigned]]>
+ getContainerId instead.
+ @return lower 32 bits of identifier of the ContainerId]]>
+ ContainerId]]>
+ ContainerId represents a globally unique identifier
+ for a {@link Container} in the cluster.]]>
+ LocalResource required by the container]]>
+ LocalResource required by the container]]>
+ + This will be used to initialize this application on the specific + {@link AuxiliaryService} running on the NodeManager by calling + {@link AuxiliaryService#initializeApplication(ApplicationInitializationContext)} +
+ + @return application-specific binary service data]]> +ApplicationACLs]]>
+ ApplicationACLs for the application]]>
+ ContainerRetryContext to relaunch container.]]>
+ ContainerRetryContext to
+ relaunch container.]]>
+ ContainerId of the container.]]>
+ Resource of the container.]]>
+ NodeId where container is running.]]>
+ Priority of the container.]]>
+ ContainerState of the container.]]>
+ exit status of the container.]]>
+ Container.]]>
+ Container.]]>
+ ContainerId of the container]]>
+ ExecutionType of the container]]>
+ ContainerState of the container]]>
+ Note: This is valid only for completed containers i.e. containers + with state {@link ContainerState#COMPLETE}. + Otherwise, it returns an ContainerExitStatus.INVALID. +
+ +Containers killed by the framework, either due to being released by + the application or being 'lost' due to node failures etc. have a special + exit code of ContainerExitStatus.ABORTED.
+ +When threshold number of the nodemanager-local-directories or + threshold number of the nodemanager-log-directories become bad, then + container is not launched and is exited with ContainersExitStatus.DISKS_FAILED. +
+ + @return exit status for the container]]> +Resource allocated to the container]]>
+ ExecutionType.]]>
+ LocalResourceType of the resource to be localized]]>
+ LocalResourceType of the resource to be localized]]>
+ LocalResourceVisibility of the resource to be
+ localized]]>
+ LocalResourceVisibility of the resource to be
+ localized]]>
+ PATTERN).
+ @return pattern that should be used to extract entries from the
+ archive.]]>
+ PATTERN).
+ @param pattern pattern that should be used to extract entries
+ from the archive.]]>
+ LocalResource represents a local resource required to
+ run a container.
+
+ The NodeManager is responsible for localizing the resource
+ prior to launching the container.
Applications can specify {@link LocalResourceType} and + {@link LocalResourceVisibility}.
+ + @see LocalResourceType + @see LocalResourceVisibility + @see ContainerLaunchContext + @see ApplicationSubmissionContext + @see ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)]]> ++ The type can be one of: +
NodeManager.
+ + The visibility can be one of: +
NodeManager for which the
+ NMToken is used to authenticate.]]>
+ NodeManager]]>
+ NodeManager
+ It is issued by ResourceMananger when ApplicationMaster
+ negotiates resource with ResourceManager and
+ validated on NodeManager side.
+ A given Node can be mapped with any kind of attribute, few examples are + HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS. +
+
+ Its not compulsory for all the attributes to have value, empty string is the
+ default value of the NodeAttributeType.STRING
+
+ Node Attribute Prefix is used as namespace to segregate the attributes. +
]]> ++ Node Attribute Prefix is used as namespace to segregate the attributes. +
]]> +node Attribute.
+
+ Based on this attribute expressions and values will be evaluated.]]>
+ NodeId is the unique identifier for a node.
+
+ It includes the hostname and port to uniquely
+ identify the node. Thus, it is unique across restarts of any
+ NodeManager.
NodeId of the node]]>
+ NodeState of the node]]>
+ Resource on the node.
+ @return used Resource on the node]]>
+ Resource on the node.
+ @return total Resource on the node]]>
+ Node.]]>
+ ResourceManager.
+ @see AllocateRequest#setAskList(List)]]>
+ ResourceManager. If the AM prefers a different set of
+ containers, then it may checkpoint or kill containers matching the
+ description in {@link #getResourceRequest}.
+ @return Set of containers at risk if the contract is not met.]]>
+ ApplicationMaster (AM) can satisfy this request according
+ to its own priorities to prevent containers from being forcibly killed by
+ the platform.
+ @see PreemptionMessage]]>
+ + In contrast, the {@link PreemptionContract} also includes a description of + resources with a set of containers. If the AM releases containers matching + that profile, then the containers enumerated in {@link + PreemptionContract#getContainers()} may not be killed. +
+ Each preemption message reflects the RM's current understanding of the + cluster state, so a request to return N containers may not + reflect containers the AM is releasing, recently exited containers the RM has + yet to learn about, or new containers allocated before the message was + generated. Conversely, an RM may request a different profile of containers in + subsequent requests. +
+ The policy enforced by the RM is part of the scheduler. Generally, only + containers that have been requested consistently should be killed, but the + details are not specified.]]> +
QueueState of the queue]]>
+ accessible node labels of the queue]]>
+ ApplicationSubmissionContext and
+ ResourceRequest don't specify their
+ NodeLabelExpression.
+
+ @return default node label expression of the queue]]>
+ queue stats of the queue]]>
+ QueueACL for the given user]]>
+ QueueUserACLInfo provides information {@link QueueACL} for
+ the given user.
+
+ @see QueueACL
+ @see ApplicationClientProtocol#getQueueUserAcls(org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest)]]>
+ + The globally unique nature of the identifier is achieved by using the + cluster timestamp i.e. start-time of the {@code ResourceManager} + along with a monotonically increasing counter for the reservation. +
]]> +Resource models a set of computer resources in the
+ cluster.
+
+ Currently it models both memory and CPU.
+ +The unit for memory is megabytes. CPU is modeled with virtual cores + (vcores), a unit for expressing parallelism. A node's capacity should + be configured with virtual cores equal to its number of physical cores. A + container should be requested with the number of cores it can saturate, i.e. + the average number of threads it expects to have runnable at a time.
+ +Virtual cores take integer values and thus currently CPU-scheduling is + very coarse. A complementary axis for CPU requests that represents + processing power will likely be added in the future to enable finer-grained + resource configuration.
+ +Typically, applications request Resource of suitable
+ capability to run their component tasks.
Priority of the request]]>
+ Priority of the request]]>
+ ResourceRequest.]]>
+ ResourceRequest.]]>
+ + +
If the flag is off on a rack-level ResourceRequest,
+ containers at that request's priority will not be assigned to nodes on that
+ request's rack unless requests specifically for those nodes have also been
+ submitted.
+ +
If the flag is off on an {@link ResourceRequest#ANY}-level
+ ResourceRequest, containers at that request's priority will
+ only be assigned on racks for which specific requests have also been
+ submitted.
+ +
For example, to request a container strictly on a specific node, the + corresponding rack-level and any-level requests should have locality + relaxation set to false. Similarly, to request a container strictly on a + specific rack, the corresponding any-level request should have locality + relaxation set to false.
+
+ @param relaxLocality whether locality relaxation is enabled with this
+ ResourceRequest.]]>
+
+ The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. + + @return the ID corresponding to this allocation request.]]> +
+ The scheduler may return multiple {@code AllocateResponse}s corresponding + to the same ID as and when scheduler allocates {@code Container}(s). + Applications can continue to completely ignore the returned ID in + the response and use the allocation for any of their outstanding requests. +
+ If one wishes to replace an entire {@code ResourceRequest} corresponding to + a specific ID, they can simply cancel the corresponding {@code + ResourceRequest} and submit a new one afresh. +
+ If the ID is not set, scheduler will continue to work as previously and all + allocated {@code Container}(s) will have the default ID, -1. + + @param allocationRequestID the ID corresponding to this allocation + request.]]> +
Resource capability of the request]]>
+ Resource capability of the request]]>
+ priority of the request
+ @return {@link ResourceRequestBuilder}]]>
+ resourceName of the request
+ @return {@link ResourceRequestBuilder}]]>
+ capability of the request
+ @return {@link ResourceRequestBuilder}]]>
+ numContainers of the request
+ @return {@link ResourceRequestBuilder}]]>
+ relaxLocality of the request
+ @return {@link ResourceRequestBuilder}]]>
+ nodeLabelExpression of the request
+ @return {@link ResourceRequestBuilder}]]>
+ executionTypeRequest of the request
+ @return {@link ResourceRequestBuilder}]]>
+ executionType of the request.
+ @return {@link ResourceRequestBuilder}]]>
+ allocationRequestId of the request
+ @return {@link ResourceRequestBuilder}]]>
+ ResourceUtilization models the utilization of a set of computer
+ resources in the cluster.
+ ]]>
+ allocationRequestId of the
+ request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+ priority of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ @see SchedulingRequest#setPriority(Priority)]]>
+ executionType of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+ allocationsTags of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+ resourceSizing of the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+ placementConstraints of
+ the request
+ @return {@link SchedulingRequest.SchedulingRequestBuilder}]]>
+ ResourceManager.
+ @return the set of {@link ContainerId} to be preempted.]]>
+ Token is the security entity used by the framework
+ to verify authenticity of any resource.]]>
+ ContainerId of the container]]>
+ ContainerUpdateType of the container.]]>
+ ContainerId of the container]]>
+ ContainerId of the container]]>
+ ExecutionType of the container]]>
+ ExecutionType of the container]]>
+ Resource capability of the request]]>
+ Resource capability of the request]]>
+ URL represents a serializable {@link java.net.URL}.]]>
+ NodeManagers in the cluster]]>
+ DecommissionedNodeManagers in the cluster]]>
+ ActiveNodeManagers in the cluster]]>
+ LostNodeManagers in the cluster]]>
+ UnhealthyNodeManagers in the cluster]]>
+ RebootedNodeManagers in the cluster]]>
+ YarnClusterMetrics represents cluster metrics.
+
+ Currently only number of NodeManagers is provided.
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what AccessControlList takes.
+
+ Primary filters will be used to index the entities in
+ TimelineStore, such that users should carefully choose the
+ information they want to store as the primary filters. The remaining can be
+ stored as other information.
+
TimelineEntityGroupId.
+
+ @return ApplicationId of the
+ TimelineEntityGroupId]]>
+ timelineEntityGroupId]]>
+ TimelineEntityGroupId is an abstract way for
+ timeline service users to represent #a group of related timeline data.
+ For example, all entities that represents one data flow DAG execution
+ can be grouped into one timeline entity group. ]]>
+
+ The reader and writer users/groups pattern that the user can supply is the
+ same as what AccessControlList takes.
+
+ It is usually used in the case where we want to recover class polymorphism + after deserializing the entity from its JSON form. +
+ @param entity the real entity that carries information]]> ++ Users can use {@link TimelineServiceHelper#invertLong(long)} to invert + the prefix if necessary. + + @param entityIdPrefix prefix for an entity.]]> ++ TimelineEntity entity = new TimelineEntity(); + entity.setIdPrefix(value); +
InetSocketAddress. On an HA cluster,
+ this fetches the address corresponding to the RM identified by
+ {@link #RM_HA_ID}.
+ @param name property name.
+ @param defaultAddress the default value
+ @param defaultPort the default port
+ @return InetSocketAddress]]>
+ + Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for + cross-platform practice i.e. submit an application from a Windows client to + a Linux/Unix server or vice versa. +
]]> +SharedCacheManager to run a cleaner task
+ @return SharedCacheManager returns an empty response
+ on success and throws an exception on rejecting the request
+ @throws YarnException
+ @throws IOException]]>
+ SharedCacheManager
+ ]]>
+ foo(3)
+ + Optional when using NodeAttribute Constraint. + + and where Pn can be any form of a valid constraint expression, + such as: + +YARN verifies access to the application, queue
+ etc. before accepting the request.
+
+ If the user does not have VIEW_APP access then the following
+ fields in the report will be set to stubbed values:
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ In secure mode, YARN verifies access to the application, queue
+ etc. before accepting the request.
+
+ In secure mode, YARN verifies access to the application, queue
+ etc. before accepting the request.
+
ResourceManager. New containers assigned to the master are
+ retrieved. Status of completed containers and node health updates are also
+ retrieved. This also doubles up as a heartbeat to the ResourceManager and
+ must be made periodically. The call may not always return any new
+ allocations of containers. App should not make concurrent allocate
+ requests. May cause request loss.
+
+ + Note : If the user has not removed container requests that have already + been satisfied, then the re-register may end up sending the entire + container requests to the RM (including matched requests). Which would mean + the RM could end up giving it a lot of new allocated containers. +
+ + @param progressIndicator Indicates progress made by the master + @return the response of the allocate request + @throws YarnException + @throws IOException]]> +Containers that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+ allocation or resource change
+ @param capability The target resource capability of the container]]>
+ UpdateContainerRequest.]]>
+ addContainerRequest earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ Resource size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @return Collection of request matching the parameters]]>
+ addContainerRequest earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+ Each collection in the list contains requests with identical
+ Resource size that fit in the given capability. In a
+ collection, requests will be returned in the same order as they were added.
+ specify an ExecutionType.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITHOUT the allocationRequestId being set.
+
+ @param priority Priority
+ @param resourceName Location
+ @param executionType ExecutionType
+ @param capability Capability
+ @return Collection of request matching the parameters]]>
+ addContainerRequest earlier in the lifecycle. For performance,
+ the AMRMClient may return its internal collection directly without creating
+ a copy. Users should not perform mutable operations on the return value.
+
+ NOTE: This API only matches Container requests that were created by the
+ client WITH the allocationRequestId being set to a non-default value.
+
+ @param allocationRequestId Allocation Request Id
+ @return Collection of request matching the parameters]]>
+ AMRMClient
+ + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> +
AMRMClient.
+ + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache.]]> +
checkEveryMillis ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call check]]>
+ checkEveryMillis ms. In the main loop, this method will log
+ the message "waiting in main loop" for each logInterval times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call check
+ @param logInterval interval to log for each]]>
+ The ApplicationMaster or other applications that use the
+ client must provide the details of the allocated container, including the
+ Id, the assigned node's Id and the token via {@link Container}. In
+ addition, the AM needs to provide the {@link ContainerLaunchContext} as
+ well.
NodeManager to launch the
+ container
+ @return a map between the auxiliary service names and their outputs
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+ The ApplicationMaster or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+
The ApplicationMaster or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+
NodeManager
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+ NodeManager
+
+ @return the status of a container.
+
+ @throws YarnException YarnException.
+ @throws IOException IOException.]]>
+ NMClient
+ + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @param nmTokenCache the NM token cache to use.]]> +
NMClient
+ + If a NM token cache is not set, the {@link NMTokenCache#getSingleton()} + singleton instance will be used. + + @return the NM token cache]]> +
+ NMTokenCache nmTokenCache = new NMTokenCache(); + AMRMClient rmClient = AMRMClient.createAMRMClient(); + NMClient nmClient = NMClient.createNMClient(); + nmClient.setNMTokenCache(nmTokenCache); + ... ++
+ NMTokenCache nmTokenCache = new NMTokenCache();
+ AMRMClient rmClient = AMRMClient.createAMRMClient();
+ NMClient nmClient = NMClient.createNMClient();
+ nmClient.setNMTokenCache(nmTokenCache);
+ AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+ NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+ ...
+
+
+ NMTokenCache nmTokenCache = new NMTokenCache();
+ ...
+ ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+ ...
+ AllocateRequest allocateRequest = ...
+ ...
+ AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+ for (NMToken token : allocateResponse.getNMTokens()) {
+ nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+ }
+ ...
+ ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+ ...
+ nmPro.startContainer(container, containerContext);
+ ...
+
+ SharedCacheManager.
+ The client uses a checksum to identify the resource and an
+ {@link ApplicationId} to identify which application will be using the
+ resource.
+
+
+
+ The SharedCacheManager responds with whether or not the
+ resource exists in the cache. If the resource exists, a URL to
+ the resource in the shared cache is returned. If the resource does not
+ exist, null is returned instead.
+
+ Once a URL has been returned for a resource, that URL is safe to use for + the lifetime of the application that corresponds to the provided + ApplicationId. +
+ + @param applicationId ApplicationId of the application using the resource + @param resourceKey the key (i.e. checksum) that identifies the resource + @return URL to the resource, or null if it does not exist]]> +SharedCacheManager.
+ This method is called once an application is no longer using a claimed
+ resource in the shared cache. The client uses a checksum to identify the
+ resource and an {@link ApplicationId} to identify which application is
+ releasing the resource.
+
+
+ + Note: This method is an optimization and the client is not required to call + it for correctness. +
+ + @param applicationId ApplicationId of the application releasing the + resource + @param resourceKey the key (i.e. checksum) that identifies the resource]]> +YARN. It is a blocking call - it
+ will not return {@link ApplicationId} until the submitted application is
+ submitted successfully and accepted by the ResourceManager.
+
+
+ + Users should provide an {@link ApplicationId} as part of the parameter + {@link ApplicationSubmissionContext} when submitting a new application, + otherwise it will throw the {@link ApplicationIdNotProvidedException}. +
+ +This internally calls {@link ApplicationClientProtocol#submitApplication + (SubmitApplicationRequest)}, and after that, it internally invokes + {@link ApplicationClientProtocol#getApplicationReport + (GetApplicationReportRequest)} and waits till it can make sure that the + application gets properly submitted. If RM fails over or RM restart + happens before ResourceManager saves the application's state, + {@link ApplicationClientProtocol + #getApplicationReport(GetApplicationReportRequest)} will throw + the {@link ApplicationNotFoundException}. This API automatically resubmits + the application with the same {@link ApplicationSubmissionContext} when it + catches the {@link ApplicationNotFoundException}
+ + @param appContext + {@link ApplicationSubmissionContext} containing all the details + needed to submit a new application + @return {@link ApplicationId} of the accepted application + @throws YarnException + @throws IOException + @see #createApplication()]]> +
+ In secure mode, YARN verifies access to the application, queue
+ etc. before accepting the request.
+
+ If the user does not have VIEW_APP access then the following
+ fields in the report will be set to stubbed values:
+
+ The AMRM token will be returned only if all the following conditions are + met: +
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ If the user does not have VIEW_APP access for an application
+ then the corresponding report will be filtered as described in
+ {@link #getApplicationReport(ApplicationId)}.
+
+ In secure mode, YARN verifies access to the application, queue
+ etc. before accepting the request.
+
+ In secure mode, YARN verifies access to the application, queue
+ etc. before accepting the request.
+
+ The client packages all details of its request in a + {@link ReservationSubmissionRequest} object. This contains information + about the amount of capacity, temporal constraints, and gang needs. + Furthermore, the reservation might be composed of multiple stages, with + ordering dependencies among them. +
+ ++ In order to respond, a new admission control component in the + {@code ResourceManager} performs an analysis of the resources that have + been committed over the period of time the user is requesting, verify that + the user requests can be fulfilled, and that it respect a sharing policy + (e.g., {@code CapacityOverTimePolicy}). Once it has positively determined + that the ReservationRequest is satisfiable the {@code ResourceManager} + answers with a {@link ReservationSubmissionResponse} that includes a + {@link ReservationId}. Upon failure to find a valid allocation the response + is an exception with the message detailing the reason of failure. +
+ ++ The semantics guarantees that the {@link ReservationId} returned, + corresponds to a valid reservation existing in the time-range request by + the user. The amount of capacity dedicated to such reservation can vary + overtime, depending of the allocation that has been determined. But it is + guaranteed to satisfy all the constraint expressed by the user in the + {@link ReservationDefinition} +
+ + @param request request to submit a new Reservation + @return response contains the {@link ReservationId} on accepting the + submission + @throws YarnException if the reservation cannot be created successfully + @throws IOException]]> ++ The allocation is attempted by virtually substituting all previous + allocations related to this Reservation with new ones, that satisfy the new + {@link ReservationDefinition}. Upon success the previous allocation is + atomically substituted by the new one, and on failure (i.e., if the system + cannot find a valid allocation for the updated request), the previous + allocation remains valid. +
+ + @param request to update an existing Reservation (the + {@link ReservationUpdateRequest} should refer to an existing valid + {@link ReservationId}) + @return response empty on successfully updating the existing reservation + @throws YarnException if the request is invalid or reservation cannot be + updated successfully + @throws IOException]]> +ResourceManager]]>
+ ResourceManager]]>
+ Containers that are returned from previous successful
+ allocations or resource changes. By passing in the existing container and a
+ target resource capability to this method, the application requests the
+ ResourceManager to change the existing resource allocation to the target
+ resource allocation.
+
+ @deprecated use
+ {@link #requestContainerUpdate(Container, UpdateContainerRequest)}
+
+ @param container The container returned from the last successful resource
+ allocation or resource change
+ @param capability The target resource capability of the container]]>
+ UpdateContainerRequest.]]>
+ checkEveryMillis ms.
+ See also {@link #waitFor(java.util.function.Supplier, int, int)}
+ @param check user defined checker
+ @param checkEveryMillis interval to call check]]>
+ checkEveryMillis ms. In the main loop, this method will log
+ the message "waiting in main loop" for each logInterval times
+ iteration to confirm the thread is alive.
+ @param check user defined checker
+ @param checkEveryMillis interval to call check
+ @param logInterval interval to log for each]]>
+
+ {@code
+ class MyCallbackHandler extends AMRMClientAsync.AbstractCallbackHandler {
+ public void onContainersAllocated(List containers) {
+ [run tasks on the containers]
+ }
+
+ public void onContainersUpdated(List containers) {
+ [determine if resource allocation of containers have been increased in
+ the ResourceManager, and if so, inform the NodeManagers to increase the
+ resource monitor/enforcement on the containers]
+ }
+
+ public void onContainersCompleted(List statuses) {
+ [update progress, check whether app is done]
+ }
+
+ public void onNodesUpdated(List updated) {}
+
+ public void onReboot() {}
+ }
+ }
+
+
+ The client's lifecycle should be managed similarly to the following:
+
+
+ {@code
+ AMRMClientAsync asyncClient =
+ createAMRMClientAsync(appAttId, 1000, new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ RegisterApplicationMasterResponse response = asyncClient
+ .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
+ appMasterTrackingUrl);
+ asyncClient.addContainerRequest(containerRequest);
+ [... wait for application to complete]
+ asyncClient.unregisterApplicationMaster(status, appMsg, trackingUrl);
+ asyncClient.stop();
+ }
+ ]]>
+ The ApplicationMaster or other applications that use the
+ client must provide the details of the container, including the Id and
+ the target resource encapsulated in the updated container token via
+ {@link Container}.
+
+ {@code
+ class MyCallbackHandler extends NMClientAsync.AbstractCallbackHandler {
+ public void onContainerStarted(ContainerId containerId,
+ Map allServiceResponse) {
+ [post process after the container is started, process the response]
+ }
+
+ public void onContainerResourceIncreased(ContainerId containerId,
+ Resource resource) {
+ [post process after the container resource is increased]
+ }
+
+ public void onContainerStatusReceived(ContainerId containerId,
+ ContainerStatus containerStatus) {
+ [make use of the status of the container]
+ }
+
+ public void onContainerStopped(ContainerId containerId) {
+ [post process after the container is stopped]
+ }
+
+ public void onStartContainerError(
+ ContainerId containerId, Throwable t) {
+ [handle the raised exception]
+ }
+
+ public void onGetContainerStatusError(
+ ContainerId containerId, Throwable t) {
+ [handle the raised exception]
+ }
+
+ public void onStopContainerError(
+ ContainerId containerId, Throwable t) {
+ [handle the raised exception]
+ }
+ }
+ }
+
+
+ The client's life-cycle should be managed like the following:
+
+
+ {@code
+ NMClientAsync asyncClient =
+ NMClientAsync.createNMClientAsync(new MyCallbackhandler());
+ asyncClient.init(conf);
+ asyncClient.start();
+ asyncClient.startContainer(container, containerLaunchContext);
+ [... wait for container being started]
+ asyncClient.getContainerStatus(container.getId(), container.getNodeId(),
+ container.getContainerToken());
+ [... handle the status in the callback instance]
+ asyncClient.stopContainer(container.getId(), container.getNodeId(),
+ container.getContainerToken());
+ [... wait for container being stopped]
+ asyncClient.stop();
+ }
+ ]]>
+ yarn.sharedcache.checksum.algo.impl)
+
+ @return SharedCacheChecksum object]]>
+ + Note that header and ellipses are not counted against {@link #limit}. +
+ An example: + +
+ {@code
+ // At the beginning it's an empty string
+ final Appendable shortAppender = new BoundedAppender(80);
+ // The whole message fits into limit
+ shortAppender.append(
+ "message1 this is a very long message but fitting into limit\n");
+ // The first message is truncated, the second not
+ shortAppender.append("message2 this is shorter than the previous one\n");
+ // The first message is deleted, the second truncated, the third
+ // preserved
+ shortAppender.append("message3 this is even shorter message, maybe.\n");
+ // The first two are deleted, the third one truncated, the last preserved
+ shortAppender.append("message4 the shortest one, yet the greatest :)");
+ // Current contents are like this:
+ // Diagnostic messages truncated, showing last 80 chars out of 199:
+ // ...s is even shorter message, maybe.
+ // message4 the shortest one, yet the greatest :)
+ }
+
+ + Note that null values are {@link #append(CharSequence) append}ed + just like in {@link StringBuilder#append(CharSequence) original + implementation}. +
+ Note that this class is not thread safe.]]> +
false]]>
+ NodeId to ResourceOption.
* @return the map of {@code + * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.federation;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Unit tests for TestFederationStateStoreServiceMetrics.
+ */
+public class TestFederationStateStoreServiceMetrics {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationStateStoreServiceMetrics.class);
+
+ private static FederationStateStoreServiceMetrics metrics =
+ FederationStateStoreServiceMetrics.getMetrics();
+
+ private MockBadFederationStateStoreService badStateStore =
+ new MockBadFederationStateStoreService();
+ private MockGoodFederationStateStoreService goodStateStore =
+ new MockGoodFederationStateStoreService();
+
+ // Records failures for all calls
+ private class MockBadFederationStateStoreService {
+ public void registerSubCluster() {
+ LOG.info("Mocked: failed registerSubCluster call");
+ FederationStateStoreServiceMetrics.failedStateStoreServiceCall();
+ }
+ }
+
+ // Records successes for all calls
+ private class MockGoodFederationStateStoreService {
+ public void registerSubCluster(long duration) {
+ LOG.info("Mocked: successful registerSubCluster call with duration {}", duration);
+ FederationStateStoreServiceMetrics.succeededStateStoreServiceCall(duration);
+ }
+ }
+
+ @Test
+ public void testFederationStateStoreServiceMetricInit() {
+ LOG.info("Test: aggregate metrics are initialized correctly");
+ assertEquals(0, FederationStateStoreServiceMetrics.getNumSucceededCalls());
+ assertEquals(0, FederationStateStoreServiceMetrics.getNumFailedCalls());
+ LOG.info("Test: aggregate metrics are updated correctly");
+ }
+
+ @Test
+ public void testRegisterSubClusterSuccessfulCalls() {
+ LOG.info("Test: Aggregate and method successful calls updated correctly.");
+
+ long totalGoodBefore = FederationStateStoreServiceMetrics.getNumSucceededCalls();
+ long apiGoodBefore = FederationStateStoreServiceMetrics.
+ getNumSucceessfulCallsForMethod("registerSubCluster");
+
+ // Call the registerSubCluster method
+ goodStateStore.registerSubCluster(100);
+
+ assertEquals(totalGoodBefore + 1,
+ FederationStateStoreServiceMetrics.getNumSucceededCalls());
+ assertEquals(100, FederationStateStoreServiceMetrics.getLatencySucceededCalls(), 0);
+ assertEquals(apiGoodBefore + 1,
+ FederationStateStoreServiceMetrics.getNumSucceededCalls());
+ double latencySucceessfulCalls =
+ FederationStateStoreServiceMetrics.getLatencySucceessfulCallsForMethod(
+ "registerSubCluster");
+ assertEquals(100, latencySucceessfulCalls, 0);
+
+ LOG.info("Test: Running stats correctly calculated for 2 metrics");
+
+ // Call the registerSubCluster method
+ goodStateStore.registerSubCluster(200);
+
+ assertEquals(totalGoodBefore + 2,
+ FederationStateStoreServiceMetrics.getNumSucceededCalls());
+ assertEquals(150, FederationStateStoreServiceMetrics.getLatencySucceededCalls(), 0);
+ assertEquals(apiGoodBefore + 2,
+ FederationStateStoreServiceMetrics.getNumSucceededCalls());
+ double latencySucceessfulCalls2 =
+ FederationStateStoreServiceMetrics.getLatencySucceessfulCallsForMethod(
+ "registerSubCluster");
+ assertEquals(150, latencySucceessfulCalls2, 0);
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 037312b716a..47db9565af1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -1251,6 +1251,17 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertNull("root.e.e1-auto should have been removed", eAuto);
}
+ @Test()
+ public void testAutoCreateInvalidParent() throws Exception {
+ startScheduler();
+ Assert.assertThrows(SchedulerDynamicEditException.class,
+ () -> createQueue("invalid.queue"));
+ Assert.assertThrows(SchedulerDynamicEditException.class,
+ () -> createQueue("invalid.queue.longer"));
+ Assert.assertThrows(SchedulerDynamicEditException.class,
+ () -> createQueue("invalidQueue"));
+ }
+
protected AbstractLeafQueue createQueue(String queuePath) throws YarnException,
IOException {
return autoQueueHandler.createQueue(new QueuePath(queuePath));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
index f3b428dab4a..bb814b65283 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
@@ -30,7 +30,7 @@ import java.net.InetAddress;
* Manages Router audit logs.
* Audit log format is written as key=value pairs. Tab separated.
*/
-public class RouterAuditLogger {
+public final class RouterAuditLogger {
private static final Logger LOG =
LoggerFactory.getLogger(RouterAuditLogger.class);
@@ -51,6 +51,43 @@ public class RouterAuditLogger {
public static final String GET_APP_REPORT = "Get Application Report";
public static final String TARGET_CLIENT_RM_SERVICE = "RouterClientRMService";
public static final String UNKNOWN = "UNKNOWN";
+ public static final String GET_APPLICATIONS = "Get Applications";
+ public static final String GET_CLUSTERMETRICS = "Get ClusterMetrics";
+ public static final String GET_CLUSTERNODES = "Get ClusterNodes";
+ public static final String GET_QUEUEINFO = "Get QueueInfo";
+ public static final String GET_QUEUE_USER_ACLS = "Get QueueUserAcls";
+ public static final String MOVE_APPLICATION_ACROSS_QUEUES = "Move ApplicationAcrossQueues";
+ public static final String GET_NEW_RESERVATION = "Get NewReservation";
+ public static final String SUBMIT_RESERVATION = "Submit Reservation";
+ public static final String LIST_RESERVATIONS = "List Reservations";
+ public static final String UPDATE_RESERVATION = "Update Reservation";
+ public static final String DELETE_RESERVATION = "Delete Reservation";
+ public static final String GET_NODETOLABELS = "Get NodeToLabels";
+ public static final String GET_LABELSTONODES = "Get LabelsToNodes";
+ public static final String GET_CLUSTERNODELABELS = "Get ClusterNodeLabels";
+ public static final String GET_APPLICATION_ATTEMPT_REPORT = "Get ApplicationAttemptReport";
+ public static final String GET_APPLICATION_ATTEMPTS = "Get ApplicationAttempts";
+ public static final String GET_CONTAINERREPORT = "Get ContainerReport";
+ public static final String GET_CONTAINERS = "Get Containers";
+ public static final String GET_DELEGATIONTOKEN = "Get DelegationToken";
+ public static final String RENEW_DELEGATIONTOKEN = "Renew DelegationToken";
+ public static final String CANCEL_DELEGATIONTOKEN = "Cancel DelegationToken";
+ public static final String FAIL_APPLICATIONATTEMPT = "Fail ApplicationAttempt";
+ public static final String UPDATE_APPLICATIONPRIORITY = "Update ApplicationPriority";
+ public static final String SIGNAL_TOCONTAINER = "Signal ToContainer";
+ public static final String UPDATE_APPLICATIONTIMEOUTS = "Update ApplicationTimeouts";
+ public static final String GET_RESOURCEPROFILES = "Get ResourceProfiles";
+ public static final String GET_RESOURCEPROFILE = "Get ResourceProfile";
+ public static final String GET_RESOURCETYPEINFO = "Get ResourceTypeInfo";
+ public static final String GET_ATTRIBUTESTONODES = "Get AttributesToNodes";
+ public static final String GET_CLUSTERNODEATTRIBUTES = "Get ClusterNodeAttributes";
+ public static final String GET_NODESTOATTRIBUTES = "Get NodesToAttributes";
+ }
+
+ public static void logSuccess(String user, String operation, String target) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info(createSuccessLog(user, operation, target, null, null));
+ }
}
/**
@@ -146,6 +183,28 @@ public class RouterAuditLogger {
}
}
+ /**
+ * Create a readable and parseable audit log string for a failed event.
+ *
+ * @param user User who made the service request.
+ * @param operation Operation requested by the user.
+ * @param perm Target permissions.
+ * @param target The target on which the operation is being performed.
+ * @param descriptionFormat the description message format string.
+ * @param args format parameter.
+ *
+ *
+ * Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
+ * delimiter and hence the value fields should not contains tabs ('\t').
+ */
+ public static void logFailure(String user, String operation, String perm,
+ String target, String descriptionFormat, Object... args) {
+ if (LOG.isInfoEnabled()) {
+ String description = String.format(descriptionFormat, args);
+ LOG.info(createFailureLog(user, operation, perm, target, description, null, null));
+ }
+ }
+
/**
* Create a readable and parseable audit log string for a failed event.
*
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
index fdcd890ea6e..a84a315b93c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
@@ -159,6 +159,18 @@ public final class RouterMetrics {
private MutableGaugeInt numAddToClusterNodeLabelsFailedRetrieved;
@Metric("# of removeFromClusterNodeLabels failed to be retrieved")
private MutableGaugeInt numRemoveFromClusterNodeLabelsFailedRetrieved;
+ @Metric("# of numUpdateSchedulerConfiguration failed to be retrieved")
+ private MutableGaugeInt numUpdateSchedulerConfigurationFailedRetrieved;
+ @Metric("# of numGetSchedulerConfiguration failed to be retrieved")
+ private MutableGaugeInt numGetSchedulerConfigurationFailedRetrieved;
+ @Metric("# of getClusterInfo failed to be retrieved")
+ private MutableGaugeInt numGetClusterInfoFailedRetrieved;
+ @Metric("# of getClusterUserInfo failed to be retrieved")
+ private MutableGaugeInt numGetClusterUserInfoFailedRetrieved;
+ @Metric("# of updateNodeResource failed to be retrieved")
+ private MutableGaugeInt numUpdateNodeResourceFailedRetrieved;
+ @Metric("# of refreshNodesResources failed to be retrieved")
+ private MutableGaugeInt numRefreshNodesResourcesFailedRetrieved;
// Aggregate metrics are shared, and don't have to be looked up per call
@Metric("Total number of successful Submitted apps and latency(ms)")
@@ -279,6 +291,18 @@ public final class RouterMetrics {
private MutableRate totalSucceededAddToClusterNodeLabelsRetrieved;
@Metric("Total number of successful Retrieved RemoveFromClusterNodeLabels and latency(ms)")
private MutableRate totalSucceededRemoveFromClusterNodeLabelsRetrieved;
+ @Metric("Total number of successful Retrieved updateSchedulerConfiguration and latency(ms)")
+ private MutableRate totalSucceededUpdateSchedulerConfigurationRetrieved;
+ @Metric("Total number of successful Retrieved getSchedulerConfiguration and latency(ms)")
+ private MutableRate totalSucceededGetSchedulerConfigurationRetrieved;
+ @Metric("Total number of successful Retrieved GetClusterInfoRetrieved and latency(ms)")
+ private MutableRate totalSucceededGetClusterInfoRetrieved;
+ @Metric("Total number of successful Retrieved GetClusterUserInfoRetrieved and latency(ms)")
+ private MutableRate totalSucceededGetClusterUserInfoRetrieved;
+ @Metric("Total number of successful Retrieved UpdateNodeResource and latency(ms)")
+ private MutableRate totalSucceededUpdateNodeResourceRetrieved;
+ @Metric("Total number of successful Retrieved RefreshNodesResources and latency(ms)")
+ private MutableRate totalSucceededRefreshNodesResourcesRetrieved;
/**
* Provide quantile counters for all latencies.
@@ -342,6 +366,12 @@ public final class RouterMetrics {
private MutableQuantiles replaceLabelsOnNodeLatency;
private MutableQuantiles addToClusterNodeLabelsLatency;
private MutableQuantiles removeFromClusterNodeLabelsLatency;
+ private MutableQuantiles updateSchedulerConfigLatency;
+ private MutableQuantiles getSchedulerConfigurationLatency;
+ private MutableQuantiles getClusterInfoLatency;
+ private MutableQuantiles getClusterUserInfoLatency;
+ private MutableQuantiles updateNodeResourceLatency;
+ private MutableQuantiles refreshNodesResourcesLatency;
private static volatile RouterMetrics instance = null;
private static MetricsRegistry registry;
@@ -551,6 +581,24 @@ public final class RouterMetrics {
removeFromClusterNodeLabelsLatency = registry.newQuantiles("removeFromClusterNodeLabelsLatency",
"latency of remove cluster nodelabels timeouts", "ops", "latency", 10);
+
+ updateSchedulerConfigLatency = registry.newQuantiles("updateSchedulerConfigurationLatency",
+ "latency of update scheduler configuration timeouts", "ops", "latency", 10);
+
+ getSchedulerConfigurationLatency = registry.newQuantiles("getSchedulerConfigurationLatency",
+ "latency of get scheduler configuration timeouts", "ops", "latency", 10);
+
+ getClusterInfoLatency = registry.newQuantiles("getClusterInfoLatency",
+ "latency of get cluster info timeouts", "ops", "latency", 10);
+
+ getClusterUserInfoLatency = registry.newQuantiles("getClusterUserInfoLatency",
+ "latency of get cluster user info timeouts", "ops", "latency", 10);
+
+ updateNodeResourceLatency = registry.newQuantiles("updateNodeResourceLatency",
+ "latency of update node resource timeouts", "ops", "latency", 10);
+
+ refreshNodesResourcesLatency = registry.newQuantiles("refreshNodesResourcesLatency",
+ "latency of refresh nodes resources timeouts", "ops", "latency", 10);
}
public static RouterMetrics getMetrics() {
@@ -847,6 +895,36 @@ public final class RouterMetrics {
return totalSucceededRemoveFromClusterNodeLabelsRetrieved.lastStat().numSamples();
}
+ @VisibleForTesting
+ public long getNumSucceededUpdateSchedulerConfigurationRetrieved() {
+ return totalSucceededUpdateSchedulerConfigurationRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededGetSchedulerConfigurationRetrieved() {
+ return totalSucceededGetSchedulerConfigurationRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededGetClusterInfoRetrieved() {
+ return totalSucceededGetClusterInfoRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededGetClusterUserInfoRetrieved() {
+ return totalSucceededGetClusterUserInfoRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededUpdateNodeResourceRetrieved() {
+ return totalSucceededUpdateNodeResourceRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededRefreshNodesResourcesRetrieved() {
+ return totalSucceededRefreshNodesResourcesRetrieved.lastStat().numSamples();
+ }
+
@VisibleForTesting
public long getNumSucceededRefreshSuperUserGroupsConfigurationRetrieved() {
return totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.lastStat().numSamples();
@@ -1137,6 +1215,36 @@ public final class RouterMetrics {
return totalSucceededRemoveFromClusterNodeLabelsRetrieved.lastStat().mean();
}
+ @VisibleForTesting
+ public double getLatencySucceededUpdateSchedulerConfigurationRetrieved() {
+ return totalSucceededUpdateSchedulerConfigurationRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededGetSchedulerConfigurationRetrieved() {
+ return totalSucceededGetSchedulerConfigurationRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededGetClusterInfoRetrieved() {
+ return totalSucceededGetClusterInfoRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededGetClusterUserInfoRetrieved() {
+ return totalSucceededGetClusterUserInfoRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededUpdateNodeResourceRetrieved() {
+ return totalSucceededUpdateNodeResourceRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededRefreshNodesResourcesRetrieved() {
+ return totalSucceededRefreshNodesResourcesRetrieved.lastStat().mean();
+ }
+
@VisibleForTesting
public double getLatencySucceededRefreshSuperUserGroupsConfigurationRetrieved() {
return totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.lastStat().mean();
@@ -1382,6 +1490,30 @@ public final class RouterMetrics {
return numRemoveFromClusterNodeLabelsFailedRetrieved.value();
}
+ public int getUpdateSchedulerConfigurationFailedRetrieved() {
+ return numUpdateSchedulerConfigurationFailedRetrieved.value();
+ }
+
+ public int getSchedulerConfigurationFailedRetrieved() {
+ return numGetSchedulerConfigurationFailedRetrieved.value();
+ }
+
+ public int getClusterInfoFailedRetrieved() {
+ return numGetClusterInfoFailedRetrieved.value();
+ }
+
+ public int getClusterUserInfoFailedRetrieved() {
+ return numGetClusterUserInfoFailedRetrieved.value();
+ }
+
+ public int getUpdateNodeResourceFailedRetrieved() {
+ return numUpdateNodeResourceFailedRetrieved.value();
+ }
+
+ public int getRefreshNodesResourcesFailedRetrieved() {
+ return numRefreshNodesResourcesFailedRetrieved.value();
+ }
+
public int getDelegationTokenFailedRetrieved() {
return numGetDelegationTokenFailedRetrieved.value();
}
@@ -1685,6 +1817,36 @@ public final class RouterMetrics {
removeFromClusterNodeLabelsLatency.add(duration);
}
+ public void succeededUpdateSchedulerConfigurationRetrieved(long duration) {
+ totalSucceededUpdateSchedulerConfigurationRetrieved.add(duration);
+ updateSchedulerConfigLatency.add(duration);
+ }
+
+ public void succeededGetSchedulerConfigurationRetrieved(long duration) {
+ totalSucceededGetSchedulerConfigurationRetrieved.add(duration);
+ getSchedulerConfigurationLatency.add(duration);
+ }
+
+ public void succeededGetClusterInfoRetrieved(long duration) {
+ totalSucceededGetClusterInfoRetrieved.add(duration);
+ getClusterInfoLatency.add(duration);
+ }
+
+ public void succeededGetClusterUserInfoRetrieved(long duration) {
+ totalSucceededGetClusterUserInfoRetrieved.add(duration);
+ getClusterUserInfoLatency.add(duration);
+ }
+
+ public void succeededUpdateNodeResourceRetrieved(long duration) {
+ totalSucceededUpdateNodeResourceRetrieved.add(duration);
+ updateNodeResourceLatency.add(duration);
+ }
+
+ public void succeededRefreshNodesResourcesRetrieved(long duration) {
+ totalSucceededRefreshNodesResourcesRetrieved.add(duration);
+ refreshNodesResourcesLatency.add(duration);
+ }
+
public void succeededRefreshSuperUserGroupsConfRetrieved(long duration) {
totalSucceededRefreshSuperUserGroupsConfigurationRetrieved.add(duration);
refreshSuperUserGroupsConfLatency.add(duration);
@@ -1905,6 +2067,30 @@ public final class RouterMetrics {
numRemoveFromClusterNodeLabelsFailedRetrieved.incr();
}
+ public void incrUpdateSchedulerConfigurationFailedRetrieved() {
+ numUpdateSchedulerConfigurationFailedRetrieved.incr();
+ }
+
+ public void incrGetSchedulerConfigurationFailedRetrieved() {
+ numGetSchedulerConfigurationFailedRetrieved.incr();
+ }
+
+ public void incrGetClusterInfoFailedRetrieved() {
+ numGetClusterInfoFailedRetrieved.incr();
+ }
+
+ public void incrGetClusterUserInfoFailedRetrieved() {
+ numGetClusterUserInfoFailedRetrieved.incr();
+ }
+
+ public void incrUpdateNodeResourceFailedRetrieved() {
+ numUpdateNodeResourceFailedRetrieved.incr();
+ }
+
+ public void incrRefreshNodesResourcesFailedRetrieved() {
+ numRefreshNodesResourcesFailedRetrieved.incr();
+ }
+
public void incrGetDelegationTokenFailedRetrieved() {
numGetDelegationTokenFailedRetrieved.incr();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
index 8fa6ca2f055..dcd7777779b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
@@ -18,13 +18,16 @@
package org.apache.hadoop.yarn.server.router;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.thirdparty.protobuf.GeneratedMessageV3;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ReservationRequest;
@@ -32,10 +35,18 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.StringLocalResourceMapProto;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -43,6 +54,8 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectOutputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -118,6 +131,19 @@ public final class RouterServerUtil {
}
}
+ /**
+ * Throws an exception due to an error.
+ *
+ * @param errMsg the error message
+ * @throws YarnException on failure
+ */
+ @Public
+ @Unstable
+ public static void logAndThrowException(String errMsg) throws YarnException {
+ LOG.error(errMsg);
+ throw new YarnException(errMsg);
+ }
+
public static