Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/trunk' into HADOOP-17725-trunk

Browse Source
This commit is contained in:
Carl Levasseur 2023-03-10 11:01:02 +01:00
commit b63e263532
2863 changed files with 487437 additions and 42615 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
.BUILDING.txt.swp
View File

Binary file not shown.

4
.asf.yaml
View File

@ -14,6 +14,8 @@
# limitations under the License.
github:
ghp_path: /
ghp_branch: gh-pages
enabled_merge_buttons:
squash: true
merge: false
@ -22,4 +24,4 @@ notifications:
commits: common-commits@hadoop.apache.org
issues: common-issues@hadoop.apache.org
pullrequests: common-issues@hadoop.apache.org
jira_options: link label worklog
jira_options: comment link label

59
.github/workflows/website.yml vendored Normal file
View File

@ -0,0 +1,59 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: website
# Controls when the action will run.
on:
push:
branches: [ trunk ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout Hadoop trunk
uses: actions/checkout@v3
with:
repository: apache/hadoop
- name: Set up JDK 8
uses: actions/setup-java@v3
with:
java-version: '8'
distribution: 'temurin'
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Build Hadoop maven plugins
run: cd hadoop-maven-plugins && mvn --batch-mode install
- name: Build Hadoop
run: mvn clean install -DskipTests -DskipShade
- name: Build document
run: mvn clean site
- name: Stage document
run: mvn site:stage -DstagingDirectory=${GITHUB_WORKSPACE}/staging/
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./staging/hadoop-project
user_name: 'github-actions[bot]'
user_email: 'github-actions[bot]@users.noreply.github.com'

18
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists-gtest.txt.in → .yetus/excludes.txt
View File

@ -1,4 +1,3 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@ -14,20 +13,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
project(googletest-download NONE)
include(ExternalProject)
ExternalProject_Add(googletest
GIT_REPOSITORY https://github.com/google/googletest.git
GIT_TAG release-1.10.0
SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src"
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)
dev-support/docker/Dockerfile_windows_10

113
BUILDING.txt
View File

@ -492,39 +492,66 @@ Building on CentOS 8
----------------------------------------------------------------------------------
Building on Windows
Building on Windows 10
----------------------------------------------------------------------------------
Requirements:
* Windows System
* Windows 10
* JDK 1.8
* Maven 3.0 or later
* Boost 1.72
* Protocol Buffers 3.7.1
* CMake 3.19 or newer
* Visual Studio 2010 Professional or Higher
* Windows SDK 8.1 (if building CPU rate control for the container executor)
* zlib headers (if building native code bindings for zlib)
* Maven 3.0 or later (maven.apache.org)
* Boost 1.72 (boost.org)
* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
* CMake 3.19 or newer (cmake.org)
* Visual Studio 2019 (visualstudio.com)
* Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from
http://msdn.microsoft.com/en-us/windows/bg162891.aspx)
* Zlib (zlib.net, if building native code bindings for zlib)
* Git (preferably, get this from https://git-scm.com/download/win since the package also contains
Unix command-line tools that are needed during packaging).
* Python (python.org, for generation of docs using 'mvn site')
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
tools must be present on your PATH.
* Python ( for generation of docs using 'mvn site')
Unix command-line tools are also included with the Windows Git package which
can be downloaded from http://git-scm.com/downloads
If using Visual Studio, it must be Professional level or higher.
Do not use Visual Studio Express. It does not support compiling for 64-bit,
which is problematic if running a 64-bit system.
The Windows SDK 8.1 is available to download at:
http://msdn.microsoft.com/en-us/windows/bg162891.aspx
Cygwin is not required.
----------------------------------------------------------------------------------
Building guidelines:
Hadoop repository provides the Dockerfile for building Hadoop on Windows 10, located at
dev-support/docker/Dockerfile_windows_10. It is highly recommended to use this and create the
Docker image for building Hadoop on Windows 10, since you don't have to install anything else
other than Docker and no additional steps are required in terms of aligning the environment with
the necessary paths etc.
However, if you still prefer taking the route of not using Docker, this Dockerfile_windows_10 will
still be immensely useful as a raw guide for all the steps involved in creating the environment
needed to build Hadoop on Windows 10.
Building using the Docker:
We first need to build the Docker image for building Hadoop on Windows 10. Run this command from
the root of the Hadoop repository.
> docker build -t hadoop-windows-10-builder -f .\dev-support\docker\Dockerfile_windows_10 .\dev-support\docker\
Start the container with the image that we just built.
> docker run --rm -it hadoop-windows-10-builder
You can now clone the Hadoop repo inside this container and proceed with the build.
NOTE:
While one may perceive the idea of mounting the locally cloned (on the host filesystem) Hadoop
repository into the container (using the -v option), we have seen the build to fail owing to some
files not being able to be located by Maven. Thus, we suggest cloning the Hadoop repository to a
non-mounted folder inside the container and proceed with the build. When the build is completed,
you may use the "docker cp" command to copy the built Hadoop tar.gz file from the docker container
to the host filesystem. If you still would like to mount the Hadoop codebase, a workaround would
be to copy the mounted Hadoop codebase into another folder (which doesn't point to a mount) in the
container's filesystem and use this for building.
However, we noticed no build issues when the Maven repository from the host filesystem was mounted
into the container. One may use this to greatly reduce the build time. Assuming that the Maven
repository is located at D:\Maven\Repository in the host filesystem, one can use the following
command to mount the same onto the default Maven repository location while launching the container.
> docker run --rm -v D:\Maven\Repository:C:\Users\ContainerAdministrator\.m2\repository -it hadoop-windows-10-builder
Building:
Keep the source code tree in a short path to avoid running into problems related
@ -540,6 +567,24 @@ configure the bit-ness of the build, and set several optional components.
Several tests require that the user must have the Create Symbolic Links
privilege.
To simplify the installation of Boost, Protocol buffers, OpenSSL and Zlib dependencies we can use
vcpkg (https://github.com/Microsoft/vcpkg.git). Upon cloning the vcpkg repo, checkout the commit
7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d to get the required versions of the dependencies
mentioned above.
> git clone https://github.com/Microsoft/vcpkg.git
> cd vcpkg
> git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d
> .\bootstrap-vcpkg.bat
> .\vcpkg.exe install boost:x64-windows
> .\vcpkg.exe install protobuf:x64-windows
> .\vcpkg.exe install openssl:x64-windows
> .\vcpkg.exe install zlib:x64-windows
Set the following environment variables -
(Assuming that vcpkg was checked out at C:\vcpkg)
> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
> set MAVEN_OPTS=-Xmx2048M -Xss128M
All Maven goals are the same as described above with the exception that
native code is built by enabling the 'native-win' Maven profile. -Pnative-win
is enabled by default when building on Windows since the native components
@ -557,6 +602,24 @@ the zlib 1.2.7 source tree.
http://www.zlib.net/
Build command:
The following command builds all the modules in the Hadoop project and generates the tar.gz file in
hadoop-dist/target upon successful build. Run these commands from an
"x64 Native Tools Command Prompt for VS 2019" which can be found under "Visual Studio 2019" in the
Windows start menu. If you're using the Docker image from Dockerfile_windows_10, you'll be
logged into "x64 Native Tools Command Prompt for VS 2019" automatically when you start the
container.
> set classpath=
> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist^
-Drequire.openssl -Drequire.test.libhadoop -Pyarn-ui -Dshell-executable=C:\Git\bin\bash.exe^
-Dtar -Dopenssl.prefix=C:\vcpkg\installed\x64-windows^
-Dcmake.prefix.path=C:\vcpkg\installed\x64-windows^
-Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo^
-Dwindows.build.hdfspp.dll=off -Dwindows.no.sasl=on -Duse.platformToolsetVersion=v142
----------------------------------------------------------------------------------
Building distributions:

169
LICENSE-binary
View File

@ -210,28 +210,28 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
com.aliyun:aliyun-java-sdk-core:3.4.0
com.aliyun:aliyun-java-sdk-ecs:4.2.0
com.aliyun:aliyun-java-sdk-ram:3.0.0
com.aliyun:aliyun-java-sdk-core:4.5.10
com.aliyun:aliyun-java-sdk-kms:2.11.0
com.aliyun:aliyun-java-sdk-ram:3.1.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.13.2
com.amazonaws:aws-java-sdk-bundle:1.11.901
com.amazonaws:aws-java-sdk-bundle:1.12.316
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.13.2
com.fasterxml.jackson.core:jackson-core:2.13.2
com.fasterxml.jackson.core:jackson-databind:2.13.2.2
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.2
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.2
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.2
com.fasterxml.jackson.core:jackson-annotations:2.12.7
com.fasterxml.jackson.core:jackson-core:2.12.7
com.fasterxml.jackson.core:jackson-databind:2.12.7.1
com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
com.fasterxml.uuid:java-uuid-generator:3.1.4
com.fasterxml.woodstox:woodstox-core:5.3.0
com.fasterxml.woodstox:woodstox-core:5.4.0
com.github.davidmoten:rxjava-extras:0.8.0.17
com.github.stephenc.jcip:jcip-annotations:1.0-1
com.google:guice:4.0
com.google:guice-servlet:4.0
com.google.api.grpc:proto-google-common-protos:1.0.0
com.google.code.gson:2.2.4
com.google.code.gson:2.9.0
com.google.errorprone:error_prone_annotations:2.2.0
com.google.j2objc:j2objc-annotations:1.1
com.google.json-simple:json-simple:1.1.1
@ -241,17 +241,16 @@ com.google.guava:guava:27.0-jre
com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
com.microsoft.azure:azure-storage:7.0.0
com.nimbusds:nimbus-jose-jwt:9.8.1
com.squareup.okhttp:okhttp:2.7.5
com.squareup.okio:okio:1.6.0
com.squareup.okhttp3:okhttp:4.10.0
com.squareup.okio:okio:3.2.0
com.zaxxer:HikariCP:4.0.3
commons-beanutils:commons-beanutils:1.9.3
commons-beanutils:commons-beanutils:1.9.4
commons-cli:commons-cli:1.2
commons-codec:commons-codec:1.11
commons-collections:commons-collections:3.2.2
commons-daemon:commons-daemon:1.0.13
commons-io:commons-io:2.8.0
commons-logging:commons-logging:1.1.3
commons-net:commons-net:3.6
commons-net:commons-net:3.9.0
de.ruedigermoeller:fst:2.50
io.grpc:grpc-api:1.26.0
io.grpc:grpc-context:1.26.0
@ -260,20 +259,42 @@ io.grpc:grpc-netty:1.26.0
io.grpc:grpc-protobuf:1.26.0
io.grpc:grpc-protobuf-lite:1.26.0
io.grpc:grpc-stub:1.26.0
io.netty:netty:3.10.6.Final
io.netty:netty-all:4.1.42.Final
io.netty:netty-buffer:4.1.27.Final
io.netty:netty-codec:4.1.27.Final
io.netty:netty-codec-http:4.1.27.Final
io.netty:netty-codec-http2:4.1.27.Final
io.netty:netty-codec-socks:4.1.27.Final
io.netty:netty-common:4.1.27.Final
io.netty:netty-handler:4.1.27.Final
io.netty:netty-handler-proxy:4.1.27.Final
io.netty:netty-resolver:4.1.27.Final
io.netty:netty-transport:4.1.27.Final
io.opencensus:opencensus-api:0.12.3
io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
io.netty:netty-all:4.1.77.Final
io.netty:netty-buffer:4.1.77.Final
io.netty:netty-codec:4.1.77.Final
io.netty:netty-codec-dns:4.1.77.Final
io.netty:netty-codec-haproxy:4.1.77.Final
io.netty:netty-codec-http:4.1.77.Final
io.netty:netty-codec-http2:4.1.77.Final
io.netty:netty-codec-memcache:4.1.77.Final
io.netty:netty-codec-mqtt:4.1.77.Final
io.netty:netty-codec-redis:4.1.77.Final
io.netty:netty-codec-smtp:4.1.77.Final
io.netty:netty-codec-socks:4.1.77.Final
io.netty:netty-codec-stomp:4.1.77.Final
io.netty:netty-codec-xml:4.1.77.Final
io.netty:netty-common:4.1.77.Final
io.netty:netty-handler:4.1.77.Final
io.netty:netty-handler-proxy:4.1.77.Final
io.netty:netty-resolver:4.1.77.Final
io.netty:netty-resolver-dns:4.1.77.Final
io.netty:netty-transport:4.1.77.Final
io.netty:netty-transport-rxtx:4.1.77.Final
io.netty:netty-transport-sctp:4.1.77.Final
io.netty:netty-transport-udt:4.1.77.Final
io.netty:netty-transport-classes-epoll:4.1.77.Final
io.netty:netty-transport-native-unix-common:4.1.77.Final
io.netty:netty-transport-classes-kqueue:4.1.77.Final
io.netty:netty-resolver-dns-classes-macos:4.1.77.Final
io.netty:netty-transport-native-epoll:4.1.77.Final
io.netty:netty-transport-native-kqueue:4.1.77.Final
io.netty:netty-resolver-dns-native-macos:4.1.77.Final
io.opencensus:opencensus-api:0.24.0
io.opencensus:opencensus-contrib-grpc-metrics:0.24.0
io.opentracing:opentracing-api:0.33.0
io.opentracing:opentracing-noop:0.33.0
io.opentracing:opentracing-util:0.33.0
io.perfmark:perfmark-api:0.19.0
io.reactivex:rxjava:1.3.8
io.reactivex:rxjava-string:1.1.1
io.reactivex:rxnetty:0.4.20
@ -286,12 +307,12 @@ net.minidev:json-smart:2.4.7
org.apache.avro:avro:1.9.2
org.apache.commons:commons-collections4:4.2
org.apache.commons:commons-compress:1.21
org.apache.commons:commons-configuration2:2.1.1
org.apache.commons:commons-csv:1.0
org.apache.commons:commons-configuration2:2.8.0
org.apache.commons:commons-csv:1.9.0
org.apache.commons:commons-digester:1.8.1
org.apache.commons:commons-lang3:3.12.0
org.apache.commons:commons-math3:3.1.1
org.apache.commons:commons-text:1.4
org.apache.commons:commons-math3:3.6.1
org.apache.commons:commons-text:1.10.0
org.apache.commons:commons-validator:1.6
org.apache.curator:curator-client:5.2.0
org.apache.curator:curator-framework:5.2.0
@ -305,46 +326,49 @@ org.apache.htrace:htrace-core:3.1.0-incubating
org.apache.htrace:htrace-core4:4.1.0-incubating
org.apache.httpcomponents:httpclient:4.5.6
org.apache.httpcomponents:httpcore:4.4.10
org.apache.kafka:kafka-clients:2.8.1
org.apache.kerby:kerb-admin:1.0.1
org.apache.kerby:kerb-client:1.0.1
org.apache.kerby:kerb-common:1.0.1
org.apache.kerby:kerb-core:1.0.1
org.apache.kerby:kerb-crypto:1.0.1
org.apache.kerby:kerb-identity:1.0.1
org.apache.kerby:kerb-server:1.0.1
org.apache.kerby:kerb-simplekdc:1.0.1
org.apache.kerby:kerb-util:1.0.1
org.apache.kerby:kerby-asn1:1.0.1
org.apache.kerby:kerby-config:1.0.1
org.apache.kerby:kerby-pkix:1.0.1
org.apache.kerby:kerby-util:1.0.1
org.apache.kerby:kerby-xdr:1.0.1
org.apache.kerby:token-provider:1.0.1
org.apache.kafka:kafka-clients:2.8.2
org.apache.kerby:kerb-admin:2.0.3
org.apache.kerby:kerb-client:2.0.3
org.apache.kerby:kerb-common:2.0.3
org.apache.kerby:kerb-core:2.0.3
org.apache.kerby:kerb-crypto:2.0.3
org.apache.kerby:kerb-identity:2.0.3
org.apache.kerby:kerb-server:2.0.3
org.apache.kerby:kerb-simplekdc:2.0.3
org.apache.kerby:kerb-util:2.0.3
org.apache.kerby:kerby-asn1:2.0.3
org.apache.kerby:kerby-config:2.0.3
org.apache.kerby:kerby-pkix:2.0.3
org.apache.kerby:kerby-util:2.0.3
org.apache.kerby:kerby-xdr:2.0.3
org.apache.kerby:token-provider:2.0.3
org.apache.solr:solr-solrj:8.8.2
org.apache.yetus:audience-annotations:0.5.0
org.apache.zookeeper:zookeeper:3.6.3
org.codehaus.jettison:jettison:1.1
org.eclipse.jetty:jetty-annotations:9.4.44.v20210927
org.eclipse.jetty:jetty-http:9.4.44.v20210927
org.eclipse.jetty:jetty-io:9.4.44.v20210927
org.eclipse.jetty:jetty-jndi:9.4.44.v20210927
org.eclipse.jetty:jetty-plus:9.4.44.v20210927
org.eclipse.jetty:jetty-security:9.4.44.v20210927
org.eclipse.jetty:jetty-server:9.4.44.v20210927
org.eclipse.jetty:jetty-servlet:9.4.44.v20210927
org.eclipse.jetty:jetty-util:9.4.44.v20210927
org.eclipse.jetty:jetty-util-ajax:9.4.44.v20210927
org.eclipse.jetty:jetty-webapp:9.4.44.v20210927
org.eclipse.jetty:jetty-xml:9.4.44.v20210927
org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.44.v20210927
org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.44.v20210927
org.codehaus.jettison:jettison:1.5.3
org.eclipse.jetty:jetty-annotations:9.4.48.v20220622
org.eclipse.jetty:jetty-http:9.4.48.v20220622
org.eclipse.jetty:jetty-io:9.4.48.v20220622
org.eclipse.jetty:jetty-jndi:9.4.48.v20220622
org.eclipse.jetty:jetty-plus:9.4.48.v20220622
org.eclipse.jetty:jetty-security:9.4.48.v20220622
org.eclipse.jetty:jetty-server:9.4.48.v20220622
org.eclipse.jetty:jetty-servlet:9.4.48.v20220622
org.eclipse.jetty:jetty-util:9.4.48.v20220622
org.eclipse.jetty:jetty-util-ajax:9.4.48.v20220622
org.eclipse.jetty:jetty-webapp:9.4.48.v20220622
org.eclipse.jetty:jetty-xml:9.4.48.v20220622
org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.48.v20220622
org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.48.v20220622
org.ehcache:ehcache:3.3.1
org.ini4j:ini4j:0.5.4
org.jetbrains.kotlin:kotlin-stdlib:1.4.10
org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
org.lz4:lz4-java:1.7.1
org.objenesis:objenesis:2.6
org.xerial.snappy:snappy-java:1.0.5
org.yaml:snakeyaml:1.16:
org.wildfly.openssl:wildfly-openssl:1.0.7.Final
org.yaml:snakeyaml:1.33
org.wildfly.openssl:wildfly-openssl:1.1.3.Final
--------------------------------------------------------------------------------
@ -408,7 +432,7 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/*
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/*
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL
@ -416,7 +440,7 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanage
bootstrap v3.3.6
broccoli-asset-rev v2.4.2
broccoli-funnel v1.0.1
datatables v1.10.8
datatables v1.11.5
em-helpers v0.5.13
em-table v0.1.6
ember v2.2.0
@ -491,7 +515,6 @@ javax.annotation:javax.annotation-api:1.3.2
javax.servlet:javax.servlet-api:3.1.0
javax.servlet.jsp:jsp-api:2.1
javax.websocket:javax.websocket-api:1.0
javax.ws.rs:javax.ws.rs-api:2.1.1
javax.ws.rs:jsr311-api:1.1.1
javax.xml.bind:jaxb-api:2.2.11
@ -500,12 +523,14 @@ Eclipse Public License 1.0
--------------------------
junit:junit:4.13.2
org.jacoco:org.jacoco.agent:0.8.5
HSQL License
------------
org.hsqldb:hsqldb:2.3.4
org.hsqldb:hsqldb:2.7.1
JDOM License

2
LICENSE.txt
View File

@ -252,7 +252,7 @@ hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.10.18/*
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/*
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL

4
dev-support/Jenkinsfile vendored
View File

@ -47,7 +47,7 @@ pipeline {
options {
buildDiscarder(logRotator(numToKeepStr: '5'))
timeout (time: 24, unit: 'HOURS')
timeout (time: 48, unit: 'HOURS')
timestamps()
checkoutToSubdirectory('src')
}
@ -55,7 +55,7 @@ pipeline {
environment {
YETUS='yetus'
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
YETUS_VERSION='rel/0.14.0'
}
parameters {

2
dev-support/bin/yetus-wrapper
View File

@ -77,7 +77,7 @@ WANTED="$1"
shift
ARGV=("$@")
HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.13.0}
HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
BIN=$(yetus_abs "${BASH_SOURCE-$0}")
BINDIR=$(dirname "${BIN}")

1
dev-support/docker/Dockerfile_debian_10
View File

@ -82,6 +82,7 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
####
# Install packages
####
RUN pkg-resolver/install-cmake.sh debian:10
RUN pkg-resolver/install-spotbugs.sh debian:10
RUN pkg-resolver/install-boost.sh debian:10
RUN pkg-resolver/install-protobuf.sh debian:10

81
dev-support/docker/Dockerfile_windows_10 Normal file
View File

@ -0,0 +1,81 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
FROM mcr.microsoft.com/windows:ltsc2019
# Need to disable the progress bar for speeding up the downloads.
# hadolint ignore=SC2086
RUN powershell $Global:ProgressPreference = 'SilentlyContinue'
# Restore the default Windows shell for correct batch processing.
SHELL ["cmd", "/S", "/C"]
# Install Visual Studio 2019 Build Tools.
RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe \
&& (start /w vs_buildtools.exe --quiet --wait --norestart --nocache \
--installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\BuildTools" \
--add Microsoft.VisualStudio.Workload.VCTools \
--add Microsoft.VisualStudio.Component.VC.ASAN \
--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \
--add Microsoft.VisualStudio.Component.Windows10SDK.19041 \
|| IF "%ERRORLEVEL%"=="3010" EXIT 0) \
&& del /q vs_buildtools.exe
# Install Chocolatey.
RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
# Install git.
RUN choco install git.install -y
RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\Git
# Install vcpkg.
# hadolint ignore=DL3003
RUN powershell git clone https://github.com/microsoft/vcpkg.git \
&& cd vcpkg \
&& git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
&& .\bootstrap-vcpkg.bat
RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
# Install Azul Java 8 JDK.
RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
# Install Apache Maven.
RUN powershell Invoke-WebRequest -URI https://dlcdn.apache.org/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
# Install CMake 3.19.0.
RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
# We get strange Javadoc errors without this.
RUN setx classpath ""
# Define the entry point for the docker container.
ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]

2
dev-support/docker/pkg-resolver/install-maven.sh
View File

@ -40,7 +40,7 @@ fi
if [ "$version_to_install" == "3.6.3" ]; then
mkdir -p /opt/maven /tmp/maven &&
curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
curl -L -s -S https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
else

1
dev-support/docker/pkg-resolver/packages.json
View File

@ -62,7 +62,6 @@
"centos:8": "clang"
},
"cmake": {
"debian:10": "cmake",
"ubuntu:focal": "cmake",
"ubuntu:focal::arch64": "cmake"
},

7
dev-support/git-jira-validation/git_jira_fix_version_check.py
View File

@ -71,10 +71,9 @@
print("Commit seems reverted. \t\t\t Commit: " + commit)
continue
ACTUAL_PROJECT_JIRA = None
for project_jira in project_jira_keys:
if project_jira in commit:
ACTUAL_PROJECT_JIRA = project_jira
break
matches = re.findall('|'.join(project_jira_keys), commit)
if matches:
ACTUAL_PROJECT_JIRA = matches[0]
if not ACTUAL_PROJECT_JIRA:
print("WARN: Jira not found. \t\t\t Commit: " + commit)
continue

3
dev-support/hadoop-vote.sh
View File

@ -189,6 +189,8 @@ pushd "${OUTPUT_DIR}"
download_and_import_keys
download_release_candidate
pushd "${HADOOP_RC_VERSION}"
execute verify_signatures
execute verify_checksums
execute unzip_from_source
@ -196,6 +198,7 @@ execute rat_test
execute build_from_source
execute build_tar_from_source
popd
popd
print_when_exit

10
hadoop-client-modules/hadoop-client-api/pom.xml
View File

@ -98,13 +98,6 @@
<createSourcesJar>true</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent>
</configuration>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
@ -254,8 +247,7 @@
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>NOTICE.txt</resource>

17
hadoop-client-modules/hadoop-client-minicluster/pom.xml
View File

@ -671,13 +671,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
@ -704,7 +697,6 @@
<exclude>org.bouncycastle:*</exclude>
<!-- Leave snappy that includes native methods which cannot be relocated. -->
<exclude>org.xerial.snappy:*</exclude>
<exclude>javax.ws.rs:javax.ws.rs-api</exclude>
</excludes>
</artifactSet>
<filters>
@ -757,6 +749,12 @@
<exclude>META-INF/versions/11/module-info.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.google.code.gson:gson</artifact>
<excludes>
<exclude>META-INF/versions/9/module-info.class</exclude>
</excludes>
</filter>
<!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
<filter>
@ -1047,8 +1045,7 @@
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>

21
hadoop-client-modules/hadoop-client-runtime/pom.xml
View File

@ -128,13 +128,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
@ -155,6 +148,7 @@
<!-- Leave javax APIs that are stable -->
<!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>io.netty:*</exclude>
<exclude>io.dropwizard.metrics:metrics-core</exclude>
<exclude>org.eclipse.jetty:jetty-servlet</exclude>
<exclude>org.eclipse.jetty:jetty-security</exclude>
@ -163,7 +157,8 @@
<exclude>org.bouncycastle:*</exclude>
<!-- Leave snappy that includes native methods which cannot be relocated. -->
<exclude>org.xerial.snappy:*</exclude>
<exclude>javax.ws.rs:javax.ws.rs-api</exclude>
<!-- leave out kotlin classes -->
<exclude>org.jetbrains.kotlin:*</exclude>
</excludes>
</artifactSet>
<filters>
@ -249,6 +244,13 @@
<exclude>META-INF/versions/11/module-info.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.google.code.gson:gson</artifact>
<excludes>
<exclude>META-INF/versions/9/module-info.class</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
@ -391,8 +393,7 @@
-->
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>

12
hadoop-client-modules/hadoop-client/pom.xml
View File

@ -114,6 +114,18 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib</artifactId>
</exclusion>
<exclusion>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-stdlib-common</artifactId>
</exclusion>
<exclusion>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>

5
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
View File

@ -127,11 +127,6 @@
<artifactId>hadoop-azure-datalake</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-openstack</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cos</artifactId>

2
hadoop-cloud-storage-project/hadoop-cos/pom.xml
View File

@ -109,7 +109,7 @@
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.19</version>
<version>5.6.69</version>
<scope>compile</scope>
</dependency>

1
hadoop-common-project/hadoop-auth-examples/src/main/java/org/apache/hadoop/security/authentication/examples/RequestLoggerFilter.java
View File

@ -116,6 +116,7 @@ private List<String> getHeaderValues(String name, boolean reset) {
public void addCookie(Cookie cookie) {
super.addCookie(cookie);
List<String> cookies = getHeaderValues("Set-Cookie", false);
cookies.addAll(getHeaderValues("set-cookie", false));
cookies.add(cookie.getName() + "=" + cookie.getValue());
}

3
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
View File

@ -92,6 +92,9 @@ public synchronized Map<String, List<String>> get(URI uri,
@Override
public void put(URI uri, Map<String, List<String>> responseHeaders) {
List<String> headers = responseHeaders.get("Set-Cookie");
if (headers == null) {
headers = responseHeaders.get("set-cookie");
}
if (headers != null) {
for (String header : headers) {
List<HttpCookie> cookies;

6
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
View File

@ -280,6 +280,9 @@ private boolean isNegotiate(HttpURLConnection conn) throws IOException {
boolean negotiate = false;
if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
if (authHeader == null) {
authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
}
negotiate = authHeader != null && authHeader.trim().startsWith(NEGOTIATE);
}
return negotiate;
@ -388,6 +391,9 @@ private byte[] readToken(HttpURLConnection conn)
int status = conn.getResponseCode();
if (status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
if (authHeader == null) {
authHeader = conn.getHeaderField(WWW_AUTHENTICATE.toLowerCase());
}
if (authHeader == null || !authHeader.trim().startsWith(NEGOTIATE)) {
throw new AuthenticationException("Invalid SPNEGO sequence, '" + WWW_AUTHENTICATE +
"' header incorrect: " + authHeader);

4
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java
View File

@ -616,7 +616,9 @@ && getMaxInactiveInterval() > 0) {
// present.. reset to 403 if not found..
if ((errCode == HttpServletResponse.SC_UNAUTHORIZED)
&& (!httpResponse.containsHeader(
KerberosAuthenticator.WWW_AUTHENTICATE))) {
KerberosAuthenticator.WWW_AUTHENTICATE)
&& !httpResponse.containsHeader(
KerberosAuthenticator.WWW_AUTHENTICATE.toLowerCase()))) {
errCode = HttpServletResponse.SC_FORBIDDEN;
}
// After Jetty 9.4.21, sendError() no longer allows a custom message.

1
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/CertificateUtil.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.security.authentication.util;
import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.security.PublicKey;
import java.security.cert.CertificateException;

77
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/JaasConfiguration.java Normal file
View File

@ -0,0 +1,77 @@
/**
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License. See accompanying LICENSE file.
*/
package org.apache.hadoop.security.authentication.util;
import java.util.HashMap;
import java.util.Map;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
/**
* Creates a programmatic version of a jaas.conf file. This can be used
* instead of writing a jaas.conf file and setting the system property,
* "java.security.auth.login.config", to point to that file. It is meant to be
* used for connecting to ZooKeeper.
*/
public class JaasConfiguration extends Configuration {
private final javax.security.auth.login.Configuration baseConfig =
javax.security.auth.login.Configuration.getConfiguration();
private final AppConfigurationEntry[] entry;
private final String entryName;
/**
* Add an entry to the jaas configuration with the passed in name,
* principal, and keytab. The other necessary options will be set for you.
*
* @param entryName The name of the entry (e.g. "Client")
* @param principal The principal of the user
* @param keytab The location of the keytab
*/
public JaasConfiguration(String entryName, String principal, String keytab) {
this.entryName = entryName;
Map<String, String> options = new HashMap<>();
options.put("keyTab", keytab);
options.put("principal", principal);
options.put("useKeyTab", "true");
options.put("storeKey", "true");
options.put("useTicketCache", "false");
options.put("refreshKrb5Config", "true");
String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
if ("true".equalsIgnoreCase(jaasEnvVar)) {
options.put("debug", "true");
}
entry = new AppConfigurationEntry[]{
new AppConfigurationEntry(getKrb5LoginModuleName(),
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
options)};
}
@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
return (entryName.equals(name)) ? entry : ((baseConfig != null)
? baseConfig.getAppConfigurationEntry(name) : null);
}
private String getKrb5LoginModuleName() {
String krb5LoginModuleName;
if (System.getProperty("java.vendor").contains("IBM")) {
krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
} else {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
return krb5LoginModuleName;
}
}

2
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/KerberosUtil.java
View File

@ -236,7 +236,7 @@ public static final String getServicePrincipal(String service,
*/
static final String[] getPrincipalNames(String keytabFileName) throws IOException {
Keytab keytab = Keytab.loadKeytab(new File(keytabFileName));
Set<String> principals = new HashSet<String>();
Set<String> principals = new HashSet<>();
List<PrincipalName> entries = keytab.getPrincipals();
for (PrincipalName entry : entries) {
principals.add(entry.getName().replace("\\", "/"));

61
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/util/ZKSignerSecretProvider.java
View File

@ -17,12 +17,9 @@
import java.nio.ByteBuffer;
import java.security.SecureRandom;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.Configuration;
import javax.servlet.ServletContext;
import org.apache.curator.RetryPolicy;
@ -429,62 +426,4 @@ public List<ACL> getAclForPath(String path) {
return saslACL;
}
}
/**
* Creates a programmatic version of a jaas.conf file. This can be used
* instead of writing a jaas.conf file and setting the system property,
* "java.security.auth.login.config", to point to that file. It is meant to be
* used for connecting to ZooKeeper.
*/
@InterfaceAudience.Private
public static class JaasConfiguration extends Configuration {
private final javax.security.auth.login.Configuration baseConfig =
javax.security.auth.login.Configuration.getConfiguration();
private static AppConfigurationEntry[] entry;
private String entryName;
/**
* Add an entry to the jaas configuration with the passed in name,
* principal, and keytab. The other necessary options will be set for you.
*
* @param entryName The name of the entry (e.g. "Client")
* @param principal The principal of the user
* @param keytab The location of the keytab
*/
public JaasConfiguration(String entryName, String principal, String keytab) {
this.entryName = entryName;
Map<String, String> options = new HashMap<String, String>();
options.put("keyTab", keytab);
options.put("principal", principal);
options.put("useKeyTab", "true");
options.put("storeKey", "true");
options.put("useTicketCache", "false");
options.put("refreshKrb5Config", "true");
String jaasEnvVar = System.getenv("HADOOP_JAAS_DEBUG");
if (jaasEnvVar != null && "true".equalsIgnoreCase(jaasEnvVar)) {
options.put("debug", "true");
}
entry = new AppConfigurationEntry[]{
new AppConfigurationEntry(getKrb5LoginModuleName(),
AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
options)};
}
@Override
public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
return (entryName.equals(name)) ? entry : ((baseConfig != null)
? baseConfig.getAppConfigurationEntry(name) : null);
}
private String getKrb5LoginModuleName() {
String krb5LoginModuleName;
if (System.getProperty("java.vendor").contains("IBM")) {
krb5LoginModuleName = "com.ibm.security.auth.module.Krb5LoginModule";
} else {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
return krb5LoginModuleName;
}
}
}

68
hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/util/PlatformName.java
View File

@ -18,6 +18,10 @@
package org.apache.hadoop.util;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -33,10 +37,10 @@ public class PlatformName {
* per the java-vm.
*/
public static final String PLATFORM_NAME =
(System.getProperty("os.name").startsWith("Windows")
? System.getenv("os") : System.getProperty("os.name"))
+ "-" + System.getProperty("os.arch")
+ "-" + System.getProperty("sun.arch.data.model");
(System.getProperty("os.name").startsWith("Windows") ?
System.getenv("os") : System.getProperty("os.name"))
+ "-" + System.getProperty("os.arch") + "-"
+ System.getProperty("sun.arch.data.model");
/**
* The java vendor name used in this platform.
@ -44,10 +48,60 @@ public class PlatformName {
public static final String JAVA_VENDOR_NAME = System.getProperty("java.vendor");
/**
* A public static variable to indicate the current java vendor is
* IBM java or not.
* Define a system class accessor that is open to changes in underlying implementations
* of the system class loader modules.
*/
public static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
private static final class SystemClassAccessor extends ClassLoader {
public Class<?> getSystemClass(String className) throws ClassNotFoundException {
return findSystemClass(className);
}
}
/**
* A public static variable to indicate the current java vendor is
* IBM and the type is Java Technology Edition which provides its
* own implementations of many security packages and Cipher suites.
* Note that these are not provided in Semeru runtimes:
* See https://developer.ibm.com/languages/java/semeru-runtimes for details.
*/
public static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM") &&
hasIbmTechnologyEditionModules();
private static boolean hasIbmTechnologyEditionModules() {
return Arrays.asList(
"com.ibm.security.auth.module.JAASLoginModule",
"com.ibm.security.auth.module.Win64LoginModule",
"com.ibm.security.auth.module.NTLoginModule",
"com.ibm.security.auth.module.AIX64LoginModule",
"com.ibm.security.auth.module.LinuxLoginModule",
"com.ibm.security.auth.module.Krb5LoginModule"
).stream().anyMatch((module) -> isSystemClassAvailable(module));
}
/**
* In rare cases where different behaviour is performed based on the JVM vendor
* this method should be used to test for a unique JVM class provided by the
* vendor rather than using the vendor method. For example if on JVM provides a
* different Kerberos login module testing for that login module being loadable
* before configuring to use it is preferable to using the vendor data.
*
* @param className the name of a class in the JVM to test for
* @return true if the class is available, false otherwise.
*/
private static boolean isSystemClassAvailable(String className) {
return AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> {
try {
// Using ClassLoader.findSystemClass() instead of
// Class.forName(className, false, null) because Class.forName with a null
// ClassLoader only looks at the boot ClassLoader with Java 9 and above
// which doesn't look at all the modules available to the findSystemClass.
new SystemClassAccessor().getSystemClass(className);
return true;
} catch (Exception ignored) {
return false;
}
});
}
public static void main(String[] args) {
System.out.println(PLATFORM_NAME);

2
hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
View File

@ -24,7 +24,7 @@ This filter must be configured in front of all the web application resources tha
The Hadoop Auth and dependent JAR files must be in the web application classpath (commonly the `WEB-INF/lib` directory).
Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define the SLF4J API dependency but it does not define the dependency on a concrete logging implementation, this must be addded explicitly to the web application. For example, if the web applicationan uses Log4j, the SLF4J-LOG4J12 and LOG4J jar files must be part part of the web application classpath as well as the Log4j configuration file.
Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define the SLF4J API dependency but it does not define the dependency on a concrete logging implementation, this must be addded explicitly to the web application. For example, if the web applicationan uses Log4j, the SLF4J-LOG4J12 and LOG4J jar files must be part of the web application classpath as well as the Log4j configuration file.
### Common Configuration parameters

4
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/KerberosTestUtils.java
View File

@ -108,9 +108,9 @@ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
public static <T> T doAs(String principal, final Callable<T> callable) throws Exception {
LoginContext loginContext = null;
try {
Set<Principal> principals = new HashSet<Principal>();
Set<Principal> principals = new HashSet<>();
principals.add(new KerberosPrincipal(KerberosTestUtils.getClientPrincipal()));
Subject subject = new Subject(false, principals, new HashSet<Object>(), new HashSet<Object>());
Subject subject = new Subject(false, principals, new HashSet<>(), new HashSet<>());
loginContext = new LoginContext("", subject, null, new KerberosConfiguration(principal));
loginContext.login();
subject = loginContext.getSubject();

38
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestAuthenticatedURL.java
View File

@ -89,6 +89,44 @@ public void testExtractTokenFail() throws Exception {
}
}
@Test
public void testExtractTokenCookieHeader() throws Exception {
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
String tokenStr = "foo";
Map<String, List<String>> headers = new HashMap<>();
List<String> cookies = new ArrayList<>();
cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
headers.put("Set-Cookie", cookies);
Mockito.when(conn.getHeaderFields()).thenReturn(headers);
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
AuthenticatedURL.extractToken(conn, token);
Assert.assertTrue(token.isSet());
}
@Test
public void testExtractTokenLowerCaseCookieHeader() throws Exception {
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_OK);
String tokenStr = "foo";
Map<String, List<String>> headers = new HashMap<>();
List<String> cookies = new ArrayList<>();
cookies.add(AuthenticatedURL.AUTH_COOKIE + "=" + tokenStr);
headers.put("set-cookie", cookies);
Mockito.when(conn.getHeaderFields()).thenReturn(headers);
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
AuthenticatedURL.extractToken(conn, token);
Assert.assertTrue(token.isSet());
}
@Test
public void testConnectionConfigurator() throws Exception {
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);

82
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/client/TestKerberosAuthenticator.java
View File

@ -21,8 +21,13 @@
import static org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.NAME_RULES;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.charset.CharacterCodingException;
import javax.security.sasl.AuthenticationException;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
@ -32,10 +37,12 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.Callable;
@ -248,4 +255,79 @@ public void testWrapExceptionWithMessage() {
Assert.assertTrue(ex.equals(ex2));
}
@Test(timeout = 60000)
public void testNegotiate() throws NoSuchMethodException, InvocationTargetException,
IllegalAccessException, IOException {
KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getHeaderField(KerberosAuthenticator.WWW_AUTHENTICATE)).
thenReturn(KerberosAuthenticator.NEGOTIATE);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
Method method = KerberosAuthenticator.class.getDeclaredMethod("isNegotiate",
HttpURLConnection.class);
method.setAccessible(true);
Assert.assertTrue((boolean)method.invoke(kerberosAuthenticator, conn));
}
@Test(timeout = 60000)
public void testNegotiateLowerCase() throws NoSuchMethodException, InvocationTargetException,
IllegalAccessException, IOException {
KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getHeaderField("www-authenticate"))
.thenReturn(KerberosAuthenticator.NEGOTIATE);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
Method method = KerberosAuthenticator.class.getDeclaredMethod("isNegotiate",
HttpURLConnection.class);
method.setAccessible(true);
Assert.assertTrue((boolean)method.invoke(kerberosAuthenticator, conn));
}
@Test(timeout = 60000)
public void testReadToken() throws NoSuchMethodException, IOException, IllegalAccessException,
InvocationTargetException {
KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
FieldUtils.writeField(kerberosAuthenticator, "base64", new Base64(), true);
Base64 base64 = new Base64();
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
Mockito.when(conn.getHeaderField(KerberosAuthenticator.WWW_AUTHENTICATE))
.thenReturn(KerberosAuthenticator.NEGOTIATE + " " +
Arrays.toString(base64.encode("foobar".getBytes())));
Method method = KerberosAuthenticator.class.getDeclaredMethod("readToken",
HttpURLConnection.class);
method.setAccessible(true);
method.invoke(kerberosAuthenticator, conn); // expecting this not to throw an exception
}
@Test(timeout = 60000)
public void testReadTokenLowerCase() throws NoSuchMethodException, IOException,
IllegalAccessException, InvocationTargetException {
KerberosAuthenticator kerberosAuthenticator = new KerberosAuthenticator();
FieldUtils.writeField(kerberosAuthenticator, "base64", new Base64(), true);
Base64 base64 = new Base64();
HttpURLConnection conn = Mockito.mock(HttpURLConnection.class);
Mockito.when(conn.getResponseCode()).thenReturn(HttpURLConnection.HTTP_UNAUTHORIZED);
Mockito.when(conn.getHeaderField("www-authenticate"))
.thenReturn(KerberosAuthenticator.NEGOTIATE +
Arrays.toString(base64.encode("foobar".getBytes())));
Method method = KerberosAuthenticator.class.getDeclaredMethod("readToken",
HttpURLConnection.class);
method.setAccessible(true);
method.invoke(kerberosAuthenticator, conn); // expecting this not to throw an exception
}
}

38
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestAuthenticationFilter.java
View File

@ -574,6 +574,44 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
}
}
@Test
public void testDoFilterNotAuthenticatedLowerCase() throws Exception {
AuthenticationFilter filter = new AuthenticationFilter();
try {
FilterConfig config = Mockito.mock(FilterConfig.class);
Mockito.when(config.getInitParameter("management.operation.return")).
thenReturn("true");
Mockito.when(config.getInitParameter(AuthenticationFilter.AUTH_TYPE)).thenReturn(
DummyAuthenticationHandler.class.getName());
Mockito.when(config.getInitParameterNames()).thenReturn(
new Vector<>(
Arrays.asList(AuthenticationFilter.AUTH_TYPE,
"management.operation.return")).elements());
getMockedServletContextWithStringSigner(config);
filter.init(config);
HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
Mockito.when(request.getRequestURL()).thenReturn(new StringBuffer("http://foo:8080/bar"));
HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
FilterChain chain = Mockito.mock(FilterChain.class);
Mockito.doAnswer((Answer<Object>) invocation -> {
Assert.fail();
return null;
}).when(chain).doFilter(any(), any());
Mockito.when(response.containsHeader("www-authenticate")).thenReturn(true);
filter.doFilter(request, response, chain);
Mockito.verify(response).sendError(
HttpServletResponse.SC_UNAUTHORIZED, "Authentication required");
} finally {
filter.destroy();
}
}
private void _testDoFilterAuthentication(boolean withDomainPath,
boolean invalidToken,
boolean expired) throws Exception {

1
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestJWTRedirectAuthenticationHandler.java
View File

@ -25,7 +25,6 @@
import java.util.List;
import java.util.ArrayList;
import java.util.Properties;
import java.util.Vector;
import java.util.Date;
import javax.servlet.ServletException;

1
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/server/TestPseudoAuthenticationHandler.java
View File

@ -13,7 +13,6 @@
*/
package org.apache.hadoop.security.authentication.server;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.junit.Assert;
import org.junit.Test;

1
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/StringSignerSecretProvider.java
View File

@ -18,7 +18,6 @@
import javax.servlet.ServletContext;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;

4
hadoop-common-project/hadoop-auth/src/test/java/org/apache/hadoop/security/authentication/util/TestJaasConfiguration.java
View File

@ -32,8 +32,8 @@ public void test() throws Exception {
krb5LoginModuleName = "com.sun.security.auth.module.Krb5LoginModule";
}
ZKSignerSecretProvider.JaasConfiguration jConf =
new ZKSignerSecretProvider.JaasConfiguration("foo", "foo/localhost",
JaasConfiguration jConf =
new JaasConfiguration("foo", "foo/localhost",
"/some/location/foo.keytab");
AppConfigurationEntry[] entries = jConf.getAppConfigurationEntry("bar");
Assert.assertNull(entries);

15
hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
View File

@ -379,21 +379,6 @@
<Bug code="JLM" />
</Match>
<!--
OpenStack Swift FS module -closes streams in a different method
from where they are opened.
-->
<Match>
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
<Method name="uploadFileAttempt"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
</Match>
<Match>
<Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
<Method name="uploadFilePartAttempt"/>
<Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
</Match>
<!-- code from maven source, null value is checked at callee side. -->
<Match>
<Class name="org.apache.hadoop.util.ComparableVersion$ListItem" />

41055
hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_2.10.2.xml Normal file
View File

File diff suppressed because one or more lines are too long

35426
hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.2.4.xml Normal file
View File

File diff suppressed because one or more lines are too long

39037
hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.3.xml Normal file
View File

File diff suppressed because one or more lines are too long

39037
hadoop-common-project/hadoop-common/dev-support/jdiff/Apache_Hadoop_Common_3.3.4.xml Normal file
View File

File diff suppressed because one or more lines are too long

113
hadoop-common-project/hadoop-common/pom.xml
View File

@ -180,11 +180,6 @@
<artifactId>jersey-server</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@ -200,11 +195,6 @@
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.glassfish.grizzly</groupId>
<artifactId>grizzly-http-servlet</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
@ -383,6 +373,11 @@
<artifactId>mockwebserver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okio</groupId>
<artifactId>okio-jvm</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>dnsjava</groupId>
<artifactId>dnsjava</artifactId>
@ -841,6 +836,36 @@
</execution>
</executions>
</plugin>
<plugin>
<!--Sets the skip.platformToolsetDetection to true if use.platformToolsetVersion is specified.
This implies that the automatic detection of which platform toolset to use will be skipped
and the one specified with use.platformToolsetVersion will be used.-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<target>
<condition property="skip.platformToolsetDetection" value="true" else="false">
<isset property="use.platformToolsetVersion"/>
</condition>
<!--Unfortunately, Maven doesn't have a way to negate a flag, thus we declare a
property which holds the negated value of skip.platformToolsetDetection.-->
<condition property="skip.platformToolsetDetection.negated" value="false" else="true">
<isset property="use.platformToolsetVersion"/>
</condition>
<echo>Skip platform toolset version detection = ${skip.platformToolsetDetection}</echo>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
@ -852,6 +877,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
<arguments>
<argument>${basedir}\src\main\winutils</argument>
@ -866,6 +892,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
@ -878,6 +905,27 @@
</arguments>
</configuration>
</execution>
<execution>
<id>compile-ms-winutils-using-build-tools</id>
<phase>compile</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection.negated}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/winutils/winutils.sln</argument>
<argument>/nologo</argument>
<argument>/p:Configuration=Release</argument>
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>convert-ms-native-dll</id>
<phase>generate-sources</phase>
@ -885,6 +933,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable>
<arguments>
<argument>${basedir}\src\main\native</argument>
@ -899,6 +948,7 @@
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/native/native.sln</argument>
@ -919,6 +969,35 @@
</arguments>
</configuration>
</execution>
<execution>
<id>compile-ms-native-dll-using-build-tools</id>
<phase>compile</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<skip>${skip.platformToolsetDetection.negated}</skip>
<executable>msbuild</executable>
<arguments>
<argument>${basedir}/src/main/native/native.sln</argument>
<argument>/nologo</argument>
<argument>/p:Configuration=Release</argument>
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
<argument>/p:CustomZstdPrefix=${zstd.prefix}</argument>
<argument>/p:CustomZstdLib=${zstd.lib}</argument>
<argument>/p:CustomZstdInclude=${zstd.include}</argument>
<argument>/p:RequireZstd=${require.zstd}</argument>
<argument>/p:CustomOpensslPrefix=${openssl.prefix}</argument>
<argument>/p:CustomOpensslLib=${openssl.lib}</argument>
<argument>/p:CustomOpensslInclude=${openssl.include}</argument>
<argument>/p:RequireOpenssl=${require.openssl}</argument>
<argument>/p:RequireIsal=${require.isal}</argument>
<argument>/p:CustomIsalPrefix=${isal.prefix}</argument>
<argument>/p:CustomIsalLib=${isal.lib}</argument>
<argument>/p:PlatformToolset=${use.platformToolsetVersion}</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
@ -1151,7 +1230,7 @@
<id>src-test-compile-protoc-legacy</id>
<phase>generate-test-sources</phase>
<goals>
<goal>compile</goal>
<goal>test-compile</goal>
</goals>
<configuration>
<skip>false</skip>
@ -1160,7 +1239,7 @@
com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
</protocArtifact>
<includeDependenciesInDescriptorSet>false</includeDependenciesInDescriptorSet>
<protoSourceRoot>${basedir}/src/test/proto</protoSourceRoot>
<protoTestSourceRoot>${basedir}/src/test/proto</protoTestSourceRoot>
<outputDirectory>${project.build.directory}/generated-test-sources/java</outputDirectory>
<clearOutputDirectory>false</clearOutputDirectory>
<includes>
@ -1171,6 +1250,16 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
<sourceFileExcludes>
<sourceFileExclude>**/FSProtos.java</sourceFileExclude>
</sourceFileExcludes>
<excludePackageNames>*.proto:*.tracing:*.protobuf</excludePackageNames>
</configuration>
</plugin>
</plugins>
</build>
</profile>

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
View File

@ -98,7 +98,7 @@ static void writeResponse(Configuration conf,
if (FORMAT_JSON.equals(format)) {
Configuration.dumpConfiguration(conf, propertyName, out);
} else if (FORMAT_XML.equals(format)) {
conf.writeXml(propertyName, out);
conf.writeXml(propertyName, out, conf);
} else {
throw new BadFormatException("Bad format: " + format);
}

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfigRedactor.java
View File

@ -37,6 +37,7 @@
public class ConfigRedactor {
private static final String REDACTED_TEXT = "<redacted>";
private static final String REDACTED_XML = "******";
private List<Pattern> compiledPatterns;
@ -57,8 +58,8 @@ public ConfigRedactor(Configuration conf) {
* Given a key / value pair, decides whether or not to redact and returns
* either the original value or text indicating it has been redacted.
*
* @param key
* @param value
* @param key param key.
* @param value param value, will return if conditions permit.
* @return Original value, or text indicating it has been redacted
*/
public String redact(String key, String value) {
@ -84,4 +85,19 @@ private boolean configIsSensitive(String key) {
}
return false;
}
/**
* Given a key / value pair, decides whether or not to redact and returns
* either the original value or text indicating it has been redacted.
*
* @param key param key.
* @param value param value, will return if conditions permit.
* @return Original value, or text indicating it has been redacted
*/
public String redactXml(String key, String value) {
if (configIsSensitive(key)) {
return REDACTED_XML;
}
return value;
}
}

93
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
View File

@ -24,7 +24,6 @@
import com.ctc.wstx.stax.WstxInputFactory;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import org.apache.hadoop.classification.VisibleForTesting;
import java.io.BufferedInputStream;
import java.io.DataInput;
@ -87,6 +86,7 @@
import org.apache.commons.collections.map.UnmodifiableMap;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@ -98,18 +98,19 @@
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.XMLUtils;
import org.codehaus.stax2.XMLStreamReader2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
@ -317,7 +318,7 @@ private static boolean getRestrictParserDefault(Object resource) {
private boolean loadDefaults = true;
/**
* Configuration objects
* Configuration objects.
*/
private static final WeakHashMap<Configuration,Object> REGISTRY =
new WeakHashMap<Configuration,Object>();
@ -774,7 +775,7 @@ private void updatePropertiesWithDeprecatedKeys(
private void handleDeprecation() {
LOG.debug("Handling deprecation for all properties in config...");
DeprecationContext deprecations = deprecationContext.get();
Set<Object> keys = new HashSet<Object>();
Set<Object> keys = new HashSet<>();
keys.addAll(getProps().keySet());
for (Object item: keys) {
LOG.debug("Handling deprecation for " + (String)item);
@ -1908,6 +1909,7 @@ public long getTimeDuration(String name, String defaultValue,
* @param name Property name
* @param vStr The string value with time unit suffix to be converted.
* @param unit Unit to convert the stored property, if it exists.
* @return time duration in given time unit.
*/
public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
return getTimeDurationHelper(name, vStr, unit, unit);
@ -1922,6 +1924,7 @@ public long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
* @param vStr The string value with time unit suffix to be converted.
* @param defaultUnit Unit to convert the stored property, if it exists.
* @param returnUnit Unit for the returned value.
* @return time duration in given time unit.
*/
private long getTimeDurationHelper(String name, String vStr,
TimeUnit defaultUnit, TimeUnit returnUnit) {
@ -2206,7 +2209,7 @@ private static int convertToInt(String value, int defaultValue) {
}
/**
* Is the given value in the set of ranges
* Is the given value in the set of ranges.
* @param value the value to check
* @return is the value in the ranges?
*/
@ -2263,7 +2266,7 @@ public Iterator<Integer> iterator() {
}
/**
* Parse the given attribute as a set of integer ranges
* Parse the given attribute as a set of integer ranges.
* @param name the attribute name
* @param defaultValue the default value if it is not set
* @return a new set of ranges from the configured value
@ -2482,7 +2485,7 @@ public char[] getPasswordFromCredentialProviders(String name)
/**
* Fallback to clear text passwords in configuration.
* @param name
* @param name the property name.
* @return clear text password or null
*/
protected char[] getPasswordFromConfig(String name) {
@ -2547,6 +2550,8 @@ public InetSocketAddress getSocketAddr(
/**
* Set the socket address for the <code>name</code> property as
* a <code>host:port</code>.
* @param name property name.
* @param addr inetSocketAddress addr.
*/
public void setSocketAddr(String name, InetSocketAddress addr) {
set(name, NetUtils.getHostPortString(addr));
@ -2724,6 +2729,7 @@ public Class<?> getClass(String name, Class<?> defaultValue) {
* @param name the conf key name.
* @param defaultValue default value.
* @param xface the interface implemented by the named class.
* @param <U> Interface class type.
* @return property value as a <code>Class</code>,
* or <code>defaultValue</code>.
*/
@ -2753,6 +2759,7 @@ else if (theClass != null)
* @param name the property name.
* @param xface the interface implemented by the classes named by
* <code>name</code>.
* @param <U> Interface class type.
* @return a <code>List</code> of objects implementing <code>xface</code>.
*/
@SuppressWarnings("unchecked")
@ -2785,15 +2792,16 @@ public void setClass(String name, Class<?> theClass, Class<?> xface) {
set(name, theClass.getName());
}
/**
/**
* Get a local file under a directory named by <i>dirsProp</i> with
* the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
* then one is chosen based on <i>path</i>'s hash code. If the selected
* directory does not exist, an attempt is made to create it.
*
*
* @param dirsProp directory in which to locate the file.
* @param path file-path.
* @return local file under the directory with the given path.
* @throws IOException raised on errors performing I/O.
*/
public Path getLocalPath(String dirsProp, String path)
throws IOException {
@ -2817,15 +2825,16 @@ public Path getLocalPath(String dirsProp, String path)
throw new IOException("No valid local directories in property: "+dirsProp);
}
/**
/**
* Get a local file name under a directory named in <i>dirsProp</i> with
* the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
* then one is chosen based on <i>path</i>'s hash code. If the selected
* directory does not exist, an attempt is made to create it.
*
*
* @param dirsProp directory in which to locate the file.
* @param path file-path.
* @return local file under the directory with the given path.
* @throws IOException raised on errors performing I/O.
*/
public File getFile(String dirsProp, String path)
throws IOException {
@ -3437,7 +3446,7 @@ void parseNext() throws IOException, XMLStreamException {
/**
* Add tags defined in HADOOP_TAGS_SYSTEM, HADOOP_TAGS_CUSTOM.
* @param prop
* @param prop properties.
*/
public void addTags(Properties prop) {
// Get all system tags
@ -3538,7 +3547,7 @@ private void loadProperty(Properties properties, String name, String attr,
/**
* Print a warning if a property with a given name already exists with a
* different value
* different value.
*/
private void checkForOverride(Properties properties, String name, String attr, String value) {
String propertyValue = properties.getProperty(attr);
@ -3548,11 +3557,12 @@ private void checkForOverride(Properties properties, String name, String attr, S
}
}
/**
/**
* Write out the non-default properties in this configuration to the given
* {@link OutputStream} using UTF-8 encoding.
*
*
* @param out the output stream to write to.
* @throws IOException raised on errors performing I/O.
*/
public void writeXml(OutputStream out) throws IOException {
writeXml(new OutputStreamWriter(out, "UTF-8"));
@ -3582,16 +3592,20 @@ public void writeXml(Writer out) throws IOException {
* the configuration, this method throws an {@link IllegalArgumentException}.
* </li>
* </ul>
* @param propertyName xml property name.
* @param out the writer to write to.
* @param config configuration.
* @throws IOException raised on errors performing I/O.
*/
public void writeXml(@Nullable String propertyName, Writer out)
public void writeXml(@Nullable String propertyName, Writer out, Configuration config)
throws IOException, IllegalArgumentException {
Document doc = asXmlDocument(propertyName);
ConfigRedactor redactor = config != null ? new ConfigRedactor(this) : null;
Document doc = asXmlDocument(propertyName, redactor);
try {
DOMSource source = new DOMSource(doc);
StreamResult result = new StreamResult(out);
TransformerFactory transFactory = TransformerFactory.newInstance();
TransformerFactory transFactory = XMLUtils.newSecureTransformerFactory();
Transformer transformer = transFactory.newTransformer();
// Important to not hold Configuration log while writing result, since
@ -3603,11 +3617,16 @@ public void writeXml(@Nullable String propertyName, Writer out)
}
}
public void writeXml(@Nullable String propertyName, Writer out)
throws IOException, IllegalArgumentException {
writeXml(propertyName, out, null);
}
/**
* Return the XML DOM corresponding to this Configuration.
*/
private synchronized Document asXmlDocument(@Nullable String propertyName)
throws IOException, IllegalArgumentException {
private synchronized Document asXmlDocument(@Nullable String propertyName,
ConfigRedactor redactor) throws IOException, IllegalArgumentException {
Document doc;
try {
doc = DocumentBuilderFactory
@ -3630,13 +3649,13 @@ private synchronized Document asXmlDocument(@Nullable String propertyName)
propertyName + " not found");
} else {
// given property is found, write single property
appendXMLProperty(doc, conf, propertyName);
appendXMLProperty(doc, conf, propertyName, redactor);
conf.appendChild(doc.createTextNode("\n"));
}
} else {
// append all elements
for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
appendXMLProperty(doc, conf, (String)e.nextElement());
appendXMLProperty(doc, conf, (String)e.nextElement(), redactor);
conf.appendChild(doc.createTextNode("\n"));
}
}
@ -3652,7 +3671,7 @@ private synchronized Document asXmlDocument(@Nullable String propertyName)
* @param propertyName
*/
private synchronized void appendXMLProperty(Document doc, Element conf,
String propertyName) {
String propertyName, ConfigRedactor redactor) {
// skip writing if given property name is empty or null
if (!Strings.isNullOrEmpty(propertyName)) {
String value = properties.getProperty(propertyName);
@ -3665,8 +3684,11 @@ private synchronized void appendXMLProperty(Document doc, Element conf,
propNode.appendChild(nameNode);
Element valueNode = doc.createElement("value");
valueNode.appendChild(doc.createTextNode(
properties.getProperty(propertyName)));
String propertyValue = properties.getProperty(propertyName);
if (redactor != null) {
propertyValue = redactor.redactXml(propertyName, propertyValue);
}
valueNode.appendChild(doc.createTextNode(propertyValue));
propNode.appendChild(valueNode);
Element finalNode = doc.createElement("final");
@ -3736,7 +3758,7 @@ private synchronized void appendXMLProperty(Document doc, Element conf,
* @param config the configuration
* @param propertyName property name
* @param out the Writer to write to
* @throws IOException
* @throws IOException raised on errors performing I/O.
* @throws IllegalArgumentException when property name is not
* empty and the property is not found in configuration
**/
@ -3783,7 +3805,7 @@ public static void dumpConfiguration(Configuration config,
*
* @param config the configuration
* @param out the Writer to write to
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static void dumpConfiguration(Configuration config,
Writer out) throws IOException {
@ -3812,7 +3834,7 @@ public static void dumpConfiguration(Configuration config,
* @param jsonGen json writer
* @param config configuration
* @param name property name
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
private static void appendJSONProperty(JsonGenerator jsonGen,
Configuration config, String name, ConfigRedactor redactor)
@ -3894,7 +3916,10 @@ synchronized boolean getQuietMode() {
return this.quietmode;
}
/** For debugging. List non-default properties to the terminal and exit. */
/** For debugging. List non-default properties to the terminal and exit.
* @param args the argument to be parsed.
* @throws Exception exception.
*/
public static void main(String[] args) throws Exception {
new Configuration().writeXml(System.out);
}
@ -3928,8 +3953,8 @@ public void write(DataOutput out) throws IOException {
}
/**
* get keys matching the the regex
* @param regex
* get keys matching the the regex.
* @param regex the regex to match against.
* @return {@literal Map<String,String>} with matching keys
*/
public Map<String,String> getValByRegex(String regex) {
@ -3974,6 +3999,8 @@ public static void dumpDeprecatedKeys() {
/**
* Returns whether or not a deprecated name has been warned. If the name is not
* deprecated then always return false
* @param name proprties.
* @return true if name is a warned deprecation.
*/
public static boolean hasWarnedDeprecation(String name) {
DeprecationContext deprecations = deprecationContext.get();

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configured.java
View File

@ -33,7 +33,9 @@ public Configured() {
this(null);
}
/** Construct a Configured. */
/** Construct a Configured.
* @param conf the Configuration object.
*/
public Configured(Configuration conf) {
setConf(conf);
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Reconfigurable.java
View File

@ -33,6 +33,9 @@ public interface Reconfigurable extends Configurable {
* (or null if it was not previously set). If newVal is null, set the property
* to its default value;
*
* @param property property name.
* @param newVal new value.
* @throws ReconfigurationException if there was an error applying newVal.
* If the property cannot be changed, throw a
* {@link ReconfigurationException}.
*/
@ -45,11 +48,14 @@ void reconfigureProperty(String property, String newVal)
* If isPropertyReconfigurable returns true for a property,
* then changeConf should not throw an exception when changing
* this property.
* @param property property name.
* @return true if property reconfigurable; false if not.
*/
boolean isPropertyReconfigurable(String property);
/**
* Return all the properties that can be changed at run time.
* @return reconfigurable propertys.
*/
Collection<String> getReconfigurableProperties();
}

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurableBase.java
View File

@ -79,6 +79,7 @@ public ReconfigurableBase() {
/**
* Construct a ReconfigurableBase with the {@link Configuration}
* conf.
* @param conf configuration.
*/
public ReconfigurableBase(Configuration conf) {
super((conf == null) ? new Configuration() : conf);
@ -91,6 +92,7 @@ public void setReconfigurationUtil(ReconfigurationUtil ru) {
/**
* Create a new configuration.
* @return configuration.
*/
protected abstract Configuration getNewConf();
@ -162,6 +164,7 @@ public void run() {
/**
* Start a reconfiguration task to reload configuration in background.
* @throws IOException raised on errors performing I/O.
*/
public void startReconfigurationTask() throws IOException {
synchronized (reconfigLock) {

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationException.java
View File

@ -59,6 +59,10 @@ public ReconfigurationException() {
/**
* Create a new instance of {@link ReconfigurationException}.
* @param property property name.
* @param newVal new value.
* @param oldVal old value.
* @param cause original exception.
*/
public ReconfigurationException(String property,
String newVal, String oldVal,
@ -71,6 +75,9 @@ public ReconfigurationException(String property,
/**
* Create a new instance of {@link ReconfigurationException}.
* @param property property name.
* @param newVal new value.
* @param oldVal old value.
*/
public ReconfigurationException(String property,
String newVal, String oldVal) {
@ -82,6 +89,7 @@ public ReconfigurationException(String property,
/**
* Get property that cannot be changed.
* @return property info.
*/
public String getProperty() {
return property;
@ -89,6 +97,7 @@ public String getProperty() {
/**
* Get value to which property was supposed to be changed.
* @return new value.
*/
public String getNewValue() {
return newVal;
@ -96,6 +105,7 @@ public String getNewValue() {
/**
* Get old value of property that cannot be changed.
* @return old value.
*/
public String getOldValue() {
return oldVal;

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ReconfigurationTaskStatus.java
View File

@ -42,7 +42,8 @@ public ReconfigurationTaskStatus(long startTime, long endTime,
/**
* Return true if
* - A reconfiguration task has finished or
* - an active reconfiguration task is running
* - an active reconfiguration task is running.
* @return true if startTime &gt; 0; false if not.
*/
public boolean hasTask() {
return startTime > 0;
@ -51,6 +52,7 @@ public boolean hasTask() {
/**
* Return true if the latest reconfiguration task has finished and there is
* no another active task running.
* @return true if endTime &gt; 0; false if not.
*/
public boolean stopped() {
return endTime > 0;

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoCodec.java
View File

@ -145,14 +145,18 @@ private static List<Class<? extends CryptoCodec>> getCodecClasses(
public abstract CipherSuite getCipherSuite();
/**
* Create a {@link org.apache.hadoop.crypto.Encryptor}.
* @return Encryptor the encryptor
* Create a {@link org.apache.hadoop.crypto.Encryptor}.
*
* @return Encryptor the encryptor.
* @throws GeneralSecurityException thrown if create encryptor error.
*/
public abstract Encryptor createEncryptor() throws GeneralSecurityException;
/**
* Create a {@link org.apache.hadoop.crypto.Decryptor}.
*
* @return Decryptor the decryptor
* @throws GeneralSecurityException thrown if create decryptor error.
*/
public abstract Decryptor createDecryptor() throws GeneralSecurityException;

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
View File

@ -157,7 +157,7 @@ public InputStream getWrappedStream() {
* @param off the buffer offset.
* @param len the maximum number of decrypted data bytes to read.
* @return int the total number of decrypted data bytes read into the buffer.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {

15
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoOutputStream.java
View File

@ -146,7 +146,7 @@ public OutputStream getWrappedStream() {
* @param b the data.
* @param off the start offset in the data.
* @param len the number of bytes to write.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@Override
public synchronized void write(byte[] b, int off, int len) throws IOException {
@ -241,12 +241,15 @@ public synchronized void close() throws IOException {
return;
}
try {
flush();
if (closeOutputStream) {
super.close();
codec.close();
try {
flush();
} finally {
if (closeOutputStream) {
super.close();
codec.close();
}
freeBuffers();
}
freeBuffers();
} finally {
closed = true;
}

35
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoStreamUtils.java
View File

@ -39,7 +39,11 @@ public class CryptoStreamUtils {
private static final Logger LOG =
LoggerFactory.getLogger(CryptoStreamUtils.class);
/** Forcibly free the direct buffer. */
/**
* Forcibly free the direct buffer.
*
* @param buffer buffer.
*/
public static void freeDB(ByteBuffer buffer) {
if (CleanerUtil.UNMAP_SUPPORTED) {
try {
@ -52,13 +56,22 @@ public static void freeDB(ByteBuffer buffer) {
}
}
/** Read crypto buffer size */
/**
* Read crypto buffer size.
*
* @param conf configuration.
* @return hadoop.security.crypto.buffer.size.
*/
public static int getBufferSize(Configuration conf) {
return conf.getInt(HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_KEY,
HADOOP_SECURITY_CRYPTO_BUFFER_SIZE_DEFAULT);
}
/** AES/CTR/NoPadding or SM4/CTR/NoPadding is required. */
/**
* AES/CTR/NoPadding or SM4/CTR/NoPadding is required.
*
* @param codec crypto codec.
*/
public static void checkCodec(CryptoCodec codec) {
if (codec.getCipherSuite() != CipherSuite.AES_CTR_NOPADDING &&
codec.getCipherSuite() != CipherSuite.SM4_CTR_NOPADDING) {
@ -67,17 +80,27 @@ public static void checkCodec(CryptoCodec codec) {
}
}
/** Check and floor buffer size */
/**
* Check and floor buffer size.
*
* @param codec crypto codec.
* @param bufferSize the size of the buffer to be used.
* @return calc buffer size.
*/
public static int checkBufferSize(CryptoCodec codec, int bufferSize) {
Preconditions.checkArgument(bufferSize >= MIN_BUFFER_SIZE,
"Minimum value of buffer size is " + MIN_BUFFER_SIZE + ".");
return bufferSize - bufferSize % codec.getCipherSuite()
.getAlgorithmBlockSize();
}
/**
* If input stream is {@link org.apache.hadoop.fs.Seekable}, return it's
* current position, otherwise return 0;
*
* @param in wrapper.
* @return current position, otherwise return 0.
* @throws IOException raised on errors performing I/O.
*/
public static long getInputStreamOffset(InputStream in) throws IOException {
if (in instanceof Seekable) {

23
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/OpensslCipher.java
View File

@ -225,34 +225,33 @@ public int update(ByteBuffer input, ByteBuffer output)
output.position(output.position() + len);
return len;
}
/**
* Finishes a multiple-part operation. The data is encrypted or decrypted,
* depending on how this cipher was initialized.
* <p>
*
* The result is stored in the output buffer. Upon return, the output buffer's
* position will have advanced by n, where n is the value returned by this
* method; the output buffer's limit will not have changed.
* <p>
*
* </p>
* If <code>output.remaining()</code> bytes are insufficient to hold the result,
* a <code>ShortBufferException</code> is thrown.
* <p>
*
* Upon finishing, this method resets this cipher object to the state it was
* in when previously initialized. That is, the object is available to encrypt
* or decrypt more data.
* <p>
*
* If any exception is thrown, this cipher object need to be reset before it
* </p>
* If any exception is thrown, this cipher object need to be reset before it
* can be used again.
*
*
* @param output the output ByteBuffer
* @return int number of bytes stored in <code>output</code>
* @throws ShortBufferException
* @throws IllegalBlockSizeException
* @throws BadPaddingException
* @throws ShortBufferException if there is insufficient space in the output buffer.
* @throws IllegalBlockSizeException This exception is thrown when the length
* of data provided to a block cipher is incorrect.
* @throws BadPaddingException This exception is thrown when a particular
* padding mechanism is expected for the input
* data but the data is not padded properly.
*/
public int doFinal(ByteBuffer output) throws ShortBufferException,
IllegalBlockSizeException, BadPaddingException {

51
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
View File

@ -242,7 +242,7 @@ protected int addVersion() {
/**
* Serialize the metadata to a set of bytes.
* @return the serialized bytes
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
protected byte[] serialize() throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
@ -281,7 +281,7 @@ protected byte[] serialize() throws IOException {
/**
* Deserialize a new metadata object from a set of bytes.
* @param bytes the serialized metadata
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
protected Metadata(byte[] bytes) throws IOException {
String cipher = null;
@ -450,7 +450,7 @@ public boolean isTransient() {
* when decrypting data.
* @param versionName the name of a specific version of the key
* @return the key material
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract KeyVersion getKeyVersion(String versionName
) throws IOException;
@ -458,14 +458,15 @@ public abstract KeyVersion getKeyVersion(String versionName
/**
* Get the key names for all keys.
* @return the list of key names
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract List<String> getKeys() throws IOException;
/**
* Get key metadata in bulk.
* @param names the names of the keys to get
* @throws IOException
* @throws IOException raised on errors performing I/O.
* @return Metadata Array.
*/
public Metadata[] getKeysMetadata(String... names) throws IOException {
Metadata[] result = new Metadata[names.length];
@ -477,8 +478,10 @@ public Metadata[] getKeysMetadata(String... names) throws IOException {
/**
* Get the key material for all versions of a specific key name.
*
* @param name the base name of the key.
* @return the list of key material
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract List<KeyVersion> getKeyVersions(String name) throws IOException;
@ -488,7 +491,7 @@ public Metadata[] getKeysMetadata(String... names) throws IOException {
* @param name the base name of the key
* @return the version name of the current version of the key or null if the
* key version doesn't exist
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public KeyVersion getCurrentKey(String name) throws IOException {
Metadata meta = getMetadata(name);
@ -502,7 +505,7 @@ public KeyVersion getCurrentKey(String name) throws IOException {
* Get metadata about the key.
* @param name the basename of the key
* @return the key's metadata or null if the key doesn't exist
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract Metadata getMetadata(String name) throws IOException;
@ -512,7 +515,7 @@ public KeyVersion getCurrentKey(String name) throws IOException {
* @param material the key material for the first version of the key.
* @param options the options for the new key.
* @return the version name of the first version of the key.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract KeyVersion createKey(String name, byte[] material,
Options options) throws IOException;
@ -537,7 +540,7 @@ private String getAlgorithm(String cipher) {
* @param size length of the key.
* @param algorithm algorithm to use for generating the key.
* @return the generated key.
* @throws NoSuchAlgorithmException
* @throws NoSuchAlgorithmException no such algorithm exception.
*/
protected byte[] generateKey(int size, String algorithm)
throws NoSuchAlgorithmException {
@ -558,8 +561,8 @@ protected byte[] generateKey(int size, String algorithm)
* @param name the base name of the key
* @param options the options for the new key.
* @return the version name of the first version of the key.
* @throws IOException
* @throws NoSuchAlgorithmException
* @throws IOException raised on errors performing I/O.
* @throws NoSuchAlgorithmException no such algorithm exception.
*/
public KeyVersion createKey(String name, Options options)
throws NoSuchAlgorithmException, IOException {
@ -570,7 +573,7 @@ public KeyVersion createKey(String name, Options options)
/**
* Delete the given key.
* @param name the name of the key to delete
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract void deleteKey(String name) throws IOException;
@ -579,7 +582,7 @@ public KeyVersion createKey(String name, Options options)
* @param name the basename of the key
* @param material the new key material
* @return the name of the new version of the key
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract KeyVersion rollNewVersion(String name,
byte[] material
@ -601,7 +604,10 @@ public void close() throws IOException {
*
* @param name the basename of the key
* @return the name of the new version of the key
* @throws IOException
* @throws IOException raised on errors performing I/O.
* @throws NoSuchAlgorithmException This exception is thrown when a particular
* cryptographic algorithm is requested
* but is not available in the environment.
*/
public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
IOException {
@ -620,7 +626,7 @@ public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
* version of the given key.
*
* @param name the basename of the key
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void invalidateCache(String name) throws IOException {
// NOP
@ -628,18 +634,19 @@ public void invalidateCache(String name) throws IOException {
/**
* Ensures that any changes to the keys are written to persistent store.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public abstract void flush() throws IOException;
/**
* Split the versionName in to a base name. Converts "/aaa/bbb/3" to
* Split the versionName in to a base name. Converts "/aaa/bbb@3" to
* "/aaa/bbb".
* @param versionName the version name to split
* @return the base name of the key
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static String getBaseName(String versionName) throws IOException {
Objects.requireNonNull(versionName, "VersionName cannot be null");
int div = versionName.lastIndexOf('@');
if (div == -1) {
throw new IOException("No version in key path " + versionName);
@ -660,9 +667,11 @@ protected static String buildVersionName(String name, int version) {
/**
* Find the provider with the given key.
*
* @param providerList the list of providers
* @param keyName the key name we are looking for
* @param keyName the key name we are looking for.
* @return the KeyProvider that has the key
* @throws IOException raised on errors performing I/O.
*/
public static KeyProvider findProvider(List<KeyProvider> providerList,
String keyName) throws IOException {
@ -680,7 +689,7 @@ public static KeyProvider findProvider(List<KeyProvider> providerList,
* means. If true, the password should be provided by the caller using
* setPassword().
* @return Whether or not the provider requires a password
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public boolean needsPassword() throws IOException {
return false;

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
View File

@ -25,10 +25,6 @@
import java.util.List;
import java.util.ListIterator;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.CryptoCodec;
@ -178,6 +174,7 @@ public interface CryptoExtension extends KeyProviderExtension.Extension {
* Calls to this method allows the underlying KeyProvider to warm-up any
* implementation specific caches used to store the Encrypted Keys.
* @param keyNames Array of Key Names
* @throws IOException thrown if the key material could not be encrypted.
*/
public void warmUpEncryptedKeys(String... keyNames)
throws IOException;
@ -474,8 +471,9 @@ public void drain(String keyName) {
/**
* This constructor is to be used by sub classes that provide
* delegating/proxying functionality to the {@link KeyProviderCryptoExtension}
* @param keyProvider
* @param extension
*
* @param keyProvider key provider.
* @param extension crypto extension.
*/
protected KeyProviderCryptoExtension(KeyProvider keyProvider,
CryptoExtension extension) {
@ -486,6 +484,7 @@ protected KeyProviderCryptoExtension(KeyProvider keyProvider,
* Notifies the Underlying CryptoExtension implementation to warm up any
* implementation specific caches for the specified KeyVersions
* @param keyNames Arrays of key Names
* @throws IOException raised on errors performing I/O.
*/
public void warmUpEncryptedKeys(String... keyNames)
throws IOException {
@ -557,7 +556,7 @@ public EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
* Calls {@link CryptoExtension#drain(String)} for the given key name on the
* underlying {@link CryptoExtension}.
*
* @param keyName
* @param keyName key name.
*/
public void drain(String keyName) {
getExtension().drain(keyName);

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderDelegationTokenExtension.java
View File

@ -48,14 +48,14 @@ public interface DelegationTokenExtension
* Renews the given token.
* @param token The token to be renewed.
* @return The token's lifetime after renewal, or 0 if it can't be renewed.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
long renewDelegationToken(final Token<?> token) throws IOException;
/**
* Cancels the given token.
* @param token The token to be cancelled.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
Void cancelDelegationToken(final Token<?> token) throws IOException;

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
View File

@ -75,7 +75,7 @@ public class KeyShell extends CommandShell {
* </pre>
* @param args Command line arguments.
* @return 0 on success, 1 on failure.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@Override
protected int init(String[] args) throws IOException {
@ -547,7 +547,7 @@ private String prettifyException(Exception e) {
* success and 1 for failure.
*
* @param args Command line arguments.
* @throws Exception
* @throws Exception raised on errors performing I/O.
*/
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new KeyShell(), args);

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
View File

@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.net.SocketException;
import java.net.URI;
import java.security.GeneralSecurityException;
import java.security.NoSuchAlgorithmException;
@ -29,7 +30,7 @@
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
@ -182,10 +183,10 @@ private <T> T doOp(ProviderCallable<T> op, int currPos,
} catch (IOException ioe) {
LOG.warn("KMS provider at [{}] threw an IOException: ",
provider.getKMSUrl(), ioe);
// SSLHandshakeException can occur here because of lost connection
// SSLException can occur here because of lost connection
// with the KMS server, creating a ConnectException from it,
// so that the FailoverOnNetworkExceptionRetry policy will retry
if (ioe instanceof SSLHandshakeException) {
if (ioe instanceof SSLException || ioe instanceof SocketException) {
Exception cause = ioe;
ioe = new ConnectException("SSLHandshakeException: "
+ cause.getMessage());

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
View File

@ -63,7 +63,7 @@ public interface QueueRefiller <E> {
* @param keyName Key name
* @param keyQueue Queue that needs to be filled
* @param numValues number of Values to be added to the queue.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void fillQueueForKey(String keyName,
Queue<E> keyQueue, int numValues) throws IOException;
@ -268,7 +268,7 @@ public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
* Initializes the Value Queues for the provided keys by calling the
* fill Method with "numInitValues" values
* @param keyNames Array of key Names
* @throws ExecutionException
* @throws ExecutionException executionException.
*/
public void initializeQueuesForKeys(String... keyNames)
throws ExecutionException {
@ -285,8 +285,8 @@ public void initializeQueuesForKeys(String... keyNames)
* function to add 1 value to Queue and then drain it.
* @param keyName String key name
* @return E the next value in the Queue
* @throws IOException
* @throws ExecutionException
* @throws IOException raised on errors performing I/O.
* @throws ExecutionException executionException.
*/
public E getNext(String keyName)
throws IOException, ExecutionException {
@ -344,8 +344,8 @@ public int getSize(String keyName) {
* @param keyName String key name
* @param num Minimum number of values to return.
* @return {@literal List<E>} values returned
* @throws IOException
* @throws ExecutionException
* @throws IOException raised on errors performing I/O.
* @throws ExecutionException execution exception.
*/
public List<E> getAtMost(String keyName, int num) throws IOException,
ExecutionException {

277
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
View File

@ -272,7 +272,7 @@ public static AbstractFileSystem get(final URI uri, final Configuration conf)
* @param supportedScheme the scheme supported by the implementor
* @param authorityNeeded if true then theURI must have authority, if false
* then the URI must have null authority.
*
* @param defaultPort default port to use if port is not specified in the URI.
* @throws URISyntaxException <code>uri</code> has syntax error
*/
public AbstractFileSystem(final URI uri, final String supportedScheme,
@ -281,11 +281,12 @@ public AbstractFileSystem(final URI uri, final String supportedScheme,
myUri = getUri(uri, supportedScheme, authorityNeeded, defaultPort);
statistics = getStatistics(uri);
}
/**
* Check that the Uri's scheme matches
* @param uri
* @param supportedScheme
* Check that the Uri's scheme matches.
*
* @param uri name URI of the FS.
* @param supportedScheme supported scheme.
*/
public void checkScheme(URI uri, String supportedScheme) {
String scheme = uri.getScheme();
@ -362,7 +363,7 @@ public URI getUri() {
* If the path is fully qualified URI, then its scheme and authority
* matches that of this file system. Otherwise the path must be
* slash-relative name.
*
* @param path the path.
* @throws InvalidPathException if the path is invalid
*/
public void checkPath(Path path) {
@ -431,7 +432,7 @@ public String getUriPath(final Path p) {
/**
* Make the path fully qualified to this file system
* @param path
* @param path the path.
* @return the qualified path
*/
public Path makeQualified(Path path) {
@ -496,9 +497,9 @@ public FsServerDefaults getServerDefaults(final Path f) throws IOException {
* through any internal symlinks or mount point
* @param p path to be resolved
* @return fully qualified path
* @throws FileNotFoundException
* @throws AccessControlException
* @throws IOException
* @throws FileNotFoundException when file not find throw.
* @throws AccessControlException when accees control error throw.
* @throws IOException raised on errors performing I/O.
* @throws UnresolvedLinkException if symbolic link on path cannot be
* resolved internally
*/
@ -513,6 +514,18 @@ public Path resolvePath(final Path p) throws FileNotFoundException,
* {@link FileContext#create(Path, EnumSet, Options.CreateOpts...)} except
* that the Path f must be fully qualified and the permission is absolute
* (i.e. umask has been applied).
*
* @param f the path.
* @param createFlag create_flag.
* @param opts create ops.
* @throws AccessControlException access controll exception.
* @throws FileAlreadyExistsException file already exception.
* @throws FileNotFoundException file not found exception.
* @throws ParentNotDirectoryException parent not dir exception.
* @throws UnsupportedFileSystemException unsupported file system exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return output stream.
*/
public final FSDataOutputStream create(final Path f,
final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
@ -630,6 +643,24 @@ public final FSDataOutputStream create(final Path f,
* The specification of this method matches that of
* {@link #create(Path, EnumSet, Options.CreateOpts...)} except that the opts
* have been declared explicitly.
*
* @param f the path.
* @param flag create flag.
* @param absolutePermission absolute permission.
* @param bufferSize buffer size.
* @param replication replications.
* @param blockSize block size.
* @param progress progress.
* @param checksumOpt check sum opt.
* @param createParent create parent.
* @throws AccessControlException access control exception.
* @throws FileAlreadyExistsException file already exists exception.
* @throws FileNotFoundException file not found exception.
* @throws ParentNotDirectoryException parent not directory exception.
* @throws UnsupportedFileSystemException unsupported filesystem exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return output stream.
*/
public abstract FSDataOutputStream createInternal(Path f,
EnumSet<CreateFlag> flag, FsPermission absolutePermission,
@ -644,6 +675,14 @@ public abstract FSDataOutputStream createInternal(Path f,
* {@link FileContext#mkdir(Path, FsPermission, boolean)} except that the Path
* f must be fully qualified and the permission is absolute (i.e.
* umask has been applied).
* @param dir directory.
* @param permission permission.
* @param createParent create parent flag.
* @throws AccessControlException access control exception.
* @throws FileAlreadyExistsException file already exists exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void mkdir(final Path dir, final FsPermission permission,
final boolean createParent) throws AccessControlException,
@ -654,6 +693,14 @@ public abstract void mkdir(final Path dir, final FsPermission permission,
* The specification of this method matches that of
* {@link FileContext#delete(Path, boolean)} except that Path f must be for
* this file system.
*
* @param f the path.
* @param recursive recursive flag.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return if successfully deleted success true, not false.
*/
public abstract boolean delete(final Path f, final boolean recursive)
throws AccessControlException, FileNotFoundException,
@ -663,6 +710,13 @@ public abstract boolean delete(final Path f, final boolean recursive)
* The specification of this method matches that of
* {@link FileContext#open(Path)} except that Path f must be for this
* file system.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return input stream.
*/
public FSDataInputStream open(final Path f) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
@ -673,6 +727,14 @@ public FSDataInputStream open(final Path f) throws AccessControlException,
* The specification of this method matches that of
* {@link FileContext#open(Path, int)} except that Path f must be for this
* file system.
*
* @param f the path.
* @param bufferSize buffer size.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return if successfully open success true, not false.
*/
public abstract FSDataInputStream open(final Path f, int bufferSize)
throws AccessControlException, FileNotFoundException,
@ -682,6 +744,14 @@ public abstract FSDataInputStream open(final Path f, int bufferSize)
* The specification of this method matches that of
* {@link FileContext#truncate(Path, long)} except that Path f must be for
* this file system.
*
* @param f the path.
* @param newLength new length.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return if successfully truncate success true, not false.
*/
public boolean truncate(Path f, long newLength)
throws AccessControlException, FileNotFoundException,
@ -694,6 +764,14 @@ public boolean truncate(Path f, long newLength)
* The specification of this method matches that of
* {@link FileContext#setReplication(Path, short)} except that Path f must be
* for this file system.
*
* @param f the path.
* @param replication replication.
* @return if successfully set replication success true, not false.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract boolean setReplication(final Path f,
final short replication) throws AccessControlException,
@ -703,6 +781,16 @@ public abstract boolean setReplication(final Path f,
* The specification of this method matches that of
* {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
* f must be for this file system.
*
* @param src src.
* @param dst dst.
* @param options options.
* @throws AccessControlException access control exception.
* @throws FileAlreadyExistsException file already exists exception.
* @throws FileNotFoundException file not found exception.
* @throws ParentNotDirectoryException parent not directory exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public final void rename(final Path src, final Path dst,
final Options.Rename... options) throws AccessControlException,
@ -727,6 +815,15 @@ public final void rename(final Path src, final Path dst,
* File systems that do not have a built in overwrite need implement only this
* method and can take advantage of the default impl of the other
* {@link #renameInternal(Path, Path, boolean)}
*
* @param src src.
* @param dst dst.
* @throws AccessControlException access control exception.
* @throws FileAlreadyExistsException file already exists exception.
* @throws FileNotFoundException file not found exception.
* @throws ParentNotDirectoryException parent not directory exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void renameInternal(final Path src, final Path dst)
throws AccessControlException, FileAlreadyExistsException,
@ -737,6 +834,16 @@ public abstract void renameInternal(final Path src, final Path dst)
* The specification of this method matches that of
* {@link FileContext#rename(Path, Path, Options.Rename...)} except that Path
* f must be for this file system.
*
* @param src src.
* @param dst dst.
* @param overwrite overwrite flag.
* @throws AccessControlException access control exception.
* @throws FileAlreadyExistsException file already exists exception.
* @throws FileNotFoundException file not found exception.
* @throws ParentNotDirectoryException parent not directory exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public void renameInternal(final Path src, final Path dst,
boolean overwrite) throws AccessControlException,
@ -800,6 +907,12 @@ public boolean supportsSymlinks() {
/**
* The specification of this method matches that of
* {@link FileContext#createSymlink(Path, Path, boolean)};
*
* @param target target.
* @param link link.
* @param createParent create parent.
* @throws IOException raised on errors performing I/O.
* @throws UnresolvedLinkException unresolved link exception.
*/
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws IOException, UnresolvedLinkException {
@ -810,6 +923,8 @@ public void createSymlink(final Path target, final Path link,
* Partially resolves the path. This is used during symlink resolution in
* {@link FSLinkResolver}, and differs from the similarly named method
* {@link FileContext#getLinkTarget(Path)}.
* @param f the path.
* @return target path.
* @throws IOException subclass implementations may throw IOException
*/
public Path getLinkTarget(final Path f) throws IOException {
@ -822,6 +937,13 @@ public Path getLinkTarget(final Path f) throws IOException {
* The specification of this method matches that of
* {@link FileContext#setPermission(Path, FsPermission)} except that Path f
* must be for this file system.
*
* @param f the path.
* @param permission permission.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void setPermission(final Path f,
final FsPermission permission) throws AccessControlException,
@ -831,6 +953,14 @@ public abstract void setPermission(final Path f,
* The specification of this method matches that of
* {@link FileContext#setOwner(Path, String, String)} except that Path f must
* be for this file system.
*
* @param f the path.
* @param username username.
* @param groupname groupname.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void setOwner(final Path f, final String username,
final String groupname) throws AccessControlException,
@ -840,6 +970,14 @@ public abstract void setOwner(final Path f, final String username,
* The specification of this method matches that of
* {@link FileContext#setTimes(Path, long, long)} except that Path f must be
* for this file system.
*
* @param f the path.
* @param mtime modify time.
* @param atime access time.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void setTimes(final Path f, final long mtime,
final long atime) throws AccessControlException, FileNotFoundException,
@ -849,6 +987,13 @@ public abstract void setTimes(final Path f, final long mtime,
* The specification of this method matches that of
* {@link FileContext#getFileChecksum(Path)} except that Path f must be for
* this file system.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return File Check sum.
*/
public abstract FileChecksum getFileChecksum(final Path f)
throws AccessControlException, FileNotFoundException,
@ -859,6 +1004,13 @@ public abstract FileChecksum getFileChecksum(final Path f)
* {@link FileContext#getFileStatus(Path)}
* except that an UnresolvedLinkException may be thrown if a symlink is
* encountered in the path.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return File Status
*/
public abstract FileStatus getFileStatus(final Path f)
throws AccessControlException, FileNotFoundException,
@ -870,8 +1022,8 @@ public abstract FileStatus getFileStatus(final Path f)
* In some FileSystem implementations such as HDFS metadata
* synchronization is essential to guarantee consistency of read requests
* particularly in HA setting.
* @throws IOException
* @throws UnsupportedOperationException
* @throws IOException raised on errors performing I/O.
* @throws UnsupportedOperationException Unsupported Operation Exception.
*/
public void msync() throws IOException, UnsupportedOperationException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
@ -883,6 +1035,13 @@ public void msync() throws IOException, UnsupportedOperationException {
* {@link FileContext#access(Path, FsAction)}
* except that an UnresolvedLinkException may be thrown if a symlink is
* encountered in the path.
*
* @param path the path.
* @param mode fsaction mode.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "Hive"})
public void access(Path path, FsAction mode) throws AccessControlException,
@ -897,6 +1056,13 @@ public void access(Path path, FsAction mode) throws AccessControlException,
* encountered in the path leading up to the final path component.
* If the file system does not support symlinks then the behavior is
* equivalent to {@link AbstractFileSystem#getFileStatus(Path)}.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnsupportedFileSystemException UnSupported File System Exception.
* @throws IOException raised on errors performing I/O.
* @return file status.
*/
public FileStatus getFileLinkStatus(final Path f)
throws AccessControlException, FileNotFoundException,
@ -908,6 +1074,15 @@ public FileStatus getFileLinkStatus(final Path f)
* The specification of this method matches that of
* {@link FileContext#getFileBlockLocations(Path, long, long)} except that
* Path f must be for this file system.
*
* @param f the path.
* @param start start.
* @param len length.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return BlockLocation Array.
*/
public abstract BlockLocation[] getFileBlockLocations(final Path f,
final long start, final long len) throws AccessControlException,
@ -917,6 +1092,13 @@ public abstract BlockLocation[] getFileBlockLocations(final Path f,
* The specification of this method matches that of
* {@link FileContext#getFsStatus(Path)} except that Path f must be for this
* file system.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return Fs Status.
*/
public FsStatus getFsStatus(final Path f) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
@ -927,6 +1109,11 @@ public FsStatus getFsStatus(final Path f) throws AccessControlException,
/**
* The specification of this method matches that of
* {@link FileContext#getFsStatus(Path)}.
*
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws IOException raised on errors performing I/O.
* @return Fs Status.
*/
public abstract FsStatus getFsStatus() throws AccessControlException,
FileNotFoundException, IOException;
@ -935,6 +1122,13 @@ public abstract FsStatus getFsStatus() throws AccessControlException,
* The specification of this method matches that of
* {@link FileContext#listStatus(Path)} except that Path f must be for this
* file system.
*
* @param f path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return FileStatus Iterator.
*/
public RemoteIterator<FileStatus> listStatusIterator(final Path f)
throws AccessControlException, FileNotFoundException,
@ -967,6 +1161,13 @@ public FileStatus next() {
* will have different formats for replicated and erasure coded file. Please
* refer to {@link FileSystem#getFileBlockLocations(FileStatus, long, long)}
* for more details.
*
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return FileStatus Iterator.
*/
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
throws AccessControlException, FileNotFoundException,
@ -999,6 +1200,12 @@ public LocatedFileStatus next() throws IOException {
* The specification of this method matches that of
* {@link FileContext.Util#listStatus(Path)} except that Path f must be
* for this file system.
* @param f the path.
* @throws AccessControlException access control exception.
* @throws FileNotFoundException file not found exception.
* @throws UnresolvedLinkException unresolved link exception.
* @throws IOException raised on errors performing I/O.
* @return FileStatus Iterator.
*/
public abstract FileStatus[] listStatus(final Path f)
throws AccessControlException, FileNotFoundException,
@ -1007,7 +1214,8 @@ public abstract FileStatus[] listStatus(final Path f)
/**
* @return an iterator over the corrupt files under the given path
* (may contain duplicates if a file has more than one corrupt block)
* @throws IOException
* @param path the path.
* @throws IOException raised on errors performing I/O.
*/
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
@ -1020,6 +1228,10 @@ public RemoteIterator<Path> listCorruptFileBlocks(Path path)
* The specification of this method matches that of
* {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f
* must be for this file system.
*
* @param verifyChecksum verify check sum flag.
* @throws AccessControlException access control exception.
* @throws IOException raised on errors performing I/O.
*/
public abstract void setVerifyChecksum(final boolean verifyChecksum)
throws AccessControlException, IOException;
@ -1041,7 +1253,7 @@ public String getCanonicalServiceName() {
* @param renewer the account name that is allowed to renew the token.
* @return List of delegation tokens.
* If delegation tokens not supported then return a list of size zero.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
public List<Token<?>> getDelegationTokens(String renewer) throws IOException {
@ -1141,7 +1353,7 @@ public AclStatus getAclStatus(Path path) throws IOException {
* @param path Path to modify
* @param name xattr name.
* @param value xattr value.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void setXAttr(Path path, String name, byte[] value)
throws IOException {
@ -1160,7 +1372,7 @@ public void setXAttr(Path path, String name, byte[] value)
* @param name xattr name.
* @param value xattr value.
* @param flag xattr set flag
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void setXAttr(Path path, String name, byte[] value,
EnumSet<XAttrSetFlag> flag) throws IOException {
@ -1178,7 +1390,7 @@ public void setXAttr(Path path, String name, byte[] value,
* @param path Path to get extended attribute
* @param name xattr name.
* @return byte[] xattr value.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public byte[] getXAttr(Path path, String name) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
@ -1196,7 +1408,7 @@ public byte[] getXAttr(Path path, String name) throws IOException {
*
* @return {@literal Map<String, byte[]>} describing the XAttrs of the file
* or directory
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
@ -1214,7 +1426,7 @@ public Map<String, byte[]> getXAttrs(Path path) throws IOException {
* @param names XAttr names.
* @return {@literal Map<String, byte[]>} describing the XAttrs of the file
* or directory
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public Map<String, byte[]> getXAttrs(Path path, List<String> names)
throws IOException {
@ -1232,7 +1444,7 @@ public Map<String, byte[]> getXAttrs(Path path, List<String> names)
* @param path Path to get extended attributes
* @return {@literal Map<String, byte[]>} describing the XAttrs of the file
* or directory
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public List<String> listXAttrs(Path path)
throws IOException {
@ -1249,7 +1461,7 @@ public List<String> listXAttrs(Path path)
*
* @param path Path to remove extended attribute
* @param name xattr name
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void removeXAttr(Path path, String name) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
@ -1259,6 +1471,11 @@ public void removeXAttr(Path path, String name) throws IOException {
/**
* The specification of this method matches that of
* {@link FileContext#createSnapshot(Path, String)}.
*
* @param path the path.
* @param snapshotName snapshot name.
* @throws IOException raised on errors performing I/O.
* @return path.
*/
public Path createSnapshot(final Path path, final String snapshotName)
throws IOException {
@ -1269,6 +1486,11 @@ public Path createSnapshot(final Path path, final String snapshotName)
/**
* The specification of this method matches that of
* {@link FileContext#renameSnapshot(Path, String, String)}.
*
* @param path the path.
* @param snapshotOldName snapshot old name.
* @param snapshotNewName snapshot new name.
* @throws IOException raised on errors performing I/O.
*/
public void renameSnapshot(final Path path, final String snapshotOldName,
final String snapshotNewName) throws IOException {
@ -1279,6 +1501,10 @@ public void renameSnapshot(final Path path, final String snapshotOldName,
/**
* The specification of this method matches that of
* {@link FileContext#deleteSnapshot(Path, String)}.
*
* @param snapshotDir snapshot dir.
* @param snapshotName snapshot name.
* @throws IOException raised on errors performing I/O.
*/
public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
throws IOException {
@ -1289,7 +1515,7 @@ public void deleteSnapshot(final Path snapshotDir, final String snapshotName)
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
@ -1303,6 +1529,7 @@ public void satisfyStoragePolicy(final Path path) throws IOException {
* @param policyName the name of the target storage policy. The list
* of supported Storage policies can be retrieved
* via {@link #getAllStoragePolicies}.
* @throws IOException raised on errors performing I/O.
*/
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
@ -1314,7 +1541,7 @@ public void setStoragePolicy(final Path path, final String policyName)
/**
* Unset the storage policy set for a given file or directory.
* @param src file or directory path.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public void unsetStoragePolicy(final Path src) throws IOException {
throw new UnsupportedOperationException(getClass().getSimpleName()
@ -1326,7 +1553,7 @@ public void unsetStoragePolicy(final Path src) throws IOException {
*
* @param src file or directory path.
* @return storage policy for give file.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
@ -1338,7 +1565,7 @@ public BlockStoragePolicySpi getStoragePolicy(final Path src)
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {

14
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AvroFSInput.java
View File

@ -36,13 +36,22 @@ public class AvroFSInput implements Closeable, SeekableInput {
private final FSDataInputStream stream;
private final long len;
/** Construct given an {@link FSDataInputStream} and its length. */
/**
* Construct given an {@link FSDataInputStream} and its length.
*
* @param in inputstream.
* @param len len.
*/
public AvroFSInput(final FSDataInputStream in, final long len) {
this.stream = in;
this.len = len;
}
/** Construct given a {@link FileContext} and a {@link Path}. */
/** Construct given a {@link FileContext} and a {@link Path}.
* @param fc filecontext.
* @param p the path.
* @throws IOException If an I/O error occurred.
* */
public AvroFSInput(final FileContext fc, final Path p) throws IOException {
FileStatus status = fc.getFileStatus(p);
this.len = status.getLen();
@ -51,7 +60,6 @@ public AvroFSInput(final FileContext fc, final Path p) throws IOException {
FS_OPTION_OPENFILE_READ_POLICY_SEQUENTIAL)
.withFileStatus(status)
.build());
fc.open(p);
}
@Override

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchedRemoteIterator.java
View File

@ -68,6 +68,7 @@ public BatchedRemoteIterator(K prevKey) {
*
* @param prevKey The key to send.
* @return A list of replies.
* @throws IOException If an I/O error occurred.
*/
public abstract BatchedEntries<E> makeRequest(K prevKey) throws IOException;
@ -102,6 +103,8 @@ public boolean hasNext() throws IOException {
/**
* Return the next list key associated with an element.
* @param element element.
* @return K Generics Type.
*/
public abstract K elementToPrevKey(E element);

45
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
View File

@ -85,6 +85,7 @@ public BlockLocation() {
/**
* Copy constructor.
* @param that blocklocation.
*/
public BlockLocation(BlockLocation that) {
this.hosts = that.hosts;
@ -100,6 +101,10 @@ public BlockLocation(BlockLocation that) {
/**
* Constructor with host, name, offset and length.
* @param names names array.
* @param hosts host array.
* @param offset offset.
* @param length length.
*/
public BlockLocation(String[] names, String[] hosts, long offset,
long length) {
@ -108,6 +113,11 @@ public BlockLocation(String[] names, String[] hosts, long offset,
/**
* Constructor with host, name, offset, length and corrupt flag.
* @param names names.
* @param hosts hosts.
* @param offset offset.
* @param length length.
* @param corrupt corrupt.
*/
public BlockLocation(String[] names, String[] hosts, long offset,
long length, boolean corrupt) {
@ -116,6 +126,11 @@ public BlockLocation(String[] names, String[] hosts, long offset,
/**
* Constructor with host, name, network topology, offset and length.
* @param names names.
* @param hosts hosts.
* @param topologyPaths topologyPaths.
* @param offset offset.
* @param length length.
*/
public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
long offset, long length) {
@ -125,6 +140,12 @@ public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
/**
* Constructor with host, name, network topology, offset, length
* and corrupt flag.
* @param names names.
* @param hosts hosts.
* @param topologyPaths topologyPaths.
* @param offset offset.
* @param length length.
* @param corrupt corrupt.
*/
public BlockLocation(String[] names, String[] hosts, String[] topologyPaths,
long offset, long length, boolean corrupt) {
@ -177,6 +198,8 @@ public BlockLocation(String[] names, String[] hosts, String[] cachedHosts,
/**
* Get the list of hosts (hostname) hosting this block.
* @return hosts array.
* @throws IOException If an I/O error occurred.
*/
public String[] getHosts() throws IOException {
return hosts;
@ -184,6 +207,7 @@ public String[] getHosts() throws IOException {
/**
* Get the list of hosts (hostname) hosting a cached replica of the block.
* @return cached hosts.
*/
public String[] getCachedHosts() {
return cachedHosts;
@ -191,6 +215,8 @@ public String[] getCachedHosts() {
/**
* Get the list of names (IP:xferPort) hosting this block.
* @return names array.
* @throws IOException If an I/O error occurred.
*/
public String[] getNames() throws IOException {
return names;
@ -199,6 +225,8 @@ public String[] getNames() throws IOException {
/**
* Get the list of network topology paths for each of the hosts.
* The last component of the path is the "name" (IP:xferPort).
* @return topology paths.
* @throws IOException If an I/O error occurred.
*/
public String[] getTopologyPaths() throws IOException {
return topologyPaths;
@ -206,6 +234,7 @@ public String[] getTopologyPaths() throws IOException {
/**
* Get the storageID of each replica of the block.
* @return storage ids.
*/
public String[] getStorageIds() {
return storageIds;
@ -213,6 +242,7 @@ public String[] getStorageIds() {
/**
* Get the storage type of each replica of the block.
* @return storage type of each replica of the block.
*/
public StorageType[] getStorageTypes() {
return storageTypes;
@ -220,6 +250,7 @@ public StorageType[] getStorageTypes() {
/**
* Get the start offset of file associated with this block.
* @return start offset of file associated with this block.
*/
public long getOffset() {
return offset;
@ -227,6 +258,7 @@ public long getOffset() {
/**
* Get the length of the block.
* @return length of the block.
*/
public long getLength() {
return length;
@ -234,6 +266,7 @@ public long getLength() {
/**
* Get the corrupt flag.
* @return corrupt flag.
*/
public boolean isCorrupt() {
return corrupt;
@ -241,6 +274,7 @@ public boolean isCorrupt() {
/**
* Return true if the block is striped (erasure coded).
* @return if the block is striped true, not false.
*/
public boolean isStriped() {
return false;
@ -248,6 +282,7 @@ public boolean isStriped() {
/**
* Set the start offset of file associated with this block.
* @param offset start offset.
*/
public void setOffset(long offset) {
this.offset = offset;
@ -255,6 +290,7 @@ public void setOffset(long offset) {
/**
* Set the length of block.
* @param length length of block.
*/
public void setLength(long length) {
this.length = length;
@ -262,6 +298,7 @@ public void setLength(long length) {
/**
* Set the corrupt flag.
* @param corrupt corrupt flag.
*/
public void setCorrupt(boolean corrupt) {
this.corrupt = corrupt;
@ -269,6 +306,8 @@ public void setCorrupt(boolean corrupt) {
/**
* Set the hosts hosting this block.
* @param hosts hosts array.
* @throws IOException If an I/O error occurred.
*/
public void setHosts(String[] hosts) throws IOException {
if (hosts == null) {
@ -280,6 +319,7 @@ public void setHosts(String[] hosts) throws IOException {
/**
* Set the hosts hosting a cached replica of this block.
* @param cachedHosts cached hosts.
*/
public void setCachedHosts(String[] cachedHosts) {
if (cachedHosts == null) {
@ -291,6 +331,8 @@ public void setCachedHosts(String[] cachedHosts) {
/**
* Set the names (host:port) hosting this block.
* @param names names.
* @throws IOException If an I/O error occurred.
*/
public void setNames(String[] names) throws IOException {
if (names == null) {
@ -302,6 +344,9 @@ public void setNames(String[] names) throws IOException {
/**
* Set the network topology paths of the hosts.
*
* @param topologyPaths topology paths.
* @throws IOException If an I/O error occurred.
*/
public void setTopologyPaths(String[] topologyPaths) throws IOException {
if (topologyPaths == null) {

27
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java
View File

@ -1,4 +1,4 @@
/**
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -22,6 +22,9 @@
import java.io.FileDescriptor;
import java.io.IOException;
import java.util.StringJoiner;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.function.IntFunction;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -158,8 +161,24 @@ public IOStatistics getIOStatistics() {
@Override
public String toString() {
return new StringJoiner(", ",
BufferedFSInputStream.class.getSimpleName() + "[", "]")
.add("in=" + in)
.toString();
BufferedFSInputStream.class.getSimpleName() + "[", "]")
.add("in=" + in)
.toString();
}
@Override
public int minSeekForVectorReads() {
return ((PositionedReadable) in).minSeekForVectorReads();
}
@Override
public int maxReadSizeForVectorReads() {
return ((PositionedReadable) in).maxReadSizeForVectorReads();
}
@Override
public void readVectored(List<? extends FileRange> ranges,
IntFunction<ByteBuffer> allocate) throws IOException {
((PositionedReadable) in).readVectored(ranges, allocate);
}
}

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferUtil.java
View File

@ -47,6 +47,12 @@ private static boolean streamHasByteBufferRead(InputStream stream) {
/**
* Perform a fallback read.
*
* @param stream input stream.
* @param bufferPool bufferPool.
* @param maxLength maxLength.
* @throws IOException raised on errors performing I/O.
* @return byte buffer.
*/
public static ByteBuffer fallbackRead(
InputStream stream, ByteBufferPool bufferPool, int maxLength)

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
View File

@ -53,6 +53,9 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
/**
* This is the constructor used by the builder.
* All overriding classes should implement this.
*
* @param builder builder.
* @throws IOException raised on errors performing I/O.
*/
public CachingGetSpaceUsed(CachingGetSpaceUsed.Builder builder)
throws IOException {
@ -140,6 +143,8 @@ public String getDirPath() {
/**
* Increment the cached value of used space.
*
* @param value dfs used value.
*/
public void incDfsUsed(long value) {
used.addAndGet(value);
@ -154,6 +159,8 @@ boolean running() {
/**
* How long in between runs of the background refresh.
*
* @return refresh interval.
*/
@VisibleForTesting
public long getRefreshInterval() {
@ -163,6 +170,8 @@ public long getRefreshInterval() {
/**
* Randomize the refresh interval timing by this amount, the actual interval will be chosen
* uniformly between {@code interval-jitter} and {@code interval+jitter}.
*
* @return between interval-jitter and interval+jitter.
*/
@VisibleForTesting
public long getJitter() {

307
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
View File

@ -22,17 +22,24 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.function.IntFunction;
import java.util.zip.CRC32;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
import org.apache.hadoop.fs.impl.CombinedFileRange;
import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
@ -47,6 +54,7 @@
import static org.apache.hadoop.fs.Options.OpenFileOptions.FS_OPTION_OPENFILE_STANDARD_OPTIONS;
import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
import static org.apache.hadoop.fs.impl.StoreImplementationUtils.isProbeForSyncable;
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
/****************************************************************
* Abstract Checksumed FileSystem.
@ -66,7 +74,7 @@ public abstract class ChecksumFileSystem extends FilterFileSystem {
public static double getApproxChkSumLength(long size) {
return ChecksumFSOutputSummer.CHKSUM_AS_FRACTION * size;
}
public ChecksumFileSystem(FileSystem fs) {
super(fs);
}
@ -82,7 +90,7 @@ public void setConf(Configuration conf) {
bytesPerChecksum);
}
}
/**
* Set whether to verify checksum.
*/
@ -95,32 +103,51 @@ public void setVerifyChecksum(boolean verifyChecksum) {
public void setWriteChecksum(boolean writeChecksum) {
this.writeChecksum = writeChecksum;
}
/** get the raw file system */
@Override
public FileSystem getRawFileSystem() {
return fs;
}
/** Return the name of the checksum file associated with a file.*/
/**
* Return the name of the checksum file associated with a file.
*
* @param file the file path.
* @return name of the checksum file associated with a file.
*/
public Path getChecksumFile(Path file) {
return new Path(file.getParent(), "." + file.getName() + ".crc");
}
/** Return true iff file is a checksum file name.*/
/**
* Return true if file is a checksum file name.
*
* @param file the file path.
* @return if file is a checksum file true, not false.
*/
public static boolean isChecksumFile(Path file) {
String name = file.getName();
return name.startsWith(".") && name.endsWith(".crc");
}
/** Return the length of the checksum file given the size of the
/**
* Return the length of the checksum file given the size of the
* actual file.
**/
*
* @param file the file path.
* @param fileSize file size.
* @return checksum length.
*/
public long getChecksumFileLength(Path file, long fileSize) {
return getChecksumLength(fileSize, getBytesPerSum());
}
/** Return the bytes Per Checksum */
/**
* Return the bytes Per Checksum.
*
* @return bytes per check sum.
*/
public int getBytesPerSum() {
return bytesPerChecksum;
}
@ -139,22 +166,23 @@ private int getSumBufferSize(int bytesPerSum, int bufferSize) {
* It verifies that data matches checksums.
*******************************************************/
private static class ChecksumFSInputChecker extends FSInputChecker implements
IOStatisticsSource {
IOStatisticsSource, StreamCapabilities {
private ChecksumFileSystem fs;
private FSDataInputStream datas;
private FSDataInputStream sums;
private static final int HEADER_LENGTH = 8;
private int bytesPerSum = 1;
private long fileLen = -1L;
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
throws IOException {
this(fs, file, fs.getConf().getInt(
LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY,
LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT));
}
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
throws IOException {
super( file, fs.getFileStatus(file).getReplication() );
@ -170,7 +198,8 @@ public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
if (!Arrays.equals(version, CHECKSUM_VERSION))
throw new IOException("Not a checksum file: "+sumFile);
this.bytesPerSum = sums.readInt();
set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4);
set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum,
FSInputChecker.CHECKSUM_SIZE);
} catch (IOException e) {
// mincing the message is terrible, but java throws permission
// exceptions as FNF because that's all the method signatures allow!
@ -182,21 +211,21 @@ public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
set(fs.verifyChecksum, null, 1, 0);
}
}
private long getChecksumFilePos( long dataPos ) {
return HEADER_LENGTH + 4*(dataPos/bytesPerSum);
return HEADER_LENGTH + FSInputChecker.CHECKSUM_SIZE*(dataPos/bytesPerSum);
}
@Override
protected long getChunkPosition( long dataPos ) {
return dataPos/bytesPerSum*bytesPerSum;
}
@Override
public int available() throws IOException {
return datas.available() + super.available();
}
@Override
public int read(long position, byte[] b, int off, int len)
throws IOException {
@ -214,7 +243,7 @@ public int read(long position, byte[] b, int off, int len)
}
return nread;
}
@Override
public void close() throws IOException {
datas.close();
@ -223,7 +252,7 @@ public void close() throws IOException {
}
set(fs.verifyChecksum, null, 1, 0);
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
@ -246,7 +275,7 @@ protected int readChunk(long pos, byte[] buf, int offset, int len,
final int checksumsToRead = Math.min(
len/bytesPerSum, // number of checksums based on len to read
checksum.length / CHECKSUM_SIZE); // size of checksum buffer
long checksumPos = getChecksumFilePos(pos);
long checksumPos = getChecksumFilePos(pos);
if(checksumPos != sums.getPos()) {
sums.seek(checksumPos);
}
@ -286,8 +315,187 @@ protected int readChunk(long pos, byte[] buf, int offset, int len,
public IOStatistics getIOStatistics() {
return IOStatisticsSupport.retrieveIOStatistics(datas);
}
public static long findChecksumOffset(long dataOffset,
int bytesPerSum) {
return HEADER_LENGTH + (dataOffset/bytesPerSum) * FSInputChecker.CHECKSUM_SIZE;
}
/**
* Calculate length of file if not already cached.
* @return file length.
* @throws IOException any IOE.
*/
private long getFileLength() throws IOException {
if (fileLen == -1L) {
fileLen = fs.getFileStatus(file).getLen();
}
return fileLen;
}
/**
* Find the checksum ranges that correspond to the given data ranges.
* @param dataRanges the input data ranges, which are assumed to be sorted
* and non-overlapping
* @return a list of AsyncReaderUtils.CombinedFileRange that correspond to
* the checksum ranges
*/
public static List<CombinedFileRange> findChecksumRanges(
List<? extends FileRange> dataRanges,
int bytesPerSum,
int minSeek,
int maxSize) {
List<CombinedFileRange> result = new ArrayList<>();
CombinedFileRange currentCrc = null;
for(FileRange range: dataRanges) {
long crcOffset = findChecksumOffset(range.getOffset(), bytesPerSum);
long crcEnd = findChecksumOffset(range.getOffset() + range.getLength() +
bytesPerSum - 1, bytesPerSum);
if (currentCrc == null ||
!currentCrc.merge(crcOffset, crcEnd, range, minSeek, maxSize)) {
currentCrc = new CombinedFileRange(crcOffset, crcEnd, range);
result.add(currentCrc);
}
}
return result;
}
/**
* Check the data against the checksums.
* @param sumsBytes the checksum data
* @param sumsOffset where from the checksum file this buffer started
* @param data the file data
* @param dataOffset where the file data started (must be a multiple of
* bytesPerSum)
* @param bytesPerSum how many bytes per a checksum
* @param file the path of the filename
* @return the data buffer
* @throws CompletionException if the checksums don't match
*/
static ByteBuffer checkBytes(ByteBuffer sumsBytes,
long sumsOffset,
ByteBuffer data,
long dataOffset,
int bytesPerSum,
Path file) {
// determine how many bytes we need to skip at the start of the sums
int offset =
(int) (findChecksumOffset(dataOffset, bytesPerSum) - sumsOffset);
IntBuffer sums = sumsBytes.asIntBuffer();
sums.position(offset / FSInputChecker.CHECKSUM_SIZE);
ByteBuffer current = data.duplicate();
int numFullChunks = data.remaining() / bytesPerSum;
boolean partialChunk = ((data.remaining() % bytesPerSum) != 0);
int totalChunks = numFullChunks;
if (partialChunk) {
totalChunks++;
}
CRC32 crc = new CRC32();
// check each chunk to ensure they match
for(int c = 0; c < totalChunks; ++c) {
// set the buffer position to the start of every chunk.
current.position(c * bytesPerSum);
if (c == numFullChunks) {
// During last chunk, there may be less than chunk size
// data preset, so setting the limit accordingly.
int lastIncompleteChunk = data.remaining() % bytesPerSum;
current.limit((c * bytesPerSum) + lastIncompleteChunk);
} else {
// set the buffer limit to end of every chunk.
current.limit((c + 1) * bytesPerSum);
}
// compute the crc
crc.reset();
crc.update(current);
int expected = sums.get();
int calculated = (int) crc.getValue();
if (calculated != expected) {
// cast of c added to silence findbugs
long errPosn = dataOffset + (long) c * bytesPerSum;
throw new CompletionException(new ChecksumException(
"Checksum error: " + file + " at " + errPosn +
" exp: " + expected + " got: " + calculated, errPosn));
}
}
// if everything matches, we return the data
return data;
}
/**
* Validates range parameters.
* In case of CheckSum FS, we already have calculated
* fileLength so failing fast here.
* @param ranges requested ranges.
* @param fileLength length of file.
* @throws EOFException end of file exception.
*/
private void validateRangeRequest(List<? extends FileRange> ranges,
final long fileLength) throws EOFException {
for (FileRange range : ranges) {
VectoredReadUtils.validateRangeRequest(range);
if (range.getOffset() + range.getLength() > fileLength) {
final String errMsg = String.format("Requested range [%d, %d) is beyond EOF for path %s",
range.getOffset(), range.getLength(), file);
LOG.warn(errMsg);
throw new EOFException(errMsg);
}
}
}
@Override
public void readVectored(List<? extends FileRange> ranges,
IntFunction<ByteBuffer> allocate) throws IOException {
final long length = getFileLength();
validateRangeRequest(ranges, length);
// If the stream doesn't have checksums, just delegate.
if (sums == null) {
datas.readVectored(ranges, allocate);
return;
}
int minSeek = minSeekForVectorReads();
int maxSize = maxReadSizeForVectorReads();
List<CombinedFileRange> dataRanges =
VectoredReadUtils.mergeSortedRanges(Arrays.asList(sortRanges(ranges)), bytesPerSum,
minSeek, maxReadSizeForVectorReads());
// While merging the ranges above, they are rounded up based on the value of bytesPerSum
// which leads to some ranges crossing the EOF thus they need to be fixed else it will
// cause EOFException during actual reads.
for (CombinedFileRange range : dataRanges) {
if (range.getOffset() + range.getLength() > length) {
range.setLength((int) (length - range.getOffset()));
}
}
List<CombinedFileRange> checksumRanges = findChecksumRanges(dataRanges,
bytesPerSum, minSeek, maxSize);
sums.readVectored(checksumRanges, allocate);
datas.readVectored(dataRanges, allocate);
for(CombinedFileRange checksumRange: checksumRanges) {
for(FileRange dataRange: checksumRange.getUnderlying()) {
// when we have both the ranges, validate the checksum
CompletableFuture<ByteBuffer> result =
checksumRange.getData().thenCombineAsync(dataRange.getData(),
(sumBuffer, dataBuffer) ->
checkBytes(sumBuffer, checksumRange.getOffset(),
dataBuffer, dataRange.getOffset(), bytesPerSum, file));
// Now, slice the read data range to the user's ranges
for(FileRange original: ((CombinedFileRange) dataRange).getUnderlying()) {
original.setData(result.thenApply(
(b) -> VectoredReadUtils.sliceTo(b, dataRange.getOffset(), original)));
}
}
}
}
@Override
public boolean hasCapability(String capability) {
return datas.hasCapability(capability);
}
}
private static class FSDataBoundedInputStream extends FSDataInputStream {
private FileSystem fs;
private Path file;
@ -298,12 +506,12 @@ private static class FSDataBoundedInputStream extends FSDataInputStream {
this.fs = fs;
this.file = file;
}
@Override
public boolean markSupported() {
return false;
}
/* Return the file length */
private long getFileLength() throws IOException {
if( fileLen==-1L ) {
@ -311,7 +519,7 @@ private long getFileLength() throws IOException {
}
return fileLen;
}
/**
* Skips over and discards <code>n</code> bytes of data from the
* input stream.
@ -335,11 +543,11 @@ public synchronized long skip(long n) throws IOException {
}
return super.skip(n);
}
/**
* Seek to the given position in the stream.
* The next read() will be from that position.
*
*
* <p>This method does not allow seek past the end of the file.
* This produces IOException.
*
@ -362,6 +570,7 @@ public synchronized void seek(long pos) throws IOException {
* Opens an FSDataInputStream at the indicated Path.
* @param f the file name to open
* @param bufferSize the size of the buffer to be used.
* @throws IOException if an I/O error occurs.
*/
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
@ -404,22 +613,22 @@ public void concat(final Path f, final Path[] psrcs) throws IOException {
*/
public static long getChecksumLength(long size, int bytesPerSum) {
//the checksum length is equal to size passed divided by bytesPerSum +
//bytes written in the beginning of the checksum file.
return ((size + bytesPerSum - 1) / bytesPerSum) * 4 +
CHECKSUM_VERSION.length + 4;
//bytes written in the beginning of the checksum file.
return ((size + bytesPerSum - 1) / bytesPerSum) * FSInputChecker.CHECKSUM_SIZE +
ChecksumFSInputChecker.HEADER_LENGTH;
}
/** This class provides an output stream for a checksummed file.
* It generates checksums for data. */
private static class ChecksumFSOutputSummer extends FSOutputSummer
implements IOStatisticsSource, StreamCapabilities {
private FSDataOutputStream datas;
private FSDataOutputStream datas;
private FSDataOutputStream sums;
private static final float CHKSUM_AS_FRACTION = 0.01f;
private boolean isClosed = false;
public ChecksumFSOutputSummer(ChecksumFileSystem fs,
Path file,
ChecksumFSOutputSummer(ChecksumFileSystem fs,
Path file,
boolean overwrite,
int bufferSize,
short replication,
@ -440,7 +649,7 @@ public ChecksumFSOutputSummer(ChecksumFileSystem fs,
sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length);
sums.writeInt(bytesPerSum);
}
@Override
public void close() throws IOException {
try {
@ -451,7 +660,7 @@ public void close() throws IOException {
isClosed = true;
}
}
@Override
protected void writeChunk(byte[] b, int offset, int len, byte[] checksum,
int ckoff, int cklen)
@ -669,7 +878,7 @@ boolean apply(Path p) throws IOException {
* Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
* @param src file name
* @param replication new replication
* @throws IOException
* @throws IOException if an I/O error occurs.
* @return true if successful;
* false if file does not exist or is a directory
*/
@ -707,7 +916,7 @@ public boolean rename(Path src, Path dst) throws IOException {
value = fs.rename(srcCheckFile, dstCheckFile);
} else if (fs.exists(dstCheckFile)) {
// no src checksum, so remove dst checksum
value = fs.delete(dstCheckFile, true);
value = fs.delete(dstCheckFile, true);
}
return value;
@ -739,7 +948,7 @@ public boolean delete(Path f, boolean recursive) throws IOException{
return fs.delete(f, true);
}
}
final private static PathFilter DEFAULT_FILTER = new PathFilter() {
@Override
public boolean accept(Path file) {
@ -750,11 +959,11 @@ public boolean accept(Path file) {
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
*
*
* @param f
* given path
* @return the statuses of the files/directories in the given path
* @throws IOException
* @throws IOException if an I/O error occurs.
*/
@Override
public FileStatus[] listStatus(Path f) throws IOException {
@ -771,18 +980,18 @@ public RemoteIterator<FileStatus> listStatusIterator(final Path p)
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
*
*
* @param f
* given path
* @return the statuses of the files/directories in the given patch
* @throws IOException
* @throws IOException if an I/O error occurs.
*/
@Override
public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
throws IOException {
return fs.listLocatedStatus(f, DEFAULT_FILTER);
}
@Override
public boolean mkdirs(Path f) throws IOException {
return fs.mkdirs(f);
@ -811,6 +1020,10 @@ public void copyToLocalFile(boolean delSrc, Path src, Path dst)
* Copy it from FS control to the local dst name.
* If src and dst are directories, the copyCrc parameter
* determines whether to copy CRC files.
* @param src src path.
* @param dst dst path.
* @param copyCrc copy csc flag.
* @throws IOException if an I/O error occurs.
*/
@SuppressWarnings("deprecation")
public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
@ -832,7 +1045,7 @@ public void copyToLocalFile(Path src, Path dst, boolean copyCrc)
} else {
FileStatus[] srcs = listStatus(src);
for (FileStatus srcFile : srcs) {
copyToLocalFile(srcFile.getPath(),
copyToLocalFile(srcFile.getPath(),
new Path(dst, srcFile.getPath().getName()), copyCrc);
}
}

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
View File

@ -70,30 +70,53 @@ public void setVerifyChecksum(boolean inVerifyChecksum) {
this.verifyChecksum = inVerifyChecksum;
}
/** get the raw file system. */
/**
* get the raw file system.
*
* @return abstract file system.
*/
public AbstractFileSystem getRawFs() {
return getMyFs();
}
/** Return the name of the checksum file associated with a file.*/
/**
* Return the name of the checksum file associated with a file.
*
* @param file the file path.
* @return the checksum file associated with a file.
*/
public Path getChecksumFile(Path file) {
return new Path(file.getParent(), "." + file.getName() + ".crc");
}
/** Return true iff file is a checksum file name.*/
/**
* Return true iff file is a checksum file name.
*
* @param file the file path.
* @return if is checksum file true,not false.
*/
public static boolean isChecksumFile(Path file) {
String name = file.getName();
return name.startsWith(".") && name.endsWith(".crc");
}
/** Return the length of the checksum file given the size of the
/**
* Return the length of the checksum file given the size of the
* actual file.
**/
*
* @param file the file path.
* @param fileSize file size.
* @return check sum file length.
*/
public long getChecksumFileLength(Path file, long fileSize) {
return getChecksumLength(fileSize, getBytesPerSum());
}
/** Return the bytes Per Checksum. */
/**
* Return the bytes Per Checksum.
*
* @return bytes per sum.
*/
public int getBytesPerSum() {
return defaultBytesPerChecksum;
}
@ -433,7 +456,7 @@ private boolean isDirectory(Path f)
* Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt>
* @param src file name
* @param replication new replication
* @throws IOException
* @throws IOException if an I/O error occurs.
* @return true if successful;
* false if file does not exist or is a directory
*/

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -401,6 +401,10 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final String ZK_AUTH = ZK_PREFIX + "auth";
/** Principal name for zookeeper servers. */
public static final String ZK_SERVER_PRINCIPAL = ZK_PREFIX + "server.principal";
/** Kerberos principal name for zookeeper connection. */
public static final String ZK_KERBEROS_PRINCIPAL = ZK_PREFIX + "kerberos.principal";
/** Kerberos keytab for zookeeper connection. */
public static final String ZK_KERBEROS_KEYTAB = ZK_PREFIX + "kerberos.keytab";
/** Address of the ZooKeeper ensemble. */
public static final String ZK_ADDRESS = ZK_PREFIX + "address";
@ -471,4 +475,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
* default hadoop temp dir on local system: {@value}.
*/
public static final String HADOOP_TMP_DIR = "hadoop.tmp.dir";
/**
* Thread-level IOStats Support.
* {@value}
*/
public static final String IOSTATISTICS_THREAD_LEVEL_ENABLED =
"fs.iostatistics.thread.level.enabled";
/**
* Default value for Thread-level IOStats Support is true.
*/
public static final boolean IOSTATISTICS_THREAD_LEVEL_ENABLED_DEFAULT =
true;
public static final String HADOOP_SECURITY_RESOLVER_IMPL =
"hadoop.security.resolver.impl";
}

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
View File

@ -169,11 +169,11 @@ public class CommonConfigurationKeysPublic {
/**
* Number of filesystems instances can be created in parallel.
* <p></p>
* <p>
* A higher number here does not necessarily improve performance, especially
* for object stores, where multiple threads may be attempting to create an FS
* instance for the same URI.
* <p></p>
* </p>
* Default value: {@value}.
*/
public static final String FS_CREATION_PARALLEL_COUNT =
@ -181,8 +181,9 @@ public class CommonConfigurationKeysPublic {
/**
* Default value for {@link #FS_CREATION_PARALLEL_COUNT}.
* <p></p>
* <p>
* Default value: {@value}.
* </p>
*/
public static final int FS_CREATION_PARALLEL_COUNT_DEFAULT =
64;
@ -999,6 +1000,7 @@ public class CommonConfigurationKeysPublic {
String.join(",",
"secret$",
"password$",
"username$",
"ssl.keystore.pass$",
"fs.s3.*[Ss]ecret.?[Kk]ey",
"fs.s3a.*.server-side-encryption.key",
@ -1053,5 +1055,13 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY =
"hadoop.http.idle_timeout.ms";
public static final int HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT = 60000;
/**
* To configure scheduling of server metrics update thread. This config is used to indicate
* initial delay and delay between each execution of the metric update runnable thread.
*/
public static final String IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL =
"ipc.server.metrics.update.runner.interval";
public static final int IPC_SERVER_METRICS_UPDATE_RUNNER_INTERVAL_DEFAULT = 5000;
}

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CompositeCrcFileChecksum.java
View File

@ -37,7 +37,13 @@ public class CompositeCrcFileChecksum extends FileChecksum {
private DataChecksum.Type crcType;
private int bytesPerCrc;
/** Create a CompositeCrcFileChecksum. */
/**
* Create a CompositeCrcFileChecksum.
*
* @param crc crc.
* @param crcType crcType.
* @param bytesPerCrc bytesPerCrc.
*/
public CompositeCrcFileChecksum(
int crc, DataChecksum.Type crcType, int bytesPerCrc) {
this.crc = crc;

26
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View File

@ -149,17 +149,31 @@ public ContentSummary build() {
@Deprecated
public ContentSummary() {}
/** Constructor, deprecated by ContentSummary.Builder
/**
* Constructor, deprecated by ContentSummary.Builder
* This constructor implicitly set spaceConsumed the same as length.
* spaceConsumed and length must be set explicitly with
* ContentSummary.Builder
* ContentSummary.Builder.
*
* @param length length.
* @param fileCount file count.
* @param directoryCount directory count.
* */
@Deprecated
public ContentSummary(long length, long fileCount, long directoryCount) {
this(length, fileCount, directoryCount, -1L, length, -1L);
}
/** Constructor, deprecated by ContentSummary.Builder */
/**
* Constructor, deprecated by ContentSummary.Builder.
*
* @param length length.
* @param fileCount file count.
* @param directoryCount directory count.
* @param quota quota.
* @param spaceConsumed space consumed.
* @param spaceQuota space quota.
* */
@Deprecated
public ContentSummary(
long length, long fileCount, long directoryCount, long quota,
@ -172,7 +186,11 @@ public ContentSummary(
setSpaceQuota(spaceQuota);
}
/** Constructor for ContentSummary.Builder*/
/**
* Constructor for ContentSummary.Builder.
*
* @param builder builder.
*/
private ContentSummary(Builder builder) {
super(builder);
this.length = builder.length;

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
View File

@ -189,6 +189,8 @@ public static void validate(Object path, boolean pathExists,
/**
* Validate the CreateFlag for the append operation. The flag must contain
* APPEND, and cannot contain OVERWRITE.
*
* @param flag enum set flag.
*/
public static void validateForAppend(EnumSet<CreateFlag> flag) {
validate(flag);

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
View File

@ -65,7 +65,10 @@ public String getDirPath() {
return dirPath;
}
/** @return a string indicating which filesystem volume we're checking. */
/**
* @return a string indicating which filesystem volume we're checking.
* @throws IOException raised on errors performing I/O.
*/
public String getFilesystem() throws IOException {
if (Shell.WINDOWS) {
this.filesystem = dirFile.getCanonicalPath().substring(0, 2);
@ -100,7 +103,10 @@ public int getPercentUsed() {
return (int) (used * 100.0 / cap);
}
/** @return the filesystem mount point for the indicated volume */
/**
* @return the filesystem mount point for the indicated volume.
* @throws IOException raised on errors performing I/O.
*/
public String getMount() throws IOException {
// Abort early if specified path does not exist
if (!dirFile.exists()) {

31
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java
View File

@ -47,7 +47,11 @@ public interface Renewable {
/** @return the renew token. */
public Token<?> getRenewToken();
/** Set delegation token. */
/**
* Set delegation token.
* @param <T> generic type T.
* @param token token.
*/
public <T extends TokenIdentifier> void setDelegationToken(Token<T> token);
}
@ -172,7 +176,11 @@ public String toString() {
/** Queue to maintain the RenewActions to be processed by the {@link #run()} */
private volatile DelayQueue<RenewAction<?>> queue = new DelayQueue<RenewAction<?>>();
/** For testing purposes */
/**
* For testing purposes.
*
* @return renew queue length.
*/
@VisibleForTesting
protected int getRenewQueueLength() {
return queue.size();
@ -211,7 +219,13 @@ static synchronized void reset() {
}
}
/** Add a renew action to the queue. */
/**
* Add a renew action to the queue.
*
* @param <T> generic type T.
* @param fs file system.
* @return renew action.
* */
@SuppressWarnings("static-access")
public <T extends FileSystem & Renewable> RenewAction<T> addRenewAction(final T fs) {
synchronized (this) {
@ -230,8 +244,10 @@ public <T extends FileSystem & Renewable> RenewAction<T> addRenewAction(final T
/**
* Remove the associated renew action from the queue
*
* @throws IOException
*
* @param <T> generic type T.
* @param fs file system.
* @throws IOException raised on errors performing I/O.
*/
public <T extends FileSystem & Renewable> void removeRenewAction(
final T fs) throws IOException {
@ -240,9 +256,8 @@ public <T extends FileSystem & Renewable> void removeRenewAction(
try {
action.cancel();
} catch (InterruptedException ie) {
LOG.error("Interrupted while canceling token for " + fs.getUri()
+ "filesystem");
LOG.debug("Exception in removeRenewAction: {}", ie);
LOG.error("Interrupted while canceling token for {} filesystem.", fs.getUri());
LOG.debug("Exception in removeRenewAction.", ie);
}
}
}

45
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSBuilder.java
View File

@ -37,12 +37,17 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set optional Builder parameter.
* @param key key.
* @param value value.
* @return generic type B.
*/
B opt(@Nonnull String key, @Nonnull String value);
/**
* Set optional boolean parameter for the Builder.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, boolean value);
@ -50,6 +55,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set optional int parameter for the Builder.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, int value);
@ -57,6 +65,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set optional float parameter for the Builder.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, float value);
@ -64,6 +75,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set optional long parameter for the Builder.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, long value);
@ -71,6 +85,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set optional double parameter for the Builder.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, double value);
@ -78,6 +95,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set an array of string values as optional parameter for the Builder.
*
* @param key key.
* @param values values.
* @return generic type B.
* @see #opt(String, String)
*/
B opt(@Nonnull String key, @Nonnull String... values);
@ -87,12 +107,19 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
*
* If the option is not supported or unavailable,
* the client should expect {@link #build()} throws IllegalArgumentException.
*
* @param key key.
* @param value value.
* @return generic type B.
*/
B must(@Nonnull String key, @Nonnull String value);
/**
* Set mandatory boolean option.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, boolean value);
@ -100,6 +127,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set mandatory int option.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, int value);
@ -107,6 +137,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set mandatory float option.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, float value);
@ -114,6 +147,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set mandatory long option.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, long value);
@ -121,6 +157,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set mandatory double option.
*
* @param key key.
* @param value value.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, double value);
@ -128,6 +167,9 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
/**
* Set a string array as mandatory option.
*
* @param key key.
* @param values values.
* @return generic type B.
* @see #must(String, String)
*/
B must(@Nonnull String key, @Nonnull String... values);
@ -139,6 +181,7 @@ public interface FSBuilder<S, B extends FSBuilder<S, B>> {
* @throws UnsupportedOperationException if the filesystem does not support
* the specific operation.
* @throws IOException on filesystem IO errors.
* @return generic type S.
*/
S build() throws IllegalArgumentException,
UnsupportedOperationException, IOException;

25
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataInputStream.java
View File

@ -1,4 +1,4 @@
/**
/*
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -26,6 +26,8 @@
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.List;
import java.util.function.IntFunction;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -51,7 +53,7 @@ public class FSDataInputStream extends DataInputStream
*/
private final IdentityHashStore<ByteBuffer, ByteBufferPool>
extendedReadBuffers
= new IdentityHashStore<ByteBuffer, ByteBufferPool>(0);
= new IdentityHashStore<>(0);
public FSDataInputStream(InputStream in) {
super(in);
@ -142,7 +144,8 @@ public boolean seekToNewSource(long targetPos) throws IOException {
*
* @return the underlying input stream
*/
@InterfaceAudience.LimitedPrivate({"HDFS"})
@InterfaceAudience.Public
@InterfaceStability.Stable
public InputStream getWrappedStream() {
return in;
}
@ -279,4 +282,20 @@ public void readFully(long position, ByteBuffer buf) throws IOException {
public IOStatistics getIOStatistics() {
return IOStatisticsSupport.retrieveIOStatistics(in);
}
@Override
public int minSeekForVectorReads() {
return ((PositionedReadable) in).minSeekForVectorReads();
}
@Override
public int maxReadSizeForVectorReads() {
return ((PositionedReadable) in).maxReadSizeForVectorReads();
}
@Override
public void readVectored(List<? extends FileRange> ranges,
IntFunction<ByteBuffer> allocate) throws IOException {
((PositionedReadable) in).readVectored(ranges, allocate);
}
}

32
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
View File

@ -123,6 +123,9 @@ public abstract class FSDataOutputStreamBuilder
/**
* Constructor.
*
* @param fileSystem file system.
* @param p the path.
*/
protected FSDataOutputStreamBuilder(@Nonnull FileSystem fileSystem,
@Nonnull Path p) {
@ -149,6 +152,9 @@ protected FsPermission getPermission() {
/**
* Set permission for the file.
*
* @param perm permission.
* @return B Generics Type.
*/
public B permission(@Nonnull final FsPermission perm) {
checkNotNull(perm);
@ -162,6 +168,9 @@ protected int getBufferSize() {
/**
* Set the size of the buffer to be used.
*
* @param bufSize buffer size.
* @return Generics Type B.
*/
public B bufferSize(int bufSize) {
bufferSize = bufSize;
@ -174,6 +183,9 @@ protected short getReplication() {
/**
* Set replication factor.
*
* @param replica replica.
* @return Generics Type B.
*/
public B replication(short replica) {
replication = replica;
@ -186,6 +198,9 @@ protected long getBlockSize() {
/**
* Set block size.
*
* @param blkSize block size.
* @return B Generics Type.
*/
public B blockSize(long blkSize) {
blockSize = blkSize;
@ -194,6 +209,8 @@ public B blockSize(long blkSize) {
/**
* Return true to create the parent directories if they do not exist.
*
* @return if create the parent directories if they do not exist true,not false.
*/
protected boolean isRecursive() {
return recursive;
@ -201,6 +218,8 @@ protected boolean isRecursive() {
/**
* Create the parent directory if they do not exist.
*
* @return B Generics Type.
*/
public B recursive() {
recursive = true;
@ -213,6 +232,9 @@ protected Progressable getProgress() {
/**
* Set the facility of reporting progress.
*
* @param prog progress.
* @return B Generics Type.
*/
public B progress(@Nonnull final Progressable prog) {
checkNotNull(prog);
@ -226,6 +248,8 @@ protected EnumSet<CreateFlag> getFlags() {
/**
* Create an FSDataOutputStream at the specified path.
*
* @return return Generics Type B.
*/
public B create() {
flags.add(CreateFlag.CREATE);
@ -236,6 +260,9 @@ public B create() {
* Set to true to overwrite the existing file.
* Set it to false, an exception will be thrown when calling {@link #build()}
* if the file exists.
*
* @param overwrite overrite.
* @return Generics Type B.
*/
public B overwrite(boolean overwrite) {
if (overwrite) {
@ -248,6 +275,8 @@ public B overwrite(boolean overwrite) {
/**
* Append to an existing file (optional operation).
*
* @return Generics Type B.
*/
public B append() {
flags.add(CreateFlag.APPEND);
@ -260,6 +289,9 @@ protected ChecksumOpt getChecksumOpt() {
/**
* Set checksum opt.
*
* @param chksumOpt check sum opt.
* @return Generics Type B.
*/
public B checksumOpt(@Nonnull final ChecksumOpt chksumOpt) {
checkNotNull(chksumOpt);

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSInputChecker.java
View File

@ -82,6 +82,7 @@ protected FSInputChecker( Path file, int numOfRetries) {
* @param sum the type of Checksum engine
* @param chunkSize maximun chunk size
* @param checksumSize the number byte of each checksum
* @param verifyChecksum verify check sum.
*/
protected FSInputChecker( Path file, int numOfRetries,
boolean verifyChecksum, Checksum sum, int chunkSize, int checksumSize ) {
@ -118,6 +119,7 @@ protected FSInputChecker( Path file, int numOfRetries,
* @param len maximum number of bytes to read
* @param checksum the data buffer into which to write checksums
* @return number of bytes read
* @throws IOException raised on errors performing I/O.
*/
abstract protected int readChunk(long pos, byte[] buf, int offset, int len,
byte[] checksum) throws IOException;
@ -129,7 +131,10 @@ abstract protected int readChunk(long pos, byte[] buf, int offset, int len,
*/
abstract protected long getChunkPosition(long pos);
/** Return true if there is a need for checksum verification */
/**
* Return true if there is a need for checksum verification.
* @return if there is a need for checksum verification true, not false.
*/
protected synchronized boolean needChecksum() {
return verifyChecksum && sum != null;
}
@ -357,6 +362,9 @@ private void verifySums(final byte b[], final int off, int read)
* Convert a checksum byte array to a long
* This is deprecated since 0.22 since it is no longer in use
* by this class.
*
* @param checksum check sum.
* @return crc.
*/
@Deprecated
static public long checksum2long(byte[] checksum) {

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java
View File

@ -74,7 +74,7 @@ abstract public T next(final AbstractFileSystem fs, final Path p)
* @param fc FileContext used to access file systems.
* @param path The path to resolve symlinks on.
* @return Generic type determined by the implementation of next.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public T resolve(final FileContext fc, final Path path) throws IOException {
int count = 0;

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
View File

@ -186,6 +186,8 @@ public void flush() throws IOException {
/**
* Return the number of valid bytes currently in the buffer.
*
* @return buffer data size.
*/
protected synchronized int getBufferedDataSize() {
return count;
@ -227,6 +229,10 @@ private void writeChecksumChunks(byte b[], int off, int len)
/**
* Converts a checksum integer value to a byte stream
*
* @param sum check sum.
* @param checksumSize check sum size.
* @return byte stream.
*/
static public byte[] convertToByteStream(Checksum sum, int checksumSize) {
return int2byte((int)sum.getValue(), new byte[checksumSize]);
@ -245,6 +251,8 @@ static byte[] int2byte(int integer, byte[] bytes) {
/**
* Resets existing buffer with a new one of the specified size.
*
* @param size size.
*/
protected synchronized void setChecksumBufSize(int size) {
this.buf = new byte[size];

25
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileChecksum.java
View File

@ -28,20 +28,37 @@
@InterfaceAudience.Public
@InterfaceStability.Stable
public abstract class FileChecksum implements Writable {
/** The checksum algorithm name */
/**
* The checksum algorithm name.
*
* @return algorithm name.
*/
public abstract String getAlgorithmName();
/** The length of the checksum in bytes */
/**
* The length of the checksum in bytes.
*
* @return length.
*/
public abstract int getLength();
/** The value of the checksum in bytes */
/**
* The value of the checksum in bytes.
*
* @return byte array.
*/
public abstract byte[] getBytes();
public ChecksumOpt getChecksumOpt() {
return null;
}
/** Return true if both the algorithms and the values are the same. */
/**
* Return true if both the algorithms and the values are the same.
*
* @param other other.
* @return if equal true, not false.
*/
@Override
public boolean equals(Object other) {
if (other == this) {

115
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
View File

@ -366,8 +366,8 @@ public AbstractFileSystem run() throws UnsupportedFileSystemException {
* Create a FileContext with specified FS as default using the specified
* config.
*
* @param defFS
* @param aConf
* @param defFS default fs.
* @param aConf configutration.
* @return new FileContext with specified FS as default.
*/
public static FileContext getFileContext(final AbstractFileSystem defFS,
@ -378,7 +378,7 @@ public static FileContext getFileContext(final AbstractFileSystem defFS,
/**
* Create a FileContext for specified file system using the default config.
*
* @param defaultFS
* @param defaultFS default fs.
* @return a FileContext with the specified AbstractFileSystem
* as the default FS.
*/
@ -411,6 +411,7 @@ protected static FileContext getFileContext(
*
* @throws UnsupportedFileSystemException If the file system from the default
* configuration is not supported
* @return file context.
*/
public static FileContext getFileContext()
throws UnsupportedFileSystemException {
@ -430,7 +431,7 @@ public static FileContext getLocalFSFileContext()
/**
* Create a FileContext for specified URI using the default config.
*
* @param defaultFsUri
* @param defaultFsUri defaultFsUri.
* @return a FileContext with the specified URI as the default FS.
*
* @throws UnsupportedFileSystemException If the file system for
@ -444,8 +445,8 @@ public static FileContext getFileContext(final URI defaultFsUri)
/**
* Create a FileContext for specified default URI using the specified config.
*
* @param defaultFsUri
* @param aConf
* @param defaultFsUri defaultFsUri.
* @param aConf configrution.
* @return new FileContext for specified uri
* @throws UnsupportedFileSystemException If the file system with specified is
* not supported
@ -476,7 +477,7 @@ public static FileContext getFileContext(final URI defaultFsUri,
* {@link #getFileContext(URI, Configuration)} instead of this one.
*
*
* @param aConf
* @param aConf configration.
* @return new FileContext
* @throws UnsupportedFileSystemException If file system in the config
* is not supported
@ -554,6 +555,7 @@ public void setWorkingDirectory(final Path newWDir) throws IOException {
/**
* Gets the working directory for wd-relative names (such a "foo/bar").
* @return the path.
*/
public Path getWorkingDirectory() {
return workingDir;
@ -600,13 +602,14 @@ public void setUMask(final FsPermission newUmask) {
* @throws FileNotFoundException If <code>f</code> does not exist
* @throws AccessControlException if access denied
* @throws IOException If an IO Error occurred
*
* @throws UnresolvedLinkException If unresolved link occurred.
*
* Exceptions applicable to file systems accessed over RPC:
* @throws RpcClientException If an exception occurred in the RPC client
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
*
*
* RuntimeExceptions:
* @throws InvalidPathException If path <code>f</code> is not valid
*/
@ -620,7 +623,7 @@ public Path resolvePath(final Path f) throws FileNotFoundException,
* A Fully-qualified path has scheme and authority specified and an absolute
* path.
* Use the default file system and working dir in this FileContext to qualify.
* @param path
* @param path the path.
* @return qualified path
*/
public Path makeQualified(final Path path) {
@ -759,6 +762,7 @@ public FSDataOutputStream build() throws IOException {
*
* Client should expect {@link FSDataOutputStreamBuilder#build()} throw the
* same exceptions as create(Path, EnumSet, CreateOpts...).
* @throws IOException If an I/O error occurred.
*/
public FSDataOutputStreamBuilder<FSDataOutputStream, ?> create(final Path f)
throws IOException {
@ -832,6 +836,8 @@ public Void next(final AbstractFileSystem fs, final Path p)
*
* RuntimeExceptions:
* @throws InvalidPathException If path <code>f</code> is invalid
*
* @return if delete success true, not false.
*/
public boolean delete(final Path f, final boolean recursive)
throws AccessControlException, FileNotFoundException,
@ -862,6 +868,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
* @return input stream.
*/
public FSDataInputStream open(final Path f) throws AccessControlException,
FileNotFoundException, UnsupportedFileSystemException, IOException {
@ -892,6 +899,7 @@ public FSDataInputStream next(final AbstractFileSystem fs, final Path p)
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
* @return output stream.
*/
public FSDataInputStream open(final Path f, final int bufferSize)
throws AccessControlException, FileNotFoundException,
@ -1001,6 +1009,7 @@ public Boolean next(final AbstractFileSystem fs, final Path p)
*
* @param src path to be renamed
* @param dst new path after rename
* @param options rename options.
*
* @throws AccessControlException If access is denied
* @throws FileAlreadyExistsException If <code>dst</code> already exists and
@ -1052,7 +1061,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
/**
* Set permission of a path.
* @param f
* @param f the path.
* @param permission - the new absolute permission (umask is not applied)
*
* @throws AccessControlException If access is denied
@ -1196,7 +1205,7 @@ public FileChecksum next(final AbstractFileSystem fs, final Path p)
* Set the verify checksum flag for the file system denoted by the path.
* This is only applicable if the
* corresponding FileSystem supports checksum. By default doesn't do anything.
* @param verifyChecksum
* @param verifyChecksum verify check sum.
* @param f set the verifyChecksum for the Filesystem containing this path
*
* @throws AccessControlException If access is denied
@ -1251,8 +1260,9 @@ public FileStatus next(final AbstractFileSystem fs, final Path p)
/**
* Synchronize client metadata state.
*
* @throws IOException
* @throws UnsupportedOperationException
* @throws IOException If an I/O error occurred.
* @throws UnsupportedOperationException If file system for <code>f</code> is
* not supported.
*/
public void msync() throws IOException, UnsupportedOperationException {
defaultFS.msync();
@ -1613,9 +1623,12 @@ public RemoteIterator<FileStatus> next(
}
/**
* List CorruptFile Blocks.
*
* @param path the path.
* @return an iterator over the corrupt files under the given path
* (may contain duplicates if a file has more than one corrupt block)
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public RemoteIterator<Path> listCorruptFileBlocks(Path path)
throws IOException {
@ -1739,6 +1752,7 @@ public class Util {
* @throws RpcServerException If an exception occurred in the RPC server
* @throws UnexpectedServerException If server implementation throws
* undeclared exception to RPC server
* @return if f exists true, not false.
*/
public boolean exists(final Path f) throws AccessControlException,
UnsupportedFileSystemException, IOException {
@ -1799,6 +1813,12 @@ public ContentSummary getContentSummary(Path f)
/**
* See {@link #listStatus(Path[], PathFilter)}
*
* @param files files.
* @throws AccessControlException If access is denied.
* @throws FileNotFoundException If <code>files</code> does not exist.
* @throws IOException If an I/O error occurred.
* @return file status array.
*/
public FileStatus[] listStatus(Path[] files) throws AccessControlException,
FileNotFoundException, IOException {
@ -2054,36 +2074,29 @@ public LocatedFileStatus next() throws IOException {
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
* <p>
* <dt> <tt> * </tt>
* <dd> Matches zero or more characters.
*
* <p>
* <dt> <tt> [<i>abc</i>] </tt>
* <dd> Matches a single character from character set
* <tt>{<i>a,b,c</i>}</tt>.
*
* <p>
* <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
* <dd> Matches a single character from the character range
* <tt>{<i>a...b</i>}</tt>. Note: character <tt><i>a</i></tt> must be
* lexicographically less than or equal to character <tt><i>b</i></tt>.
*
* <p>
* <dt> <tt> [^<i>a</i>] </tt>
* <dd> Matches a single char that is not from character set or range
* <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
* immediately to the right of the opening bracket.
*
* <p>
* <dt> <tt> \<i>c</i> </tt>
* <dd> Removes (escapes) any special meaning of character <i>c</i>.
*
* <p>
* <dt> <tt> {ab,cd} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
*
* <p>
*
* <dt> <tt> {ab,c{de,fh}} </tt>
* <dd> Matches a string from string set <tt>{<i>ab, cde, cfh</i>}</tt>
*
@ -2144,6 +2157,18 @@ public FileStatus[] globStatus(final Path pathPattern,
/**
* Copy file from src to dest. See
* {@link #copy(Path, Path, boolean, boolean)}
*
* @param src src.
* @param dst dst.
* @throws AccessControlException If access is denied.
* @throws FileAlreadyExistsException If file <code>src</code> already exists.
* @throws FileNotFoundException if next file does not exist any more.
* @throws ParentNotDirectoryException If parent of <code>src</code> is not a
* directory.
* @throws UnsupportedFileSystemException If file system for
* <code>src/dst</code> is not supported.
* @throws IOException If an I/O error occurred.
* @return if success copy true, not false.
*/
public boolean copy(final Path src, final Path dst)
throws AccessControlException, FileAlreadyExistsException,
@ -2154,8 +2179,8 @@ public boolean copy(final Path src, final Path dst)
/**
* Copy from src to dst, optionally deleting src and overwriting dst.
* @param src
* @param dst
* @param src src.
* @param dst dst.
* @param deleteSource - delete src if true
* @param overwrite overwrite dst if true; throw IOException if dst exists
* and overwrite is false.
@ -2276,7 +2301,7 @@ private static void checkDependencies(Path qualSrc, Path qualDst)
* Are qualSrc and qualDst of the same file system?
* @param qualPath1 - fully qualified path
* @param qualPath2 - fully qualified path
* @return
* @return is same fs true,not false.
*/
private static boolean isSameFS(Path qualPath1, Path qualPath2) {
URI srcUri = qualPath1.toUri();
@ -2299,6 +2324,13 @@ public synchronized void run() {
/**
* Resolves all symbolic links in the specified path.
* Returns the new path object.
*
* @param f the path.
* @throws FileNotFoundException If <code>f</code> does not exist.
* @throws UnresolvedLinkException If unresolved link occurred.
* @throws AccessControlException If access is denied.
* @throws IOException If an I/O error occurred.
* @return resolve path.
*/
protected Path resolve(final Path f) throws FileNotFoundException,
UnresolvedLinkException, AccessControlException, IOException {
@ -2316,6 +2348,7 @@ public Path next(final AbstractFileSystem fs, final Path p)
* to, but not including the final path component.
* @param f path to resolve
* @return the new path object.
* @throws IOException If an I/O error occurred.
*/
protected Path resolveIntermediate(final Path f) throws IOException {
return new FSLinkResolver<FileStatus>() {
@ -2334,13 +2367,12 @@ public FileStatus next(final AbstractFileSystem fs, final Path p)
* @param f
* Path which needs to be resolved
* @return List of AbstractFileSystems accessed in the path
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
Set<AbstractFileSystem> resolveAbstractFileSystems(final Path f)
throws IOException {
final Path absF = fixRelativePart(f);
final HashSet<AbstractFileSystem> result
= new HashSet<AbstractFileSystem>();
final HashSet<AbstractFileSystem> result = new HashSet<>();
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
@ -2395,7 +2427,7 @@ public static Map<URI, Statistics> getAllStatistics() {
* @param p Path for which delegations tokens are requested.
* @param renewer the account name that is allowed to renew the token.
* @return List of delegation tokens.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
@InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" })
public List<Token<?>> getDelegationTokens(
@ -2547,7 +2579,7 @@ public AclStatus next(final AbstractFileSystem fs, final Path p)
* @param path Path to modify
* @param name xattr name.
* @param value xattr value.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void setXAttr(Path path, String name, byte[] value)
throws IOException {
@ -2566,7 +2598,7 @@ public void setXAttr(Path path, String name, byte[] value)
* @param name xattr name.
* @param value xattr value.
* @param flag xattr set flag
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void setXAttr(Path path, final String name, final byte[] value,
final EnumSet<XAttrSetFlag> flag) throws IOException {
@ -2591,7 +2623,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
* @param path Path to get extended attribute
* @param name xattr name.
* @return byte[] xattr value.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public byte[] getXAttr(Path path, final String name) throws IOException {
final Path absF = fixRelativePart(path);
@ -2614,7 +2646,7 @@ public byte[] next(final AbstractFileSystem fs, final Path p)
* @param path Path to get extended attributes
* @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
* of the file or directory
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public Map<String, byte[]> getXAttrs(Path path) throws IOException {
final Path absF = fixRelativePart(path);
@ -2638,7 +2670,7 @@ public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p)
* @param names XAttr names.
* @return Map{@literal <}String, byte[]{@literal >} describing the XAttrs
* of the file or directory
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public Map<String, byte[]> getXAttrs(Path path, final List<String> names)
throws IOException {
@ -2661,7 +2693,7 @@ public Map<String, byte[]> next(final AbstractFileSystem fs, final Path p)
*
* @param path Path to remove extended attribute
* @param name xattr name
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void removeXAttr(Path path, final String name) throws IOException {
final Path absF = fixRelativePart(path);
@ -2685,7 +2717,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
* @param path Path to get extended attributes
* @return List{@literal <}String{@literal >} of the XAttr names of the
* file or directory
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public List<String> listXAttrs(Path path) throws IOException {
final Path absF = fixRelativePart(path);
@ -2802,7 +2834,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void satisfyStoragePolicy(final Path path)
throws IOException {
@ -2824,6 +2856,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
* @param policyName the name of the target storage policy. The list
* of supported Storage policies can be retrieved
* via {@link #getAllStoragePolicies}.
* @throws IOException If an I/O error occurred.
*/
public void setStoragePolicy(final Path path, final String policyName)
throws IOException {
@ -2841,7 +2874,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
/**
* Unset the storage policy set for a given file or directory.
* @param src file or directory path.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void unsetStoragePolicy(final Path src) throws IOException {
final Path absF = fixRelativePart(src);
@ -2860,7 +2893,7 @@ public Void next(final AbstractFileSystem fs, final Path p)
*
* @param path file or directory path.
* @return storage policy for give file.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public BlockStoragePolicySpi getStoragePolicy(Path path) throws IOException {
final Path absF = fixRelativePart(path);
@ -2878,7 +2911,7 @@ public BlockStoragePolicySpi next(final AbstractFileSystem fs,
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException {

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
View File

@ -52,6 +52,7 @@ public class FileEncryptionInfo implements Serializable {
* @param keyName name of the key used for the encryption zone
* @param ezKeyVersionName name of the KeyVersion used to encrypt the
* encrypted data encryption key.
* @param version version.
*/
public FileEncryptionInfo(final CipherSuite suite,
final CryptoProtocolVersion version, final byte[] edek,
@ -134,6 +135,8 @@ public String toString() {
*
* NOTE:
* Currently this method is used by CLI for backward compatibility.
*
* @return stable string.
*/
public String toStringStable() {
StringBuilder builder = new StringBuilder("{")

87
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileRange.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import java.nio.ByteBuffer;
import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.fs.impl.FileRangeImpl;
/**
* A byte range of a file.
* This is used for the asynchronous gather read API of
* {@link PositionedReadable#readVectored}.
*/
public interface FileRange {
/**
* Get the starting offset of the range.
* @return the byte offset of the start
*/
long getOffset();
/**
* Get the length of the range.
* @return the number of bytes in the range.
*/
int getLength();
/**
* Get the future data for this range.
* @return the future for the {@link ByteBuffer} that contains the data
*/
CompletableFuture<ByteBuffer> getData();
/**
* Set a future for this range's data.
* This method is called by {@link PositionedReadable#readVectored} to store the
* data for the user to pick up later via {@link #getData}.
* @param data the future of the ByteBuffer that will have the data
*/
void setData(CompletableFuture<ByteBuffer> data);
/**
* Get any reference passed in to the file range constructor.
* This is not used by any implementation code; it is to help
* bind this API to libraries retrieving multiple stripes of
* data in parallel.
* @return a reference or null.
*/
Object getReference();
/**
* Factory method to create a FileRange object.
* @param offset starting offset of the range.
* @param length length of the range.
* @return a new instance of FileRangeImpl.
*/
static FileRange createFileRange(long offset, int length) {
return new FileRangeImpl(offset, length, null);
}
/**
* Factory method to create a FileRange object.
* @param offset starting offset of the range.
* @param length length of the range.
* @param reference nullable reference to store in the range.
* @return a new instance of FileRangeImpl.
*/
static FileRange createFileRange(long offset, int length, Object reference) {
return new FileRangeImpl(offset, length, reference);
}
}

20
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
View File

@ -116,6 +116,17 @@ public FileStatus(long length, boolean isdir, int block_replication,
/**
* Constructor for file systems on which symbolic links are not supported
*
* @param length length.
* @param isdir isdir.
* @param block_replication block replication.
* @param blocksize block size.
* @param modification_time modification time.
* @param access_time access_time.
* @param permission permission.
* @param owner owner.
* @param group group.
* @param path the path.
*/
public FileStatus(long length, boolean isdir,
int block_replication,
@ -182,6 +193,7 @@ public FileStatus(long length, boolean isdir, int block_replication,
* Copy constructor.
*
* @param other FileStatus to copy
* @throws IOException raised on errors performing I/O.
*/
public FileStatus(FileStatus other) throws IOException {
// It's important to call the getters here instead of directly accessing the
@ -375,6 +387,8 @@ protected void setGroup(String group) {
/**
* @return The contents of the symbolic link.
*
* @throws IOException raised on errors performing I/O.
*/
public Path getSymlink() throws IOException {
if (!isSymlink()) {
@ -388,7 +402,8 @@ public void setSymlink(final Path p) {
}
/**
* Compare this FileStatus to another FileStatus
* Compare this FileStatus to another FileStatus based on lexicographical
* order of path.
* @param o the FileStatus to be compared.
* @return a negative integer, zero, or a positive integer as this object
* is less than, equal to, or greater than the specified object.
@ -398,7 +413,8 @@ public int compareTo(FileStatus o) {
}
/**
* Compare this FileStatus to another FileStatus.
* Compare this FileStatus to another FileStatus based on lexicographical
* order of path.
* This method was added back by HADOOP-14683 to keep binary compatibility.
*
* @param o the FileStatus to be compared.

210
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -21,7 +21,6 @@
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.lang.ref.WeakReference;
import java.lang.ref.ReferenceQueue;
import java.net.URI;
@ -104,13 +103,13 @@
* All user code that may potentially use the Hadoop Distributed
* File System should be written to use a FileSystem object or its
* successor, {@link FileContext}.
*
* </p>
* <p>
* The local implementation is {@link LocalFileSystem} and distributed
* implementation is DistributedFileSystem. There are other implementations
* for object stores and (outside the Apache Hadoop codebase),
* third party filesystems.
* <p>
* </p>
* Notes
* <ol>
* <li>The behaviour of the filesystem is
@ -133,13 +132,12 @@
* New methods may be marked as Unstable or Evolving for their initial release,
* as a warning that they are new and may change based on the
* experience of use in applications.
* <p></p>
* <p>
* <b>Important note for developers</b>
* <p></p>
* </p>
* If you are making changes here to the public API or protected methods,
* you must review the following subclasses and make sure that
* they are filtering/passing through new methods as appropriate.
* <p></p>
*
* {@link FilterFileSystem}: methods are passed through. If not,
* then {@code TestFilterFileSystem.MustNotImplement} must be
@ -148,21 +146,22 @@
* {@link #hasPathCapability(Path, String)} then
* {@link FilterFileSystem#hasPathCapability(Path, String)}
* must return false, always.
* <p></p>
* <p>
* {@link ChecksumFileSystem}: checksums are created and
* verified.
* <p></p>
* </p>
* {@code TestHarFileSystem} will need its {@code MustNotImplement}
* interface updated.
* <p></p>
*
* <p>
* There are some external places your changes will break things.
* Do co-ordinate changes here.
* <p></p>
* </p>
*
* HBase: HBoss
* <p></p>
* <p>
* Hive: HiveShim23
* </p>
* {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}
*
*****************************************************************/
@ -281,6 +280,8 @@ public FileSystem run() throws IOException {
/**
* Returns the configured FileSystem implementation.
* @param conf the configuration to use
* @return FileSystem.
* @throws IOException If an I/O error occurred.
*/
public static FileSystem get(Configuration conf) throws IOException {
return get(getDefaultUri(conf), conf);
@ -375,6 +376,7 @@ public String getScheme() {
* implement that method.
*
* @see #canonicalizeUri(URI)
* @return the URI of this filesystem.
*/
protected URI getCanonicalUri() {
return canonicalizeUri(getUri());
@ -391,6 +393,7 @@ protected URI getCanonicalUri() {
* not specified and if {@link #getDefaultPort()} returns a
* default port.
*
* @param uri url.
* @return URI
* @see NetUtils#getCanonicalUri(URI, int)
*/
@ -454,11 +457,21 @@ public String getCanonicalServiceName() {
: null;
}
/** @deprecated call {@link #getUri()} instead.*/
/**
* @return uri to string.
* @deprecated call {@link #getUri()} instead.
*/
@Deprecated
public String getName() { return getUri().toString(); }
/** @deprecated call {@link #get(URI, Configuration)} instead. */
/**
* @deprecated call {@link #get(URI, Configuration)} instead.
*
* @param name name.
* @param conf configuration.
* @return file system.
* @throws IOException If an I/O error occurred.
*/
@Deprecated
public static FileSystem getNamed(String name, Configuration conf)
throws IOException {
@ -513,6 +526,9 @@ public static LocalFileSystem getLocal(Configuration conf)
* configuration and URI, cached and returned to the caller.
* </li>
* </ol>
* @param uri uri of the filesystem.
* @param conf configrution.
* @return filesystem instance.
* @throws IOException if the FileSystem cannot be instantiated.
*/
public static FileSystem get(URI uri, Configuration conf) throws IOException {
@ -542,7 +558,7 @@ public static FileSystem get(URI uri, Configuration conf) throws IOException {
/**
* Returns the FileSystem for this URI's scheme and authority and the
* given user. Internally invokes {@link #newInstance(URI, Configuration)}
* @param uri of the filesystem
* @param uri uri of the filesystem.
* @param conf the configuration to use
* @param user to perform the get as
* @return filesystem instance
@ -860,6 +876,7 @@ protected void checkPath(Path path) {
* @param start offset into the given file
* @param len length for which to get locations for
* @throws IOException IO failure
* @return block location array.
*/
public BlockLocation[] getFileBlockLocations(FileStatus file,
long start, long len) throws IOException {
@ -900,6 +917,7 @@ public BlockLocation[] getFileBlockLocations(FileStatus file,
* @param len length for which to get locations for
* @throws FileNotFoundException when the path does not exist
* @throws IOException IO failure
* @return block location array.
*/
public BlockLocation[] getFileBlockLocations(Path p,
long start, long len) throws IOException {
@ -962,6 +980,7 @@ public Path resolvePath(final Path p) throws IOException {
* @param f the file name to open
* @param bufferSize the size of the buffer to be used.
* @throws IOException IO failure
* @return input stream.
*/
public abstract FSDataInputStream open(Path f, int bufferSize)
throws IOException;
@ -970,6 +989,7 @@ public abstract FSDataInputStream open(Path f, int bufferSize)
* Opens an FSDataInputStream at the indicated Path.
* @param f the file to open
* @throws IOException IO failure
* @return input stream.
*/
public FSDataInputStream open(Path f) throws IOException {
return open(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@ -987,6 +1007,7 @@ public FSDataInputStream open(Path f) throws IOException {
* @throws IOException IO failure
* @throws UnsupportedOperationException If {@link #open(PathHandle, int)}
* not overridden by subclass
* @return input stream.
*/
public FSDataInputStream open(PathHandle fd) throws IOException {
return open(fd, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@ -1004,6 +1025,7 @@ public FSDataInputStream open(PathHandle fd) throws IOException {
* not satisfied
* @throws IOException IO failure
* @throws UnsupportedOperationException If not overridden by subclass
* @return input stream.
*/
public FSDataInputStream open(PathHandle fd, int bufferSize)
throws IOException {
@ -1021,6 +1043,7 @@ public FSDataInputStream open(PathHandle fd, int bufferSize)
* not overridden by subclass.
* @throws UnsupportedOperationException If this FileSystem cannot enforce
* the specified constraints.
* @return path handle.
*/
public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
// method is final with a default so clients calling getPathHandle(stat)
@ -1036,6 +1059,7 @@ public final PathHandle getPathHandle(FileStatus stat, HandleOpt... opt) {
* @param stat Referent in the target FileSystem
* @param opt Constraints that determine the validity of the
* {@link PathHandle} reference.
* @return path handle.
*/
protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
throw new UnsupportedOperationException();
@ -1046,6 +1070,7 @@ protected PathHandle createPathHandle(FileStatus stat, HandleOpt... opt) {
* Files are overwritten by default.
* @param f the file to create
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f) throws IOException {
return create(f, true);
@ -1057,6 +1082,7 @@ public FSDataOutputStream create(Path f) throws IOException {
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an exception will be thrown.
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f, boolean overwrite)
throws IOException {
@ -1074,6 +1100,7 @@ public FSDataOutputStream create(Path f, boolean overwrite)
* @param f the file to create
* @param progress to report progress
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f, Progressable progress)
throws IOException {
@ -1090,6 +1117,7 @@ public FSDataOutputStream create(Path f, Progressable progress)
* @param f the file to create
* @param replication the replication factor
* @throws IOException IO failure
* @return output stream1
*/
public FSDataOutputStream create(Path f, short replication)
throws IOException {
@ -1108,6 +1136,7 @@ public FSDataOutputStream create(Path f, short replication)
* @param replication the replication factor
* @param progress to report progress
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f, short replication,
Progressable progress) throws IOException {
@ -1125,6 +1154,7 @@ public FSDataOutputStream create(Path f, short replication,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@ -1144,7 +1174,9 @@ public FSDataOutputStream create(Path f,
* @param overwrite if a file with this name already exists, then if true,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param progress to report progress.
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@ -1164,7 +1196,9 @@ public FSDataOutputStream create(Path f,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
* @param blockSize the size of the buffer to be used.
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@ -1182,7 +1216,10 @@ public FSDataOutputStream create(Path f,
* the file will be overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used.
* @param replication required block replication for the file.
* @param blockSize the size of the buffer to be used.
* @param progress to report progress.
* @throws IOException IO failure
* @return output stream.
*/
public FSDataOutputStream create(Path f,
boolean overwrite,
@ -1209,6 +1246,7 @@ public FSDataOutputStream create(Path f,
* @param progress the progress reporter
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public abstract FSDataOutputStream create(Path f,
FsPermission permission,
@ -1230,6 +1268,7 @@ public abstract FSDataOutputStream create(Path f,
* @param progress the progress reporter
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public FSDataOutputStream create(Path f,
FsPermission permission,
@ -1256,6 +1295,7 @@ public FSDataOutputStream create(Path f,
* found in conf will be used.
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public FSDataOutputStream create(Path f,
FsPermission permission,
@ -1277,6 +1317,16 @@ public FSDataOutputStream create(Path f,
* the permission with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
*
* @param f path.
* @param absolutePermission permission.
* @param flag create flag.
* @param bufferSize buffer size.
* @param replication replication.
* @param blockSize block size.
* @param progress progress.
* @param checksumOpt check sum opt.
* @return output stream.
* @throws IOException IO failure
*/
@Deprecated
@ -1331,6 +1381,11 @@ protected boolean primitiveMkdir(Path f, FsPermission absolutePermission)
* with umask before calling this method.
* This a temporary method added to support the transition from FileSystem
* to FileContext for user applications.
*
* @param f the path.
* @param absolutePermission permission.
* @param createParent create parent.
* @throws IOException IO failure.
*/
@Deprecated
protected void primitiveMkdir(Path f, FsPermission absolutePermission,
@ -1370,6 +1425,7 @@ protected void primitiveMkdir(Path f, FsPermission absolutePermission,
* @param progress the progress reporter
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public FSDataOutputStream createNonRecursive(Path f,
boolean overwrite,
@ -1393,6 +1449,7 @@ public FSDataOutputStream createNonRecursive(Path f,
* @param progress the progress reporter
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize,
@ -1416,6 +1473,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
* @param progress the progress reporter
* @throws IOException IO failure
* @see #setPermission(Path, FsPermission)
* @return output stream.
*/
public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
EnumSet<CreateFlag> flags, int bufferSize, short replication, long blockSize,
@ -1430,6 +1488,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
* <i>Important: the default implementation is not atomic</i>
* @param f path to use for create
* @throws IOException IO failure
* @return if create new file success true,not false.
*/
public boolean createNewFile(Path f) throws IOException {
if (exists(f)) {
@ -1450,6 +1509,7 @@ public boolean createNewFile(Path f) throws IOException {
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
* @return output stream.
*/
public FSDataOutputStream append(Path f) throws IOException {
return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
@ -1464,6 +1524,7 @@ public FSDataOutputStream append(Path f) throws IOException {
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
* @return output stream.
*/
public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
return append(f, bufferSize, null);
@ -1477,10 +1538,44 @@ public FSDataOutputStream append(Path f, int bufferSize) throws IOException {
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
* @return output stream.
*/
public abstract FSDataOutputStream append(Path f, int bufferSize,
Progressable progress) throws IOException;
/**
* Append to an existing file (optional operation).
* @param f the existing file to be appended.
* @param appendToNewBlock whether to append data to a new block
* instead of the end of the last partial block
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
* @return output stream.
*/
public FSDataOutputStream append(Path f, boolean appendToNewBlock) throws IOException {
return append(f, getConf().getInt(IO_FILE_BUFFER_SIZE_KEY,
IO_FILE_BUFFER_SIZE_DEFAULT), null, appendToNewBlock);
}
/**
* Append to an existing file (optional operation).
* This function is used for being overridden by some FileSystem like DistributedFileSystem
* @param f the existing file to be appended.
* @param bufferSize the size of the buffer to be used.
* @param progress for reporting progress if it is not null.
* @param appendToNewBlock whether to append data to a new block
* instead of the end of the last partial block
* @throws IOException IO failure
* @throws UnsupportedOperationException if the operation is unsupported
* (default).
* @return output stream.
*/
public FSDataOutputStream append(Path f, int bufferSize,
Progressable progress, boolean appendToNewBlock) throws IOException {
return append(f, bufferSize, progress);
}
/**
* Concat existing files together.
* @param trg the path to the target destination.
@ -1515,7 +1610,7 @@ public short getReplication(Path src) throws IOException {
* This is the default behavior.
* @param src file name
* @param replication new replication
* @throws IOException
* @throws IOException an IO failure.
* @return true if successful, or the feature in unsupported;
* false if replication is supported but the file does not exist,
* or is a directory
@ -1544,11 +1639,12 @@ public boolean setReplication(Path src, short replication)
* <p>
* If OVERWRITE option is not passed as an argument, rename fails
* if the dst already exists.
* </p>
* <p>
* If OVERWRITE option is passed as an argument, rename overwrites
* the dst if it is a file or an empty directory. Rename fails if dst is
* a non-empty directory.
* <p>
* </p>
* Note that atomicity of rename is dependent on the file system
* implementation. Please refer to the file system documentation for
* details. This default implementation is non atomic.
@ -1556,9 +1652,11 @@ public boolean setReplication(Path src, short replication)
* This method is deprecated since it is a temporary method added to
* support the transition from FileSystem to FileContext for user
* applications.
* </p>
*
* @param src path to be renamed
* @param dst new path after rename
* @param options rename options.
* @throws FileNotFoundException src path does not exist, or the parent
* path of dst does not exist.
* @throws FileAlreadyExistsException dest path exists and is a file
@ -1653,6 +1751,9 @@ public boolean truncate(Path f, long newLength) throws IOException {
/**
* Delete a file/directory.
* @param f the path.
* @throws IOException IO failure.
* @return if delete success true, not false.
* @deprecated Use {@link #delete(Path, boolean)} instead.
*/
@Deprecated
@ -1769,6 +1870,7 @@ public boolean exists(Path f) throws IOException {
* @param f path to check
* @throws IOException IO failure
* @deprecated Use {@link #getFileStatus(Path)} instead
* @return if f is directory true, not false.
*/
@Deprecated
public boolean isDirectory(Path f) throws IOException {
@ -1786,6 +1888,7 @@ public boolean isDirectory(Path f) throws IOException {
* @param f path to check
* @throws IOException IO failure
* @deprecated Use {@link #getFileStatus(Path)} instead
* @return if f is file true, not false.
*/
@Deprecated
public boolean isFile(Path f) throws IOException {
@ -1798,6 +1901,7 @@ public boolean isFile(Path f) throws IOException {
/**
* The number of bytes in a file.
* @param f the path.
* @return the number of bytes; 0 for a directory
* @deprecated Use {@link #getFileStatus(Path)} instead.
* @throws FileNotFoundException if the path does not resolve
@ -1812,6 +1916,7 @@ public long getLength(Path f) throws IOException {
* @param f path to use
* @throws FileNotFoundException if the path does not resolve
* @throws IOException IO failure
* @return content summary.
*/
public ContentSummary getContentSummary(Path f) throws IOException {
FileStatus status = getFileStatus(f);
@ -1946,9 +2051,9 @@ public boolean hasMore() {
* @param f Path to list
* @param token opaque iteration token returned by previous call, or null
* if this is the first call.
* @return
* @throws FileNotFoundException
* @throws IOException
* @return directory entries.
* @throws FileNotFoundException when the path does not exist.
* @throws IOException If an I/O error occurred.
*/
@InterfaceAudience.Private
protected DirectoryEntries listStatusBatch(Path f, byte[] token) throws
@ -1979,6 +2084,8 @@ private void listStatus(ArrayList<FileStatus> results, Path f,
/**
* List corrupted file blocks.
*
* @param path the path.
* @return an iterator over the corrupt files under the given path
* (may contain duplicates if a file has more than one corrupt block)
* @throws UnsupportedOperationException if the operation is unsupported
@ -2072,36 +2179,29 @@ public FileStatus[] listStatus(Path[] files, PathFilter filter)
* <dt> <tt> ? </tt>
* <dd> Matches any single character.
*
* <p>
* <dt> <tt> * </tt>
* <dd> Matches zero or more characters.
*
* <p>
* <dt> <tt> [<i>abc</i>] </tt>
* <dd> Matches a single character from character set
* <tt>{<i>a,b,c</i>}</tt>.
*
* <p>
* <dt> <tt> [<i>a</i>-<i>b</i>] </tt>
* <dd> Matches a single character from the character range
* <tt>{<i>a...b</i>}</tt>. Note that character <tt><i>a</i></tt> must be
* lexicographically less than or equal to character <tt><i>b</i></tt>.
*
* <p>
* <dt> <tt> [^<i>a</i>] </tt>
* <dd> Matches a single character that is not from character set or range
* <tt>{<i>a</i>}</tt>. Note that the <tt>^</tt> character must occur
* immediately to the right of the opening bracket.
*
* <p>
* <dt> <tt> \<i>c</i> </tt>
* <dd> Removes (escapes) any special meaning of character <i>c</i>.
*
* <p>
* <dt> <tt> {ab,cd} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cd</i>} </tt>
*
* <p>
* <dt> <tt> {ab,c{de,fh}} </tt>
* <dd> Matches a string from the string set <tt>{<i>ab, cde, cfh</i>}</tt>
*
@ -2332,6 +2432,7 @@ public LocatedFileStatus next() throws IOException {
/** Return the current user's home directory in this FileSystem.
* The default implementation returns {@code "/user/$USER/"}.
* @return the path.
*/
public Path getHomeDirectory() {
String username;
@ -2394,6 +2495,7 @@ public boolean mkdirs(Path f) throws IOException {
* @param f path to create
* @param permission to apply to f
* @throws IOException IO failure
* @return if mkdir success true, not false.
*/
public abstract boolean mkdirs(Path f, FsPermission permission
) throws IOException;
@ -2441,6 +2543,7 @@ public void moveFromLocalFile(Path src, Path dst)
* @param delSrc whether to delete the src
* @param src path
* @param dst path
* @throws IOException IO failure.
*/
public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
throws IOException {
@ -2555,6 +2658,7 @@ public void copyToLocalFile(boolean delSrc, Path src, Path dst,
* @param fsOutputFile path of output file
* @param tmpLocalFile path of local tmp file
* @throws IOException IO failure
* @return the path.
*/
public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
throws IOException {
@ -2602,6 +2706,7 @@ public void close() throws IOException {
/**
* Return the total size of all files in the filesystem.
* @throws IOException IO failure
* @return the number of path used.
*/
public long getUsed() throws IOException {
Path path = new Path("/");
@ -2610,7 +2715,9 @@ public long getUsed() throws IOException {
/**
* Return the total size of all files from a specified path.
* @param path the path.
* @throws IOException IO failure
* @return the number of path content summary.
*/
public long getUsed(Path path) throws IOException {
return getContentSummary(path).getLength();
@ -2633,6 +2740,7 @@ public long getBlockSize(Path f) throws IOException {
* Return the number of bytes that large input files should be optimally
* be split into to minimize I/O time.
* @deprecated use {@link #getDefaultBlockSize(Path)} instead
* @return default block size.
*/
@Deprecated
public long getDefaultBlockSize() {
@ -2685,8 +2793,8 @@ public short getDefaultReplication(Path path) {
* In some FileSystem implementations such as HDFS metadata
* synchronization is essential to guarantee consistency of read requests
* particularly in HA setting.
* @throws IOException
* @throws UnsupportedOperationException
* @throws IOException If an I/O error occurred.
* @throws UnsupportedOperationException if the operation is unsupported.
*/
public void msync() throws IOException, UnsupportedOperationException {
throw new UnsupportedOperationException(getClass().getCanonicalName() +
@ -2762,6 +2870,8 @@ static void checkAccessPermissions(FileStatus stat, FsAction mode)
/**
* See {@link FileContext#fixRelativePart}.
* @param p the path.
* @return relative part.
*/
protected Path fixRelativePart(Path p) {
if (p.isUriPathAbsolute()) {
@ -2773,6 +2883,18 @@ protected Path fixRelativePart(Path p) {
/**
* See {@link FileContext#createSymlink(Path, Path, boolean)}.
*
* @param target target path.
* @param link link.
* @param createParent create parent.
* @throws AccessControlException if access is denied.
* @throws FileAlreadyExistsException when the path does not exist.
* @throws FileNotFoundException when the path does not exist.
* @throws ParentNotDirectoryException if the parent path of dest is not
* a directory.
* @throws UnsupportedFileSystemException if there was no known implementation
* for the scheme.
* @throws IOException raised on errors performing I/O.
*/
public void createSymlink(final Path target, final Path link,
final boolean createParent) throws AccessControlException,
@ -2786,8 +2908,14 @@ public void createSymlink(final Path target, final Path link,
/**
* See {@link FileContext#getFileLinkStatus(Path)}.
* @throws FileNotFoundException when the path does not exist
* @throws IOException see specific implementation
*
* @param f the path.
* @throws AccessControlException if access is denied.
* @throws FileNotFoundException when the path does not exist.
* @throws IOException raised on errors performing I/O.
* @throws UnsupportedFileSystemException if there was no known implementation
* for the scheme.
* @return file status
*/
public FileStatus getFileLinkStatus(final Path f)
throws AccessControlException, FileNotFoundException,
@ -2798,6 +2926,7 @@ public FileStatus getFileLinkStatus(final Path f)
/**
* See {@link AbstractFileSystem#supportsSymlinks()}.
* @return if support symlinkls true, not false.
*/
public boolean supportsSymlinks() {
return false;
@ -2805,8 +2934,11 @@ public boolean supportsSymlinks() {
/**
* See {@link FileContext#getLinkTarget(Path)}.
* @param f the path.
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
* @throws IOException IO failure.
* @return the path.
*/
public Path getLinkTarget(Path f) throws IOException {
// Supporting filesystems should override this method
@ -2816,8 +2948,11 @@ public Path getLinkTarget(Path f) throws IOException {
/**
* See {@link AbstractFileSystem#getLinkTarget(Path)}.
* @param f the path.
* @throws UnsupportedOperationException if the operation is unsupported
* (default outcome).
* @throws IOException IO failure.
* @return the path.
*/
protected Path resolveLink(Path f) throws IOException {
// Supporting filesystems should override this method
@ -3221,7 +3356,7 @@ public void removeXAttr(Path path, String name) throws IOException {
/**
* Set the source path to satisfy storage policy.
* @param path The source path referring to either a directory or a file.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
throw new UnsupportedOperationException(
@ -3529,7 +3664,7 @@ FileSystem getUnique(URI uri, Configuration conf) throws IOException{
* @param conf configuration
* @param key key to store/retrieve this FileSystem in the cache
* @return a cached or newly instantiated FileSystem.
* @throws IOException
* @throws IOException If an I/O error occurred.
*/
private FileSystem getInternal(URI uri, Configuration conf, Key key)
throws IOException{
@ -3544,11 +3679,7 @@ private FileSystem getInternal(URI uri, Configuration conf, Key key)
// to construct an instance.
try (DurationInfo d = new DurationInfo(LOGGER, false,
"Acquiring creator semaphore for %s", uri)) {
creatorPermits.acquire();
} catch (InterruptedException e) {
// acquisition was interrupted; convert to an IOE.
throw (IOException)new InterruptedIOException(e.toString())
.initCause(e);
creatorPermits.acquireUninterruptibly();
}
FileSystem fsToClose = null;
try {
@ -4024,6 +4155,7 @@ public void run() {
/**
* Get or create the thread-local data associated with the current thread.
* @return statistics data.
*/
public StatisticsData getThreadStatistics() {
StatisticsData data = threadData.get();
@ -4382,6 +4514,7 @@ public static synchronized Map<String, Statistics> getStatistics() {
/**
* Return the FileSystem classes that have Statistics.
* @deprecated use {@link #getGlobalStorageStatistics()}
* @return statistics lists.
*/
@Deprecated
public static synchronized List<Statistics> getAllStatistics() {
@ -4390,6 +4523,7 @@ public static synchronized List<Statistics> getAllStatistics() {
/**
* Get the statistics for a particular file system.
* @param scheme scheme.
* @param cls the class to lookup
* @return a statistics object
* @deprecated use {@link #getGlobalStorageStatistics()}
@ -4424,6 +4558,7 @@ public static synchronized void clearStatistics() {
/**
* Print all statistics for all file systems to {@code System.out}
* @throws IOException If an I/O error occurred.
*/
public static synchronized
void printStatistics() throws IOException {
@ -4464,6 +4599,7 @@ public StorageStatistics getStorageStatistics() {
/**
* Get the global storage statistics.
* @return global storage statistics.
*/
public static GlobalStorageStatistics getGlobalStorageStatistics() {
return GlobalStorageStatistics.INSTANCE;

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java
View File

@ -38,8 +38,8 @@ public abstract class FileSystemLinkResolver<T> {
* an UnresolvedLinkException if called on an unresolved {@link Path}.
* @param p Path on which to perform an operation
* @return Generic type returned by operation
* @throws IOException
* @throws UnresolvedLinkException
* @throws IOException raised on errors performing I/O.
* @throws UnresolvedLinkException unresolved link exception.
*/
abstract public T doCall(final Path p) throws IOException,
UnresolvedLinkException;
@ -54,7 +54,7 @@ abstract public T doCall(final Path p) throws IOException,
* @param p
* Resolved Target of path
* @return Generic type determined by implementation
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
abstract public T next(final FileSystem fs, final Path p) throws IOException;
@ -66,7 +66,7 @@ abstract public T doCall(final Path p) throws IOException,
* @param filesys FileSystem with which to try call
* @param path Path with which to try call
* @return Generic type determined by implementation
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public T resolve(final FileSystem filesys, final Path path)
throws IOException {

87
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
View File

@ -162,6 +162,8 @@ public static void fullyDeleteOnExit(final File file) {
* (3) If dir is a normal file, it is deleted.
* (4) If dir is a normal directory, then dir and all its contents recursively
* are deleted.
* @param dir dir.
* @return fully delete status.
*/
public static boolean fullyDelete(final File dir) {
return fullyDelete(dir, false);
@ -257,6 +259,9 @@ private static boolean deleteImpl(final File f, final boolean doLog) {
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
*
* @param dir dir.
* @return fullyDeleteContents Status.
*/
public static boolean fullyDeleteContents(final File dir) {
return fullyDeleteContents(dir, false);
@ -267,8 +272,11 @@ public static boolean fullyDeleteContents(final File dir) {
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
*
* @param dir dir.
* @param tryGrantPermissions if 'true', try grant +rwx permissions to this
* and all the underlying directories before trying to delete their contents.
* @return fully delete contents status.
*/
public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
if (tryGrantPermissions) {
@ -311,7 +319,7 @@ public static boolean fullyDeleteContents(final File dir, final boolean tryGrant
*
* @param fs {@link FileSystem} on which the path is present
* @param dir directory to recursively delete
* @throws IOException
* @throws IOException raised on errors performing I/O.
* @deprecated Use {@link FileSystem#delete(Path, boolean)}
*/
@Deprecated
@ -343,7 +351,17 @@ private static void checkDependencies(FileSystem srcFS,
}
}
/** Copy files between FileSystems. */
/**
* Copy files between FileSystems.
* @param srcFS src fs.
* @param src src.
* @param dstFS dst fs.
* @param dst dst.
* @param deleteSource delete source.
* @param conf configuration.
* @return if copy success true, not false.
* @throws IOException raised on errors performing I/O.
*/
public static boolean copy(FileSystem srcFS, Path src,
FileSystem dstFS, Path dst,
boolean deleteSource,
@ -391,7 +409,19 @@ public static boolean copy(FileSystem srcFS, Path[] srcs,
return returnVal;
}
/** Copy files between FileSystems. */
/**
* Copy files between FileSystems.
*
* @param srcFS srcFs.
* @param src src.
* @param dstFS dstFs.
* @param dst dst.
* @param deleteSource delete source.
* @param overwrite overwrite.
* @param conf configuration.
* @throws IOException raised on errors performing I/O.
* @return true if the operation succeeded.
*/
public static boolean copy(FileSystem srcFS, Path src,
FileSystem dstFS, Path dst,
boolean deleteSource,
@ -403,20 +433,21 @@ public static boolean copy(FileSystem srcFS, Path src,
/**
* Copy a file/directory tree within/between filesystems.
* <p></p>
* <p>
* returns true if the operation succeeded. When deleteSource is true,
* this means "after the copy, delete(source) returned true"
* If the destination is a directory, and mkdirs (dest) fails,
* the operation will return false rather than raise any exception.
* <p></p>
* </p>
* The overwrite flag is about overwriting files; it has no effect about
* handing an attempt to copy a file atop a directory (expect an IOException),
* or a directory over a path which contains a file (mkdir will fail, so
* "false").
* <p></p>
* <p>
* The operation is recursive, and the deleteSource operation takes place
* as each subdirectory is copied. Therefore, if an operation fails partway
* through, the source tree may be partially deleted.
* </p>
* @param srcFS source filesystem
* @param srcStatus status of source
* @param dstFS destination filesystem
@ -471,7 +502,17 @@ public static boolean copy(FileSystem srcFS, FileStatus srcStatus,
}
/** Copy local files to a FileSystem. */
/**
* Copy local files to a FileSystem.
*
* @param src src.
* @param dstFS dstFs.
* @param dst dst.
* @param deleteSource delete source.
* @param conf configuration.
* @throws IOException raised on errors performing I/O.
* @return true if the operation succeeded.
*/
public static boolean copy(File src,
FileSystem dstFS, Path dst,
boolean deleteSource,
@ -514,7 +555,17 @@ public static boolean copy(File src,
}
}
/** Copy FileSystem files to local files. */
/**
* Copy FileSystem files to local files.
*
* @param srcFS srcFs.
* @param src src.
* @param dst dst.
* @param deleteSource delete source.
* @param conf configuration.
* @throws IOException raised on errors performing I/O.
* @return true if the operation succeeded.
*/
public static boolean copy(FileSystem srcFS, Path src,
File dst, boolean deleteSource,
Configuration conf) throws IOException {
@ -958,7 +1009,7 @@ public static void unTar(InputStream inputStream, File untarDir,
*
* @param inFile The tar file as input.
* @param untarDir The untar directory where to untar the tar file.
* @throws IOException
* @throws IOException an exception occurred.
*/
public static void unTar(File inFile, File untarDir) throws IOException {
if (!untarDir.mkdirs()) {
@ -1169,6 +1220,7 @@ public static class HardLink extends org.apache.hadoop.fs.HardLink {
* @param target the target for symlink
* @param linkname the symlink
* @return 0 on success
* @throws IOException raised on errors performing I/O.
*/
public static int symLink(String target, String linkname) throws IOException{
@ -1230,8 +1282,8 @@ public static int symLink(String target, String linkname) throws IOException{
* @param filename the name of the file to change
* @param perm the permission string
* @return the exit code from the command
* @throws IOException
* @throws InterruptedException
* @throws IOException raised on errors performing I/O.
* @throws InterruptedException command interrupted.
*/
public static int chmod(String filename, String perm
) throws IOException, InterruptedException {
@ -1245,7 +1297,7 @@ public static int chmod(String filename, String perm
* @param perm permission string
* @param recursive true, if permissions should be changed recursively
* @return the exit code from the command.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static int chmod(String filename, String perm, boolean recursive)
throws IOException {
@ -1271,7 +1323,7 @@ public static int chmod(String filename, String perm, boolean recursive)
* @param file the file to change
* @param username the new user owner name
* @param groupname the new group owner name
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static void setOwner(File file, String username,
String groupname) throws IOException {
@ -1288,7 +1340,7 @@ public static void setOwner(File file, String username,
* Platform independent implementation for {@link File#setReadable(boolean)}
* File#setReadable does not work as expected on Windows.
* @param f input file
* @param readable
* @param readable readable.
* @return true on success, false otherwise
*/
public static boolean setReadable(File f, boolean readable) {
@ -1309,7 +1361,7 @@ public static boolean setReadable(File f, boolean readable) {
* Platform independent implementation for {@link File#setWritable(boolean)}
* File#setWritable does not work as expected on Windows.
* @param f input file
* @param writable
* @param writable writable.
* @return true on success, false otherwise
*/
public static boolean setWritable(File f, boolean writable) {
@ -1333,7 +1385,7 @@ public static boolean setWritable(File f, boolean writable) {
* behavior on Windows as on Unix platforms. Creating, deleting or renaming
* a file within that folder will still succeed on Windows.
* @param f input file
* @param executable
* @param executable executable.
* @return true on success, false otherwise
*/
public static boolean setExecutable(File f, boolean executable) {
@ -1412,7 +1464,7 @@ public static boolean canExecute(File f) {
* of forking if group == other.
* @param f the file to change
* @param permission the new permissions
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static void setPermission(File f, FsPermission permission
) throws IOException {
@ -1717,6 +1769,7 @@ public static List<Path> getJarsInDirectory(String path) {
* wildcard path to return all jars from the directory to use in a classpath.
*
* @param path the path to the directory. The path may include the wildcard.
* @param useLocal use local.
* @return the list of jars as URLs, or an empty list if there are no jars, or
* the directory does not exist
*/

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
View File

@ -233,7 +233,7 @@ public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
*
* @param src file name
* @param replication new replication
* @throws IOException
* @throws IOException raised on errors performing I/O.
* @return true if successful;
* false if file does not exist or is a directory
*/
@ -304,7 +304,7 @@ public Path getHomeDirectory() {
* Set the current working directory for the given file system. All relative
* paths will be resolved relative to it.
*
* @param newDir
* @param newDir new dir.
*/
@Override
public void setWorkingDirectory(Path newDir) {

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
View File

@ -130,7 +130,7 @@ public Path getCurrentTrashDir() throws IOException {
* Returns the current trash location for the path specified
* @param path to be deleted
* @return path to the trash
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public Path getCurrentTrashDir(Path path) throws IOException {
return getTrash().getCurrentTrashDir(path);

23
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsStatus.java
View File

@ -35,24 +35,39 @@ public class FsStatus implements Writable {
private long used;
private long remaining;
/** Construct a FsStatus object, using the specified statistics */
/**
* Construct a FsStatus object, using the specified statistics.
*
* @param capacity capacity.
* @param used used.
* @param remaining remaining.
*/
public FsStatus(long capacity, long used, long remaining) {
this.capacity = capacity;
this.used = used;
this.remaining = remaining;
}
/** Return the capacity in bytes of the file system */
/**
* Return the capacity in bytes of the file system.
* @return capacity.
*/
public long getCapacity() {
return capacity;
}
/** Return the number of bytes used on the file system */
/**
* Return the number of bytes used on the file system.
* @return used.
*/
public long getUsed() {
return used;
}
/** Return the number of remaining bytes on the file system */
/**
* Return the number of remaining bytes on the file system.
* @return remaining.
*/
public long getRemaining() {
return remaining;
}

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobExpander.java
View File

@ -56,9 +56,9 @@ public StringWithOffset(String string, int offset) {
* {a,b}/{c/\d} - {a,b}/c/d
* </pre>
*
* @param filePattern
* @param filePattern file pattern.
* @return expanded file patterns
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public static List<String> expand(String filePattern) throws IOException {
List<String> fullyExpanded = new ArrayList<String>();

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GlobalStorageStatistics.java
View File

@ -104,6 +104,8 @@ public synchronized void reset() {
/**
* Get an iterator that we can use to iterate throw all the global storage
* statistics objects.
*
* @return StorageStatistics Iterator.
*/
synchronized public Iterator<StorageStatistics> iterator() {
Entry<String, StorageStatistics> first = map.firstEntry();

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
View File

@ -463,7 +463,7 @@ static BlockLocation[] fixBlockLocations(BlockLocation[] locations,
* @param start the start of the desired range in the contained file
* @param len the length of the desired range
* @return block locations for this segment of file
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
@ -525,7 +525,7 @@ private void fileStatusesInIndex(HarStatus parent, List<FileStatus> statuses)
* Combine the status stored in the index and the underlying status.
* @param h status stored in the index
* @return the combined file status
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
private FileStatus toFileStatus(HarStatus h) throws IOException {
final Path p = h.isDir ? archivePath : new Path(archivePath, h.partName);
@ -635,7 +635,7 @@ public long getModificationTime() {
* while creating a hadoop archive.
* @param f the path in har filesystem
* @return filestatus.
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
@Override
public FileStatus getFileStatus(Path f) throws IOException {
@ -1104,7 +1104,7 @@ public void setDropBehind(Boolean dropBehind) throws IOException {
* @param start the start position in the part file
* @param length the length of valid data in the part file
* @param bufsize the buffer size
* @throws IOException
* @throws IOException raised on errors performing I/O.
*/
public HarFSDataInputStream(FileSystem fs, Path p, long start,
long length, int bufsize) throws IOException {

Some files were not shown because too many files have changed in this diff Show More