fix print0 consequences

.asf.yaml

@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-github:
-  ghp_path: /
-  ghp_branch: gh-pages
-  enabled_merge_buttons:
-    squash: true
-    merge: false
-    rebase: false
-notifications:
-  commits:      common-commits@hadoop.apache.org
-  issues:       common-issues@hadoop.apache.org
-  pullrequests: common-issues@hadoop.apache.org
-  jira_options: comment link label

.github/pull_request_template.md

@@ -1,19 +0,0 @@
-<!--
-  Thanks for sending a pull request!
-    1. If this is your first time, please read our contributor guidelines: https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute
-    2. Make sure your PR title starts with JIRA issue id, e.g., 'HADOOP-17799. Your PR title ...'.
--->
-
-### Description of PR
-
-
-### How was this patch tested?
-
-
-### For code changes:
-
-- [ ] Does the title or this PR starts with the corresponding JIRA issue id (e.g. 'HADOOP-17799. Your PR title ...')?
-- [ ] Object storage: have the integration tests been executed and the endpoint declared according to the connector-specific documentation?
-- [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?
-- [ ] If applicable, have you updated the `LICENSE`, `LICENSE-binary`, `NOTICE-binary` files?
-

.github/workflows/website.yml

@@ -1,59 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-name: website
-
-# Controls when the action will run.
-on:
-  push:
-    branches: [ trunk ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Hadoop trunk
-        uses: actions/checkout@v3
-        with:
-          repository: apache/hadoop
-      - name: Set up JDK 8
-        uses: actions/setup-java@v3
-        with:
-          java-version: '8'
-          distribution: 'temurin'
-      - name: Cache local Maven repository
-        uses: actions/cache@v3
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-      - name: Build Hadoop maven plugins
-        run: cd hadoop-maven-plugins && mvn --batch-mode install
-      - name: Build Hadoop
-        run: mvn clean install -DskipTests -DskipShade
-      - name: Build document
-        run: mvn clean site
-      - name: Stage document
-        run: mvn site:stage -DstagingDirectory=${GITHUB_WORKSPACE}/staging/
-      - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./staging/hadoop-project
-          user_name: 'github-actions[bot]'
-          user_email: 'github-actions[bot]@users.noreply.github.com'
-

.gitignore

@@ -9,7 +9,6 @@
 *.suo
 *.vcxproj.user
 *.patch
-*.diff
 .idea
 .svn
 .classpath
@@ -51,13 +50,16 @@ patchprocess/
 .history/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
-phantomjsdriver.log
+
+# Ignore files generated by HDDS acceptance tests.
+hadoop-ozone/acceptance-test/docker-compose.log
+hadoop-ozone/acceptance-test/junit-results.xml
 
 #robotframework outputs
 log.html
 output.xml
 report.html
 
+hadoop-hdds/docs/public
 
 .mvn

.yetus/excludes.txt

@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-dev-support/docker/Dockerfile_windows_10

BUILDING.txt

@@ -6,12 +6,11 @@ Requirements:
 * Unix System
 * JDK 1.8
 * Maven 3.3 or later
-* Boost 1.72 (if compiling native code)
-* Protocol Buffers 3.7.1 (if compiling native code)
-* CMake 3.19 or newer (if compiling native code)
+* ProtocolBuffer 2.5.0
+* CMake 3.1 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
-* One of the compilers that support thread_local storage: GCC 9.3.0 or later, Visual Studio,
+* One of the compilers that support thread_local storage: GCC 4.8.1 or later, Visual Studio,
   Clang (community version), Clang (version for iOS 9 and later) (if compiling native code)
 * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
 * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
@@ -51,47 +50,25 @@ Known issues:
   and run your IDE and Docker etc inside that VM.
 
 ----------------------------------------------------------------------------------
-Installing required packages for clean install of Ubuntu 18.04 LTS Desktop.
-(For Ubuntu 20.04, gcc/g++ and cmake bundled with Ubuntu can be used.
-Refer to  dev-support/docker/Dockerfile):
+Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
 
-* Open JDK 1.8
+* Oracle JDK 1.8 (preferred)
+  $ sudo apt-get purge openjdk*
+  $ sudo apt-get install software-properties-common
+  $ sudo add-apt-repository ppa:webupd8team/java
   $ sudo apt-get update
-  $ sudo apt-get -y install openjdk-8-jdk
+  $ sudo apt-get install oracle-java8-installer
 * Maven
   $ sudo apt-get -y install maven
 * Native libraries
   $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev
-* GCC 9.3.0
-  $ sudo apt-get -y install software-properties-common
-  $ sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  $ sudo apt-get update
-  $ sudo apt-get -y install g++-9 gcc-9
-  $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9
-* CMake 3.19
-  $ curl -L https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz > cmake-3.19.0.tar.gz
-  $ tar -zxvf cmake-3.19.0.tar.gz && cd cmake-3.19.0
-  $ ./bootstrap
-  $ make -j$(nproc)
-  $ sudo make install
-* Protocol Buffers 3.7.1 (required to build native code)
-  $ curl -L -s -S https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz -o protobuf-3.7.1.tar.gz
-  $ mkdir protobuf-3.7-src
-  $ tar xzf protobuf-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7-src && cd protobuf-3.7-src
-  $ ./configure
-  $ make -j$(nproc)
-  $ sudo make install
-* Boost
-  $ curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2
-  $ tar --bzip2 -xf boost_1_72_0.tar.bz2 && cd boost_1_72_0
-  $ ./bootstrap.sh --prefix=/usr/
-  $ ./b2 --without-python
-  $ sudo ./b2 --without-python install
+* ProtocolBuffer 2.5.0 (required)
+  $ sudo apt-get -y install protobuf-compiler
 
 Optional packages:
 
-* Snappy compression (only used for hadoop-mapreduce-client-nativetask)
-  $ sudo apt-get install libsnappy-dev
+* Snappy compression
+  $ sudo apt-get install snappy libsnappy-dev
 * Intel ISA-L library for erasure coding
   Please refer to https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
   (OR https://github.com/01org/isa-l)
@@ -100,7 +77,7 @@ Optional packages:
 * Linux FUSE
   $ sudo apt-get install fuse libfuse-dev
 * ZStandard compression
-  $ sudo apt-get install libzstd1-dev
+    $ sudo apt-get install zstd
 * PMDK library for storage class memory(SCM) as HDFS cache backend
   Please refer to http://pmem.io/ and https://github.com/pmem/pmdk
 
@@ -111,7 +88,7 @@ Maven main modules:
          - hadoop-project               (Parent POM for all Hadoop Maven modules.             )
                                         (All plugins & dependencies versions are defined here.)
          - hadoop-project-dist          (Parent POM for modules that generate distributions.)
-         - hadoop-annotations           (Generates the Hadoop doclet used to generate the Javadocs)
+         - hadoop-annotations           (Generates the Hadoop doclet used to generated the Javadocs)
          - hadoop-assemblies            (Maven assemblies used by the different modules)
          - hadoop-maven-plugins         (Maven plugins used in project)
          - hadoop-build-tools           (Build tools like checkstyle, etc.)
@@ -119,6 +96,8 @@ Maven main modules:
          - hadoop-hdfs-project          (Hadoop HDFS)
          - hadoop-yarn-project          (Hadoop YARN)
          - hadoop-mapreduce-project     (Hadoop MapReduce)
+         - hadoop-ozone                 (Hadoop Ozone)
+         - hadoop-hdds                  (Hadoop Distributed Data Store)
          - hadoop-tools                 (Hadoop tools like Streaming, Distcp, etc.)
          - hadoop-dist                  (Hadoop distribution assembler)
          - hadoop-client-modules        (Hadoop client modules)
@@ -128,7 +107,7 @@ Maven main modules:
 ----------------------------------------------------------------------------------
 Where to run Maven from?
 
-  It can be run from any module. The only catch is that if not run from trunk
+  It can be run from any module. The only catch is that if not run from utrunk
   all modules that are not part of the build run must be installed in the local
   Maven cache or available in a Maven repository.
 
@@ -139,11 +118,11 @@ Maven build goals:
  * Compile                   : mvn compile [-Pnative]
  * Run tests                 : mvn test [-Pnative] [-Pshelltest]
  * Create JAR                : mvn package
- * Run spotbugs              : mvn compile spotbugs:spotbugs
+ * Run findbugs              : mvn compile findbugs:findbugs
  * Run checkstyle            : mvn compile checkstyle:checkstyle
  * Install JAR in M2 cache   : mvn install
  * Deploy JAR to Maven repo  : mvn deploy
- * Run clover                : mvn test -Pclover
+ * Run clover                : mvn test -Pclover [-DcloverLicenseLocation=${user.name}/.clover.license]
  * Run Rat                   : mvn apache-rat:check
  * Build javadocs            : mvn javadoc:javadoc
  * Build distribution        : mvn package [-Pdist][-Pdocs][-Psrc][-Pnative][-Dtar][-Preleasedocs][-Pyarn-ui]
@@ -176,14 +155,14 @@ Maven build goals:
 
    Snappy is a compression library that can be utilized by the native code.
    It is currently an optional component, meaning that Hadoop can be built with
-   or without this dependency. Snappy library as optional dependency is only
-   used for hadoop-mapreduce-client-nativetask.
+   or without this dependency.
 
   * Use -Drequire.snappy to fail the build if libsnappy.so is not found.
     If this option is not specified and the snappy library is missing,
     we silently build a version of libhadoop.so that cannot make use of snappy.
     This option is recommended if you plan on making use of snappy and want
     to get more repeatable builds.
+
   * Use -Dsnappy.prefix to specify a nonstandard location for the libsnappy
     header files and library files. You do not need this option if you have
     installed snappy using a package manager.
@@ -323,38 +302,53 @@ level once; and then work from the submodule. Keep in mind that SNAPSHOTs
 time out after a while, using the Maven '-nsu' will stop Maven from trying
 to update SNAPSHOTs from external repos.
 
+----------------------------------------------------------------------------------
+Protocol Buffer compiler
+
+The version of Protocol Buffer compiler, protoc, must match the version of the
+protobuf JAR.
+
+If you have multiple versions of protoc in your system, you can set in your
+build shell the HADOOP_PROTOC_PATH environment variable to point to the one you
+want to use for the Hadoop build. If you don't define this environment variable,
+protoc is looked up in the PATH.
 ----------------------------------------------------------------------------------
 Importing projects to eclipse
 
-At first, install artifacts including hadoop-maven-plugins at the top of the source tree.
+When you import the project to eclipse, install hadoop-maven-plugins at first.
 
-  $ mvn clean install -DskipTests -DskipShade
+  $ cd hadoop-maven-plugins
+  $ mvn install
 
-Then, import to eclipse by specifying the root directory of the project via
-[File] > [Import] > [Maven] > [Existing Maven Projects].
+Then, generate eclipse project files.
+
+  $ mvn eclipse:eclipse -DskipTests
+
+At last, import to eclipse by specifying the root directory of the project via
+[File] > [Import] > [Existing Projects into Workspace].
 
 ----------------------------------------------------------------------------------
 Building distributions:
 
-Create binary distribution without native code and without Javadocs:
+Create binary distribution without native code and without documentation:
 
   $ mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true
 
-Create binary distribution with native code:
+Create binary distribution with native code and with documentation:
 
-  $ mvn package -Pdist,native -DskipTests -Dtar
+  $ mvn package -Pdist,native,docs -DskipTests -Dtar
 
 Create source distribution:
 
   $ mvn package -Psrc -DskipTests
 
-Create source and binary distributions with native code:
+Create source and binary distributions with native code and documentation:
 
-  $ mvn package -Pdist,native,src -DskipTests -Dtar
+  $ mvn package -Pdist,native,docs,src -DskipTests -Dtar
 
 Create a local staging version of the website (in /tmp/hadoop-site)
 
-  $ mvn site site:stage -Preleasedocs,docs -DstagingDirectory=/tmp/hadoop-site
+  $ mvn clean site -Preleasedocs; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
 
 Note that the site needs to be built in a second pass after other artifacts.
 
@@ -403,10 +397,11 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.7.1 (required to compile native code)
-  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
-  $ mkdir -p protobuf-3.7 && tar zxvf protobuf-java-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7
-  $ cd protobuf-3.7
+* Protocol Buffers 2.5.0 (required), since 2.5.0 is no longer in Homebrew,
+we need to compile it from source
+  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
+  $ tar zxvf protobuf-2.5.0.tar.gz
+  $ cd protobuf-2.5.0
   $ ./configure
   $ make
   $ make check
@@ -427,131 +422,40 @@ Building command example:
 Note that the command above manually specified the openssl library and include
 path. This is necessary at least for Homebrewed OpenSSL.
 
-
 ----------------------------------------------------------------------------------
 
-Building on CentOS 8
-
-----------------------------------------------------------------------------------
-
-
-* Install development tools such as GCC, autotools, OpenJDK and Maven.
-  $ sudo dnf group install --with-optional 'Development Tools'
-  $ sudo dnf install java-1.8.0-openjdk-devel maven
-
-* Install python2 for building documentation.
-  $ sudo dnf install python2
-
-* Install Protocol Buffers v3.7.1.
-  $ git clone https://github.com/protocolbuffers/protobuf
-  $ cd protobuf
-  $ git checkout v3.7.1
-  $ autoreconf -i
-  $ ./configure --prefix=/usr/local
-  $ make
-  $ sudo make install
-  $ cd ..
-
-* Install libraries provided by CentOS 8.
-  $ sudo dnf install libtirpc-devel zlib-devel lz4-devel bzip2-devel openssl-devel cyrus-sasl-devel libpmem-devel
-
-* Install GCC 9.3.0
-  $ sudo dnf -y install gcc-toolset-9-gcc gcc-toolset-9-gcc-c++
-  $ source /opt/rh/gcc-toolset-9/enable
-
-* Install CMake 3.19
-  $ curl -L https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz > cmake-3.19.0.tar.gz
-  $ tar -zxvf cmake-3.19.0.tar.gz && cd cmake-3.19.0
-  $ ./bootstrap
-  $ make -j$(nproc)
-  $ sudo make install
-
-* Install boost.
-  $ curl -L -o boost_1_72_0.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download
-  $ tar xjf boost_1_72_0.tar.bz2
-  $ cd boost_1_72_0
-  $ ./bootstrap.sh --prefix=/usr/local
-  $ ./b2
-  $ sudo ./b2 install
-
-* Install optional dependencies (snappy-devel).
-  $ sudo dnf --enablerepo=PowerTools install snappy-devel
-
-* Install optional dependencies (libzstd-devel).
-  $ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-  $ sudo dnf --enablerepo=epel install libzstd-devel
-
-* Install optional dependencies (isa-l).
-  $ sudo dnf --enablerepo=PowerTools install nasm
-  $ git clone https://github.com/intel/isa-l
-  $ cd isa-l/
-  $ ./autogen.sh
-  $ ./configure
-  $ make
-  $ sudo make install
-
-----------------------------------------------------------------------------------
-
-Building on Windows 10
+Building on Windows
 
 ----------------------------------------------------------------------------------
 Requirements:
 
-* Windows 10
+* Windows System
 * JDK 1.8
-* Maven 3.0 or later (maven.apache.org)
-* Boost 1.72 (boost.org)
-* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
-* CMake 3.19 or newer (cmake.org)
-* Visual Studio 2019 (visualstudio.com)
-* Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from
-                   http://msdn.microsoft.com/en-us/windows/bg162891.aspx)
-* Zlib (zlib.net, if building native code bindings for zlib)
-* Git (preferably, get this from https://git-scm.com/download/win since the package also contains
-       Unix command-line tools that are needed during packaging).
-* Python (python.org, for generation of docs using 'mvn site')
+* Maven 3.0 or later
+* ProtocolBuffer 2.5.0
+* CMake 3.1 or newer
+* Visual Studio 2010 Professional or Higher
+* Windows SDK 8.1 (if building CPU rate control for the container executor)
+* zlib headers (if building native code bindings for zlib)
 * Internet connection for first build (to fetch all Maven and Hadoop dependencies)
+* Unix command-line tools from GnuWin32: sh, mkdir, rm, cp, tar, gzip. These
+  tools must be present on your PATH.
+* Python ( for generation of docs using 'mvn site')
+
+Unix command-line tools are also included with the Windows Git package which
+can be downloaded from http://git-scm.com/downloads
+
+If using Visual Studio, it must be Professional level or higher.
+Do not use Visual Studio Express.  It does not support compiling for 64-bit,
+which is problematic if running a 64-bit system.
+
+The Windows SDK 8.1 is available to download at:
+
+http://msdn.microsoft.com/en-us/windows/bg162891.aspx
+
+Cygwin is not required.
 
 ----------------------------------------------------------------------------------
-
-Building guidelines:
-
-Hadoop repository provides the Dockerfile for building Hadoop on Windows 10, located at
-dev-support/docker/Dockerfile_windows_10. It is highly recommended to use this and create the
-Docker image for building Hadoop on Windows 10, since you don't have to install anything else
-other than Docker and no additional steps are required in terms of aligning the environment with
-the necessary paths etc.
-
-However, if you still prefer taking the route of not using Docker, this Dockerfile_windows_10 will
-still be immensely useful as a raw guide for all the steps involved in creating the environment
-needed to build Hadoop on Windows 10.
-
-Building using the Docker:
-We first need to build the Docker image for building Hadoop on Windows 10. Run this command from
-the root of the Hadoop repository.
-> docker build -t hadoop-windows-10-builder -f .\dev-support\docker\Dockerfile_windows_10 .\dev-support\docker\
-
-Start the container with the image that we just built.
-> docker run --rm -it hadoop-windows-10-builder
-
-You can now clone the Hadoop repo inside this container and proceed with the build.
-
-NOTE:
-While one may perceive the idea of mounting the locally cloned (on the host filesystem) Hadoop
-repository into the container (using the -v option), we have seen the build to fail owing to some
-files not being able to be located by Maven. Thus, we suggest cloning the Hadoop repository to a
-non-mounted folder inside the container and proceed with the build. When the build is completed,
-you may use the "docker cp" command to copy the built Hadoop tar.gz file from the docker container
-to the host filesystem. If you still would like to mount the Hadoop codebase, a workaround would
-be to copy the mounted Hadoop codebase into another folder (which doesn't point to a mount) in the
-container's filesystem and use this for building.
-
-However, we noticed no build issues when the Maven repository from the host filesystem was mounted
-into the container. One may use this to greatly reduce the build time. Assuming that the Maven
-repository is located at D:\Maven\Repository in the host filesystem, one can use the following
-command to mount the same onto the default Maven repository location while launching the container.
-> docker run --rm -v D:\Maven\Repository:C:\Users\ContainerAdministrator\.m2\repository -it hadoop-windows-10-builder
-
 Building:
 
 Keep the source code tree in a short path to avoid running into problems related
@@ -567,24 +471,6 @@ configure the bit-ness of the build, and set several optional components.
 Several tests require that the user must have the Create Symbolic Links
 privilege.
 
-To simplify the installation of Boost, Protocol buffers, OpenSSL and Zlib dependencies we can use
-vcpkg (https://github.com/Microsoft/vcpkg.git). Upon cloning the vcpkg repo, checkout the commit
-7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d to get the required versions of the dependencies
-mentioned above.
-> git clone https://github.com/Microsoft/vcpkg.git
-> cd vcpkg
-> git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d
-> .\bootstrap-vcpkg.bat
-> .\vcpkg.exe install boost:x64-windows
-> .\vcpkg.exe install protobuf:x64-windows
-> .\vcpkg.exe install openssl:x64-windows
-> .\vcpkg.exe install zlib:x64-windows
-
-Set the following environment variables -
-(Assuming that vcpkg was checked out at C:\vcpkg)
-> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
-> set MAVEN_OPTS=-Xmx2048M -Xss128M
-
 All Maven goals are the same as described above with the exception that
 native code is built by enabling the 'native-win' Maven profile. -Pnative-win
 is enabled by default when building on Windows since the native components
@@ -602,24 +488,6 @@ the zlib 1.2.7 source tree.
 
 http://www.zlib.net/
 
-
-Build command:
-The following command builds all the modules in the Hadoop project and generates the tar.gz file in
-hadoop-dist/target upon successful build. Run these commands from an
-"x64 Native Tools Command Prompt for VS 2019" which can be found under "Visual Studio 2019" in the
-Windows start menu. If you're using the Docker image from Dockerfile_windows_10, you'll be
-logged into "x64 Native Tools Command Prompt for VS 2019" automatically when you start the
-container.
-
-> set classpath=
-> set PROTOBUF_HOME=C:\vcpkg\installed\x64-windows
-> mvn clean package -Dhttps.protocols=TLSv1.2 -DskipTests -DskipDocs -Pnative-win,dist^
-    -Drequire.openssl -Drequire.test.libhadoop -Pyarn-ui -Dshell-executable=C:\Git\bin\bash.exe^
-    -Dtar -Dopenssl.prefix=C:\vcpkg\installed\x64-windows^
-    -Dcmake.prefix.path=C:\vcpkg\installed\x64-windows^
-    -Dwindows.cmake.toolchain.file=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -Dwindows.cmake.build.type=RelWithDebInfo^
-    -Dwindows.build.hdfspp.dll=off -Dwindows.no.sasl=on -Duse.platformToolsetVersion=v142
-
 ----------------------------------------------------------------------------------
 Building distributions:
 

Jenkinsfile

@@ -0,0 +1,209 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+    agent {
+        label 'Hadoop'
+    }
+
+    options {
+        buildDiscarder(logRotator(numToKeepStr: '5'))
+        timeout (time: 5, unit: 'HOURS')
+        timestamps()
+        checkoutToSubdirectory('src')
+    }
+
+    environment {
+        SOURCEDIR = 'src'
+        // will also need to change notification section below
+        PATCHDIR = 'out'
+        DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
+        YETUS='yetus'
+        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
+        YETUS_VERSION='rel/0.10.0'
+    }
+
+    parameters {
+        string(name: 'JIRA_ISSUE_KEY',
+               defaultValue: '',
+               description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
+    }
+
+    stages {
+        stage ('install yetus') {
+            steps {
+                dir("${WORKSPACE}/${YETUS}") {
+                    checkout([
+                        $class: 'GitSCM',
+                        branches: [[name: "${env.YETUS_VERSION}"]],
+                        userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
+                    )
+                }
+            }
+        }
+
+        stage ('precommit-run') {
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_PASSWORD',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        set -e
+
+                        TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
+
+                        # this must be clean for every run
+                        if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
+                          rm -rf "${WORKSPACE}/${PATCHDIR}"
+                        fi
+                        mkdir -p "${WORKSPACE}/${PATCHDIR}"
+
+                        # if given a JIRA issue, process it. If CHANGE_URL is set
+                        # (e.g., Github Branch Source plugin), process it.
+                        # otherwise exit, because we don't want Hadoop to do a
+                        # full build.  We wouldn't normally do this check for smaller
+                        # projects. :)
+                        if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
+                            YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
+                        elif [[ -z "${CHANGE_URL}" ]]; then
+                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
+                            exit 0
+                        fi
+
+                        YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+
+                        # where the source is located
+                        YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+
+                        # our project defaults come from a personality file
+                        # which will get loaded automatically by setting the project name
+                        YETUS_ARGS+=("--project=hadoop")
+
+                        # lots of different output formats
+                        YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
+                        YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
+                        YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+
+                        # enable writing back to Github
+                        YETUS_ARGS+=(--github-password="${GITHUB_PASSWORD}")
+                        YETUS_ARGS+=(--github-user=${GITHUB_USER})
+
+                        # enable writing back to ASF JIRA
+                        YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
+                        YETUS_ARGS+=(--jira-user="${JIRA_USER}")
+
+                        # auto-kill any surefire stragglers during unit test runs
+                        YETUS_ARGS+=("--reapermode=kill")
+
+                        # set relatively high limits for ASF machines
+                        # changing these to higher values may cause problems
+                        # with other jobs on systemd-enabled machines
+                        YETUS_ARGS+=("--proclimit=5500")
+                        YETUS_ARGS+=("--dockermemlimit=20g")
+
+                        # -1 findbugs issues that show up prior to the patch being applied
+                        YETUS_ARGS+=("--findbugs-strict-precheck")
+
+                        # rsync these files back into the archive dir
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+                        # URL for user-side presentation in reports and such to our artifacts
+                        # (needs to match the archive bits below)
+                        YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+                        # plugins to enable
+                        YETUS_ARGS+=("--plugins=all")
+
+                        # use Hadoop's bundled shelldocs
+                        YETUS_ARGS+=("--shelldocs=/testptch/hadoop/dev-support/bin/shelldocs")
+
+                        # don't let these tests cause -1s because we aren't really paying that
+                        # much attention to them
+                        YETUS_ARGS+=("--tests-filter=checkstyle")
+
+                        # run in docker mode and specifically point to our
+                        # Dockerfile since we don't want to use the auto-pulled version.
+                        YETUS_ARGS+=("--docker")
+                        YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+
+                        # effectively treat dev-suport as a custom maven module
+                        YETUS_ARGS+=("--skip-dir=dev-support")
+
+                        # help keep the ASF boxes clean
+                        YETUS_ARGS+=("--sentinel")
+
+                        "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+                        '''
+                }
+            }
+        }
+
+    }
+
+    post {
+        always {
+          script {
+            // Yetus output
+            archiveArtifacts "${env.PATCHDIR}/**"
+            // Publish the HTML report so that it can be looked at
+            // Has to be relative to WORKSPACE.
+            publishHTML (target: [
+                          allowMissing: true,
+                          keepAll: true,
+                          alwaysLinkToLastBuild: true,
+                          // Has to be relative to WORKSPACE
+                          reportDir: "${env.PATCHDIR}",
+                          reportFiles: 'report.html',
+                          reportName: 'Yetus Report'
+            ])
+            // Publish JUnit results
+            try {
+                junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
+            } catch(e) {
+                echo 'junit processing: ' + e.toString()
+            }
+          }
+        }
+
+        // Jenkins pipeline jobs fill slaves on PRs without this :(
+        cleanup() {
+            script {
+                sh '''
+                    # See YETUS-764
+                    if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
+                      echo "test-patch process appears to still be running: killing"
+                      kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
+                      sleep 10
+                    fi
+                    if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
+                      echo "test-patch container appears to still be running: killing"
+                      docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
+                    fi
+                    # See HADOOP-13951
+                    chmod -R u+rxw "${WORKSPACE}"
+                    '''
+                deleteDir()
+            }
+        }
+    }
+}

LICENSE-binary

@@ -1,540 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
---------------------------------------------------------------------------------
-This project bundles some components that are also licensed under the Apache
-License Version 2.0:
-
-
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js files)
-hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
-hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
-
-com.aliyun:aliyun-java-sdk-core:4.5.10
-com.aliyun:aliyun-java-sdk-kms:2.11.0
-com.aliyun:aliyun-java-sdk-ram:3.1.0
-com.aliyun:aliyun-java-sdk-sts:3.0.0
-com.aliyun.oss:aliyun-sdk-oss:3.13.2
-com.amazonaws:aws-java-sdk-bundle:1.12.316
-com.cedarsoftware:java-util:1.9.0
-com.cedarsoftware:json-io:2.5.1
-com.fasterxml.jackson.core:jackson-annotations:2.12.7
-com.fasterxml.jackson.core:jackson-core:2.12.7
-com.fasterxml.jackson.core:jackson-databind:2.12.7.1
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.12.7
-com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.12.7
-com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.12.7
-com.fasterxml.uuid:java-uuid-generator:3.1.4
-com.fasterxml.woodstox:woodstox-core:5.4.0
-com.github.davidmoten:rxjava-extras:0.8.0.17
-com.github.stephenc.jcip:jcip-annotations:1.0-1
-com.google:guice:4.0
-com.google:guice-servlet:4.0
-com.google.api.grpc:proto-google-common-protos:1.0.0
-com.google.code.gson:2.9.0
-com.google.errorprone:error_prone_annotations:2.2.0
-com.google.j2objc:j2objc-annotations:1.1
-com.google.json-simple:json-simple:1.1.1
-com.google.guava:failureaccess:1.0
-com.google.guava:guava:20.0
-com.google.guava:guava:27.0-jre
-com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
-com.microsoft.azure:azure-storage:7.0.0
-com.nimbusds:nimbus-jose-jwt:9.31
-com.squareup.okhttp3:okhttp:4.10.0
-com.squareup.okio:okio:3.2.0
-com.zaxxer:HikariCP:4.0.3
-commons-beanutils:commons-beanutils:1.9.4
-commons-cli:commons-cli:1.5.0
-commons-codec:commons-codec:1.11
-commons-collections:commons-collections:3.2.2
-commons-daemon:commons-daemon:1.0.13
-commons-io:commons-io:2.8.0
-commons-net:commons-net:3.9.0
-de.ruedigermoeller:fst:2.50
-io.grpc:grpc-api:1.26.0
-io.grpc:grpc-context:1.26.0
-io.grpc:grpc-core:1.26.0
-io.grpc:grpc-netty:1.26.0
-io.grpc:grpc-protobuf:1.26.0
-io.grpc:grpc-protobuf-lite:1.26.0
-io.grpc:grpc-stub:1.26.0
-io.netty:netty-all:4.1.77.Final
-io.netty:netty-buffer:4.1.77.Final
-io.netty:netty-codec:4.1.77.Final
-io.netty:netty-codec-dns:4.1.77.Final
-io.netty:netty-codec-haproxy:4.1.77.Final
-io.netty:netty-codec-http:4.1.77.Final
-io.netty:netty-codec-http2:4.1.77.Final
-io.netty:netty-codec-memcache:4.1.77.Final
-io.netty:netty-codec-mqtt:4.1.77.Final
-io.netty:netty-codec-redis:4.1.77.Final
-io.netty:netty-codec-smtp:4.1.77.Final
-io.netty:netty-codec-socks:4.1.77.Final
-io.netty:netty-codec-stomp:4.1.77.Final
-io.netty:netty-codec-xml:4.1.77.Final
-io.netty:netty-common:4.1.77.Final
-io.netty:netty-handler:4.1.77.Final
-io.netty:netty-handler-proxy:4.1.77.Final
-io.netty:netty-resolver:4.1.77.Final
-io.netty:netty-resolver-dns:4.1.77.Final
-io.netty:netty-transport:4.1.77.Final
-io.netty:netty-transport-rxtx:4.1.77.Final
-io.netty:netty-transport-sctp:4.1.77.Final
-io.netty:netty-transport-udt:4.1.77.Final
-io.netty:netty-transport-classes-epoll:4.1.77.Final
-io.netty:netty-transport-native-unix-common:4.1.77.Final
-io.netty:netty-transport-classes-kqueue:4.1.77.Final
-io.netty:netty-resolver-dns-classes-macos:4.1.77.Final
-io.netty:netty-transport-native-epoll:4.1.77.Final
-io.netty:netty-transport-native-kqueue:4.1.77.Final
-io.netty:netty-resolver-dns-native-macos:4.1.77.Final
-io.opencensus:opencensus-api:0.12.3
-io.opencensus:opencensus-contrib-grpc-metrics:0.12.3
-io.reactivex:rxjava:1.3.8
-io.reactivex:rxjava-string:1.1.1
-io.reactivex:rxnetty:0.4.20
-io.swagger:swagger-annotations:1.5.4
-javax.inject:javax.inject:1
-log4j:log4j:1.2.17
-net.java.dev.jna:jna:5.2.0
-net.minidev:accessors-smart:1.2
-org.apache.avro:avro:1.9.2
-org.apache.commons:commons-collections4:4.2
-org.apache.commons:commons-compress:1.21
-org.apache.commons:commons-configuration2:2.8.0
-org.apache.commons:commons-csv:1.9.0
-org.apache.commons:commons-digester:1.8.1
-org.apache.commons:commons-lang3:3.12.0
-org.apache.commons:commons-math3:3.6.1
-org.apache.commons:commons-text:1.10.0
-org.apache.commons:commons-validator:1.6
-org.apache.curator:curator-client:5.2.0
-org.apache.curator:curator-framework:5.2.0
-org.apache.curator:curator-recipes:5.2.0
-org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1
-org.apache.hbase:hbase-annotations:1.7.1
-org.apache.hbase:hbase-client:1.7.1
-org.apache.hbase:hbase-common:1.7.1
-org.apache.hbase:hbase-protocol:1.7.1
-org.apache.htrace:htrace-core:3.1.0-incubating
-org.apache.htrace:htrace-core4:4.1.0-incubating
-org.apache.httpcomponents:httpclient:4.5.6
-org.apache.httpcomponents:httpcore:4.4.10
-org.apache.kafka:kafka-clients:2.8.2
-org.apache.kerby:kerb-admin:2.0.3
-org.apache.kerby:kerb-client:2.0.3
-org.apache.kerby:kerb-common:2.0.3
-org.apache.kerby:kerb-core:2.0.3
-org.apache.kerby:kerb-crypto:2.0.3
-org.apache.kerby:kerb-identity:2.0.3
-org.apache.kerby:kerb-server:2.0.3
-org.apache.kerby:kerb-simplekdc:2.0.3
-org.apache.kerby:kerb-util:2.0.3
-org.apache.kerby:kerby-asn1:2.0.3
-org.apache.kerby:kerby-config:2.0.3
-org.apache.kerby:kerby-pkix:2.0.3
-org.apache.kerby:kerby-util:2.0.3
-org.apache.kerby:kerby-xdr:2.0.3
-org.apache.kerby:token-provider:2.0.3
-org.apache.solr:solr-solrj:8.8.2
-org.apache.yetus:audience-annotations:0.5.0
-org.apache.zookeeper:zookeeper:3.6.3
-org.codehaus.jettison:jettison:1.5.4
-org.eclipse.jetty:jetty-annotations:9.4.51.v20230217
-org.eclipse.jetty:jetty-http:9.4.51.v20230217
-org.eclipse.jetty:jetty-io:9.4.51.v20230217
-org.eclipse.jetty:jetty-jndi:9.4.51.v20230217
-org.eclipse.jetty:jetty-plus:9.4.51.v20230217
-org.eclipse.jetty:jetty-security:9.4.51.v20230217
-org.eclipse.jetty:jetty-server:9.4.51.v20230217
-org.eclipse.jetty:jetty-servlet:9.4.51.v20230217
-org.eclipse.jetty:jetty-util:9.4.51.v20230217
-org.eclipse.jetty:jetty-util-ajax:9.4.51.v20230217
-org.eclipse.jetty:jetty-webapp:9.4.51.v20230217
-org.eclipse.jetty:jetty-xml:9.4.51.v20230217
-org.eclipse.jetty.websocket:javax-websocket-client-impl:9.4.51.v20230217
-org.eclipse.jetty.websocket:javax-websocket-server-impl:9.4.51.v20230217
-org.ehcache:ehcache:3.3.1
-org.ini4j:ini4j:0.5.4
-org.jetbrains.kotlin:kotlin-stdlib:1.4.10
-org.jetbrains.kotlin:kotlin-stdlib-common:1.4.10
-org.lz4:lz4-java:1.7.1
-org.objenesis:objenesis:2.6
-org.xerial.snappy:snappy-java:1.0.5
-org.yaml:snakeyaml:2.0
-org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-
-
---------------------------------------------------------------------------------
-This product bundles various third-party components under other open source
-licenses. This section summarizes those components and their licenses.
-See licenses-binary/ for text of these licenses.
-
-
-BSD 2-Clause
-------------
-
-hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}
-hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h
-
-com.github.luben:zstd-jni:1.4.9-1
-dnsjava:dnsjava:2.1.7
-org.codehaus.woodstox:stax2-api:4.2.1
-
-
-BSD 3-Clause
-------------
-
-hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/bloom/*
-hadoop-common-project/hadoop-common/src/main/native/gtest/gtest-all.cc
-hadoop-common-project/hadoop-common/src/main/native/gtest/include/gtest/gtest.h
-hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32_x86.c
-hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
-leveldb v1.13
-
-com.google.protobuf:protobuf-java:2.5.0
-com.google.protobuf:protobuf-java:3.6.1
-com.google.re2j:re2j:1.1
-com.jcraft:jsch:0.1.54
-com.thoughtworks.paranamer:paranamer:2.3
-jakarta.activation:jakarta.activation-api:1.2.1
-org.fusesource.leveldbjni:leveldbjni-all:1.8
-org.jline:jline:3.9.0
-org.hamcrest:hamcrest-core:1.3
-org.ow2.asm:asm:5.0.4
-org.ow2.asm:asm-commons:6.0
-org.ow2.asm:asm-tree:6.0
-
-
-MIT License
------------
-
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.4.1
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
-hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js
-hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js
-hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css
-hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.11.5/*
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js
-hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/TERMINAL
-
-bootstrap v3.3.6
-broccoli-asset-rev v2.4.2
-broccoli-funnel v1.0.1
-datatables v1.11.5
-em-helpers v0.5.13
-em-table v0.1.6
-ember v2.2.0
-ember-array-contains-helper v1.0.2
-ember-bootstrap v0.5.1
-ember-cli v1.13.13
-ember-cli-app-version v1.0.0
-ember-cli-babel v5.1.6
-ember-cli-content-security-policy v0.4.0
-ember-cli-dependency-checker v1.2.0
-ember-cli-htmlbars v1.0.2
-ember-cli-htmlbars-inline-precompile v0.3.1
-ember-cli-ic-ajax v0.2.1
-ember-cli-inject-live-reload v1.4.0
-ember-cli-jquery-ui v0.0.20
-ember-cli-qunit v1.2.1
-ember-cli-release v0.2.8
-ember-cli-shims v0.0.6
-ember-cli-sri v1.2.1
-ember-cli-test-loader v0.2.1
-ember-cli-uglify v1.2.0
-ember-d3 v0.1.0
-ember-data v2.1.0
-ember-disable-proxy-controllers v1.0.1
-ember-export-application-global v1.0.5
-ember-load-initializers v0.1.7
-ember-qunit v0.4.16
-ember-qunit-notifications v0.1.0
-ember-resolver v2.0.3
-ember-spin-spinner v0.2.3
-ember-truth-helpers v1.2.0
-jquery v2.1.4
-jquery-ui v1.11.4
-loader.js v3.3.0
-momentjs v2.10.6
-qunit v1.19.0
-select2 v4.0.0
-snippet-ss v1.11.0
-spin.js v2.3.2
-
-com.microsoft.azure:azure-cosmosdb:2.4.5
-com.microsoft.azure:azure-cosmosdb-commons:2.4.5
-com.microsoft.azure:azure-cosmosdb-direct:2.4.5
-com.microsoft.azure:azure-cosmosdb-gateway:2.4.5
-com.microsoft.azure:azure-data-lake-store-sdk:2.3.3
-com.microsoft.azure:azure-keyvault-core:1.0.0
-com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre7
-org.bouncycastle:bcpkix-jdk15on:1.68
-org.bouncycastle:bcprov-jdk15on:1.68
-org.checkerframework:checker-qual:2.5.2
-org.codehaus.mojo:animal-sniffer-annotations:1.17
-org.jruby.jcodings:jcodings:1.0.13
-org.jruby.joni:joni:2.1.2
-org.slf4j:jul-to-slf4j:jar:1.7.25
-org.ojalgo:ojalgo:43.0:compile
-org.slf4j:jul-to-slf4j:1.7.25
-org.slf4j:slf4j-api:1.7.25
-org.slf4j:slf4j-log4j12:1.7.25
-
-
-CDDL 1.1 + GPLv2 with classpath exception
------------------------------------------
-
-com.github.pjfanning:jersey-json:1.20
-com.sun.jersey:jersey-client:1.19.4
-com.sun.jersey:jersey-core:1.19.4
-com.sun.jersey:jersey-guice:1.19.4
-com.sun.jersey:jersey-server:1.19.4
-com.sun.jersey:jersey-servlet:1.19.4
-com.sun.xml.bind:jaxb-impl:2.2.3-1
-javax.annotation:javax.annotation-api:1.3.2
-javax.servlet:javax.servlet-api:3.1.0
-javax.servlet.jsp:jsp-api:2.1
-javax.websocket:javax.websocket-api:1.0
-javax.ws.rs:jsr311-api:1.1.1
-javax.xml.bind:jaxb-api:2.2.11
-
-
-Eclipse Public License 1.0
---------------------------
-
-junit:junit:4.13.2
-org.jacoco:org.jacoco.agent:0.8.5
-
-
-
-HSQL License
-------------
-
-org.hsqldb:hsqldb:2.7.1
-
-
-JDOM License
-------------
-
-org.jdom:jdom2:2.0.6.1
-
-
-Public Domain
--------------
-
-aopalliance:aopalliance:1.0

NOTICE-binary

@@ -1,840 +0,0 @@
-Apache Hadoop
-Copyright 2006 and onwards The Apache Software Foundation.
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-Export Control Notice
----------------------
-
-This distribution includes cryptographic software.  The country in
-which you currently reside may have restrictions on the import,
-possession, use, and/or re-export to another country, of
-encryption software.  BEFORE using any encryption software, please
-check your country's laws, regulations and policies concerning the
-import, possession, or use, and re-export of encryption software, to
-see if this is permitted.  See <http://www.wassenaar.org/> for more
-information.
-
-The U.S. Government Department of Commerce, Bureau of Industry and
-Security (BIS), has classified this software as Export Commodity
-Control Number (ECCN) 5D002.C.1, which includes information security
-software using or performing cryptographic functions with asymmetric
-algorithms.  The form and manner of this Apache Software Foundation
-distribution makes it eligible for export under the License Exception
-ENC Technology Software Unrestricted (TSU) exception (see the BIS
-Export Administration Regulations, Section 740.13) for both object
-code and source code.
-
-The following provides more details on the included cryptographic software:
-
-This software uses the SSL libraries from the Jetty project written
-by mortbay.org.
-Hadoop Yarn Server Web Proxy uses the BouncyCastle Java
-cryptography APIs written by the Legion of the Bouncy Castle Inc.
-
-// ------------------------------------------------------------------
-// NOTICE file corresponding to the section 4d of The Apache License,
-// Version 2.0, in this case for
-// ------------------------------------------------------------------
-
-
-Apache Yetus
-Copyright 2008-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
----
-Additional licenses for the Apache Yetus Source/Website:
----
-
-
-See LICENSE for terms.
-
-
-
-Apache Avro
-Copyright 2010 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-C JSON parsing provided by Jansson and
-written by Petri Lehtinen. The original software is
-available from http://www.digip.org/jansson/.
-
-
-AWS SDK for Java
-Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-
-This product includes software developed by
-Amazon Technologies, Inc (http://www.amazon.com/).
-
-**********************
-THIRD PARTY COMPONENTS
-**********************
-This software includes third party software subject to the following copyrights:
-- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
-- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
-
-The licenses for these third party components are included in LICENSE.txt
-
-
-Apache Commons BeanUtils
-Copyright 2000-2016 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons CLI
-Copyright 2001-2009 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Codec
-Copyright 2002-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
-contains test data from http://aspell.net/test/orig/batch0.tab.
-Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
-
-===============================================================================
-
-The content of package org.apache.commons.codec.language.bm has been translated
-from the original php source code available at http://stevemorse.org/phoneticinfo.htm
-with permission from the original authors.
-Original source copyright:
-Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
-
-
-Apache Commons Collections
-Copyright 2001-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Compress
-Copyright 2002-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (https://www.apache.org/).
-
-The files in the package org.apache.commons.compress.archivers.sevenz
-were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/),
-which has been placed in the public domain:
-
-"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html)
-
-
-Apache Commons Configuration
-Copyright 2001-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons CSV
-Copyright 2005-2014 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-src/main/resources/contract.txt
-This file was downloaded from http://www.ferc.gov/docs-filing/eqr/soft-tools/sample-csv/contract.txt and contains neither copyright notice nor license.
-
-src/main/resources/transaction.txt
-This file was downloaded from http://www.ferc.gov/docs-filing/eqr/soft-tools/sample-csv/transaction.txt and contains neither copyright notice nor license.
-
-src/test/resources/CSVFileParser/bom.csv
-src/test/resources/CSVFileParser/test.csv
-src/test/resources/CSVFileParser/test_default.txt
-src/test/resources/CSVFileParser/test_default_comment.txt
-src/test/resources/CSVFileParser/test_rfc4180.txt
-src/test/resources/CSVFileParser/test_rfc4180_trim.txt
-src/test/resources/CSVFileParser/testCSV85.csv
-src/test/resources/CSVFileParser/testCSV85_default.txt
-src/test/resources/CSVFileParser/testCSV85_ignoreEmpty.txt
-These files are used as test data and test result specifications.
-
-
-Apache Commons Daemon
-Copyright 1999-2013 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Digester
-Copyright 2001-2008 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons IO
-Copyright 2002-2016 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Lang
-Copyright 2001-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-This product includes software from the Spring Framework,
-under the Apache License 2.0 (see: StringUtils.containsWhitespace())
-
-
-Apache Commons Logging
-Copyright 2003-2013 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Math
-Copyright 2001-2012 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-===============================================================================
-
-The BracketFinder (package org.apache.commons.math3.optimization.univariate)
-and PowellOptimizer (package org.apache.commons.math3.optimization.general)
-classes are based on the Python code in module "optimize.py" (version 0.5)
-developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/)
-Copyright © 2003-2009 SciPy Developers.
-===============================================================================
-
-The LinearConstraint, LinearObjectiveFunction, LinearOptimizer,
-RelationShip, SimplexSolver and SimplexTableau classes in package
-org.apache.commons.math3.optimization.linear include software developed by
-Benjamin McCann (http://www.benmccann.com) and distributed with
-the following copyright: Copyright 2009 Google Inc.
-===============================================================================
-
-This product includes software developed by the
-University of Chicago, as Operator of Argonne National
-Laboratory.
-The LevenbergMarquardtOptimizer class in package
-org.apache.commons.math3.optimization.general includes software
-translated from the lmder, lmpar and qrsolv Fortran routines
-from the Minpack package
-Minpack Copyright Notice (1999) University of Chicago.  All rights reserved
-===============================================================================
-
-The GraggBulirschStoerIntegrator class in package
-org.apache.commons.math3.ode.nonstiff includes software translated
-from the odex Fortran routine developed by E. Hairer and G. Wanner.
-Original source copyright:
-Copyright (c) 2004, Ernst Hairer
-===============================================================================
-
-The EigenDecompositionImpl class in package
-org.apache.commons.math3.linear includes software translated
-from some LAPACK Fortran routines.  Original source copyright:
-Copyright (c) 1992-2008 The University of Tennessee.  All rights reserved.
-===============================================================================
-
-The MersenneTwister class in package org.apache.commons.math3.random
-includes software translated from the 2002-01-26 version of
-the Mersenne-Twister generator written in C by Makoto Matsumoto and Takuji
-Nishimura. Original source copyright:
-Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
-All rights reserved
-===============================================================================
-
-The LocalizedFormatsTest class in the unit tests is an adapted version of
-the OrekitMessagesTest class from the orekit library distributed under the
-terms of the Apache 2 licence. Original source copyright:
-Copyright 2010 CS Systèmes d'Information
-===============================================================================
-
-The HermiteInterpolator class and its corresponding test have been imported from
-the orekit library distributed under the terms of the Apache 2 licence. Original
-source copyright:
-Copyright 2010-2012 CS Systèmes d'Information
-===============================================================================
-
-The creation of the package "o.a.c.m.analysis.integration.gauss" was inspired
-by an original code donated by Sébastien Brisard.
-===============================================================================
-
-
-The complete text of licenses and disclaimers associated with the the original
-sources enumerated above at the time of code translation are in the LICENSE.txt
-file.
-
-
-Apache Commons Net
-Copyright 2001-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Text
-Copyright 2014-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Commons Validator
-Copyright 2001-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Curator
-Copyright 2013-2014 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Ehcache V3
-Copyright 2014-2016 Terracotta, Inc.
-
-The product includes software from the Apache Commons Lang project,
-under the Apache License 2.0 (see: org.ehcache.impl.internal.classes.commonslang)
-
-
-Apache Geronimo
-Copyright 2003-2018 The Apache Software Foundation
-
-This product includes software developed by
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Copyright 2014 The gRPC Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
------------------------------------------------------------------------
-
-This product contains a modified portion of 'OkHttp', an open source
-HTTP & SPDY client for Android and Java applications, which can be obtained
-at:
-
-  * LICENSE:
-    * okhttp/third_party/okhttp/LICENSE (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/square/okhttp
-  * LOCATION_IN_GRPC:
-    * okhttp/third_party/okhttp
-
-This product contains a modified portion of 'Netty', an open source
-networking library, which can be obtained at:
-
-  * LICENSE:
-    * netty/third_party/netty/LICENSE.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://netty.io
-  * LOCATION_IN_GRPC:
-    * netty/third_party/netty
-
-
-Apache HBase
-Copyright 2007-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
---
-This product incorporates portions of the 'Hadoop' project
-
-Copyright 2007-2009 The Apache Software Foundation
-
-Licensed under the Apache License v2.0
---
-Our Orca logo we got here: http://www.vectorfree.com/jumping-orca
-It is licensed Creative Commons Attribution 3.0.
-See https://creativecommons.org/licenses/by/3.0/us/
-We changed the logo by stripping the colored background, inverting
-it and then rotating it some.
-
-Later we found that vectorfree.com image is not properly licensed.
-The original is owned by vectorportal.com. The original was
-relicensed so we could use it as Creative Commons Attribution 3.0.
-The license is bundled with the download available here:
-http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp
---
-This product includes portions of the Bootstrap project v3.0.0
-
-Copyright 2013 Twitter, Inc.
-
-Licensed under the Apache License v2.0
-
-This product uses the Glyphicons Halflings icon set.
-
-http://glyphicons.com/
-
-Copyright Jan Kovařík
-
-Licensed under the Apache License v2.0 as a part of the Bootstrap project.
-
---
-This product includes portions of the Guava project v14 and v21, specifically
-'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java'
-'hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java'
-
-Copyright (C) 2007 The Guava Authors
-
-Licensed under the Apache License, Version 2.0
-
-
-Apache HTrace
-Copyright 2016 The Apache Software Foundation
-
-This product includes software developed at The Apache Software
-Foundation (http://www.apache.org/).
-
-In addition, this product includes software dependencies. See
-the accompanying LICENSE.txt for a listing of dependencies
-that are NOT Apache licensed (with pointers to their licensing)
-
-Apache HTrace includes an Apache Thrift connector to Zipkin. Zipkin
-is a distributed tracing system that is Apache 2.0 Licensed.
-Copyright 2012 Twitter, Inc.
-
-
-Apache HttpComponents Client
-Copyright 1999-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache HttpComponents Core
-Copyright 2005-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-==============================================================
- Jetty Web Container
- Copyright 1995-2017 Mort Bay Consulting Pty Ltd.
-==============================================================
-
-The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd
-unless otherwise noted.
-
-Jetty is dual licensed under both
-
-  * The Apache 2.0 License
-    http://www.apache.org/licenses/LICENSE-2.0.html
-
-      and
-
-  * The Eclipse Public 1.0 License
-    http://www.eclipse.org/legal/epl-v10.html
-
-Jetty may be distributed under either license.
-
-------
-Eclipse
-
-The following artifacts are EPL.
- * org.eclipse.jetty.orbit:org.eclipse.jdt.core
-
-The following artifacts are EPL and ASL2.
- * org.eclipse.jetty.orbit:javax.security.auth.message
-
-
-The following artifacts are EPL and CDDL 1.0.
- * org.eclipse.jetty.orbit:javax.mail.glassfish
-
-
-------
-Oracle
-
-The following artifacts are CDDL + GPLv2 with classpath exception.
-https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
-
- * javax.servlet:javax.servlet-api
- * javax.annotation:javax.annotation-api
- * javax.transaction:javax.transaction-api
- * javax.websocket:javax.websocket-api
-
-------
-Oracle OpenJDK
-
-If ALPN is used to negotiate HTTP/2 connections, then the following
-artifacts may be included in the distribution or downloaded when ALPN
-module is selected.
-
- * java.sun.security.ssl
-
-These artifacts replace/modify OpenJDK classes.  The modififications
-are hosted at github and both modified and original are under GPL v2 with
-classpath exceptions.
-http://openjdk.java.net/legal/gplv2+ce.html
-
-
-------
-OW2
-
-The following artifacts are licensed by the OW2 Foundation according to the
-terms of http://asm.ow2.org/license.html
-
-org.ow2.asm:asm-commons
-org.ow2.asm:asm
-
-
-------
-Apache
-
-The following artifacts are ASL2 licensed.
-
-org.apache.taglibs:taglibs-standard-spec
-org.apache.taglibs:taglibs-standard-impl
-
-
-------
-MortBay
-
-The following artifacts are ASL2 licensed.  Based on selected classes from
-following Apache Tomcat jars, all ASL2 licensed.
-
-org.mortbay.jasper:apache-jsp
-  org.apache.tomcat:tomcat-jasper
-  org.apache.tomcat:tomcat-juli
-  org.apache.tomcat:tomcat-jsp-api
-  org.apache.tomcat:tomcat-el-api
-  org.apache.tomcat:tomcat-jasper-el
-  org.apache.tomcat:tomcat-api
-  org.apache.tomcat:tomcat-util-scan
-  org.apache.tomcat:tomcat-util
-
-org.mortbay.jasper:apache-el
-  org.apache.tomcat:tomcat-jasper-el
-  org.apache.tomcat:tomcat-el-api
-
-
-------
-Mortbay
-
-The following artifacts are CDDL + GPLv2 with classpath exception.
-
-https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
-
-org.eclipse.jetty.toolchain:jetty-schemas
-
-------
-Assorted
-
-The UnixCrypt.java code implements the one way cryptography used by
-Unix systems for simple password protection.  Copyright 1996 Aki Yoshida,
-modified April 2001  by Iris Van den Broeke, Daniel Deville.
-Permission to use, copy, modify and distribute UnixCrypt
-for non-commercial or commercial purposes and without fee is
-granted provided that the copyright notice appears in all copies.
-
-
-Apache Kafka
-Copyright 2012 The Apache Software Foundation.
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache Kerby
-Copyright 2015-2017 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Apache log4j
-Copyright 2010 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-
-Metrics
-Copyright 2010-2013 Coda Hale and Yammer, Inc.
-
-This product includes software developed by Coda Hale and Yammer, Inc.
-
-This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64,
-LongAdder), which was released with the following comments:
-
-    Written by Doug Lea with assistance from members of JCP JSR-166
-    Expert Group and released to the public domain, as explained at
-    http://creativecommons.org/publicdomain/zero/1.0/
-
-
-
-                            The Netty Project
-                            =================
-
-Please visit the Netty web site for more information:
-
-  * http://netty.io/
-
-Copyright 2014 The Netty Project
-
-The Netty Project licenses this file to you under the Apache License,
-version 2.0 (the "License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at:
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations
-under the License.
-
-Also, please refer to each LICENSE.<component>.txt file, which is located in
-the 'license' directory of the distribution file, for the license terms of the
-components that this product depends on.
-
--------------------------------------------------------------------------------
-This product contains the extensions to Java Collections Framework which has
-been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
-
-  * LICENSE:
-    * license/LICENSE.jsr166y.txt (Public Domain)
-  * HOMEPAGE:
-    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
-    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
-
-This product contains a modified version of Robert Harder's Public Domain
-Base64 Encoder and Decoder, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.base64.txt (Public Domain)
-  * HOMEPAGE:
-    * http://iharder.sourceforge.net/current/java/base64/
-
-This product contains a modified portion of 'Webbit', an event based
-WebSocket and HTTP server, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.webbit.txt (BSD License)
-  * HOMEPAGE:
-    * https://github.com/joewalnes/webbit
-
-This product contains a modified portion of 'SLF4J', a simple logging
-facade for Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.slf4j.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.slf4j.org/
-
-This product contains a modified portion of 'Apache Harmony', an open source
-Java SE, which can be obtained at:
-
-  * NOTICE:
-    * license/NOTICE.harmony.txt
-  * LICENSE:
-    * license/LICENSE.harmony.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://archive.apache.org/dist/harmony/
-
-This product contains a modified portion of 'jbzip2', a Java bzip2 compression
-and decompression library written by Matthew J. Francis. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jbzip2.txt (MIT License)
-  * HOMEPAGE:
-    * https://code.google.com/p/jbzip2/
-
-This product contains a modified portion of 'libdivsufsort', a C API library to construct
-the suffix array and the Burrows-Wheeler transformed string for any input string of
-a constant-size alphabet written by Yuta Mori. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.libdivsufsort.txt (MIT License)
-  * HOMEPAGE:
-    * https://github.com/y-256/libdivsufsort
-
-This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM,
- which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jctools.txt (ASL2 License)
-  * HOMEPAGE:
-    * https://github.com/JCTools/JCTools
-
-This product optionally depends on 'JZlib', a re-implementation of zlib in
-pure Java, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jzlib.txt (BSD style License)
-  * HOMEPAGE:
-    * http://www.jcraft.com/jzlib/
-
-This product optionally depends on 'Compress-LZF', a Java library for encoding and
-decoding data in LZF format, written by Tatu Saloranta. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.compress-lzf.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/ning/compress
-
-This product optionally depends on 'lz4', a LZ4 Java compression
-and decompression library written by Adrien Grand. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.lz4.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/jpountz/lz4-java
-
-This product optionally depends on 'lzma-java', a LZMA Java compression
-and decompression library, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.lzma-java.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/jponge/lzma-java
-
-This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression
-and decompression library written by William Kinney. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jfastlz.txt (MIT License)
-  * HOMEPAGE:
-    * https://code.google.com/p/jfastlz/
-
-This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data
-interchange format, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.protobuf.txt (New BSD License)
-  * HOMEPAGE:
-    * https://github.com/google/protobuf
-
-This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
-a temporary self-signed X.509 certificate when the JVM does not provide the
-equivalent functionality.  It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.bouncycastle.txt (MIT License)
-  * HOMEPAGE:
-    * http://www.bouncycastle.org/
-
-This product optionally depends on 'Snappy', a compression library produced
-by Google Inc, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.snappy.txt (New BSD License)
-  * HOMEPAGE:
-    * https://github.com/google/snappy
-
-This product optionally depends on 'JBoss Marshalling', an alternative Java
-serialization API, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1)
-  * HOMEPAGE:
-    * http://www.jboss.org/jbossmarshalling
-
-This product optionally depends on 'Caliper', Google's micro-
-benchmarking framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.caliper.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/google/caliper
-
-This product optionally depends on 'Apache Commons Logging', a logging
-framework, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-logging.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://commons.apache.org/logging/
-
-This product optionally depends on 'Apache Log4J', a logging framework, which
-can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.log4j.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://logging.apache.org/log4j/
-
-This product optionally depends on 'Aalto XML', an ultra-high performance
-non-blocking XML processor, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.aalto-xml.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * http://wiki.fasterxml.com/AaltoHome
-
-This product contains a modified version of 'HPACK', a Java implementation of
-the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.hpack.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/twitter/hpack
-
-This product contains a modified portion of 'Apache Commons Lang', a Java library
-provides utilities for the java.lang API, which can be obtained at:
-
-  * LICENSE:
-    * license/LICENSE.commons-lang.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://commons.apache.org/proper/commons-lang/
-
-
-This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build.
-
-  * LICENSE:
-    * license/LICENSE.mvn-wrapper.txt (Apache License 2.0)
-  * HOMEPAGE:
-    * https://github.com/takari/maven-wrapper
-
-
-This product includes software developed by Google
- Snappy: http://code.google.com/p/snappy/ (New BSD License)
-
-This product includes software developed by Apache
- PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/
- (Apache 2.0 license)
-
-This library containd statically linked libstdc++. This inclusion is allowed by
-"GCC RUntime Library Exception"
-http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
-
-== Contributors ==
-  * Tatu Saloranta
-    * Providing benchmark suite
-  * Alec Wysoker
-    * Performance and memory usage improvement
-
-
-Apache ZooKeeper
-Copyright 2009-2018 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).

NOTICE.txt

@@ -1,34 +1,632 @@
-Apache Hadoop
-Copyright 2006 and onwards The Apache Software Foundation.
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
 
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
+The binary distribution of this product bundles binaries of
+org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the
+following notices:
+* Copyright 2011 Dain Sundstrom <dain@iq80.com>
+* Copyright 2011 FuseSource Corp. http://fusesource.com
 
-Export Control Notice
----------------------
+The binary distribution of this product bundles binaries of
+AWS SDK for Java - Bundle 1.11.563 (https://github.com/aws/aws-sdk-java),
+which has the following notices:
 
-This distribution includes cryptographic software.  The country in
-which you currently reside may have restrictions on the import,
-possession, use, and/or re-export to another country, of
-encryption software.  BEFORE using any encryption software, please
-check your country's laws, regulations and policies concerning the
-import, possession, or use, and re-export of encryption software, to
-see if this is permitted.  See <http://www.wassenaar.org/> for more
-information.
+This software includes third party software subject to the following copyrights:
+- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
 
-The U.S. Government Department of Commerce, Bureau of Industry and
-Security (BIS), has classified this software as Export Commodity
-Control Number (ECCN) 5D002.C.1, which includes information security
-software using or performing cryptographic functions with asymmetric
-algorithms.  The form and manner of this Apache Software Foundation
-distribution makes it eligible for export under the License Exception
-ENC Technology Software Unrestricted (TSU) exception (see the BIS
-Export Administration Regulations, Section 740.13) for both object
-code and source code.
+The binary distribution of this product bundles binaries of
+Gson 2.2.4,
+which has the following notices:
 
-The following provides more details on the included cryptographic software:
+                            The Netty Project
+                            =================
 
-This software uses the SSL libraries from the Jetty project written
-by mortbay.org.
-Hadoop Yarn Server Web Proxy uses the BouncyCastle Java
-cryptography APIs written by the Legion of the Bouncy Castle Inc.
+Please visit the Netty web site for more information:
+
+  * http://netty.io/
+
+Copyright 2014 The Netty Project
+
+The Netty Project licenses this file to you under the Apache License,
+version 2.0 (the "License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at:
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+
+Also, please refer to each LICENSE.<component>.txt file, which is located in
+the 'license' directory of the distribution file, for the license terms of the
+components that this product depends on.
+
+-------------------------------------------------------------------------------
+This product contains the extensions to Java Collections Framework which has
+been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene:
+
+  * LICENSE:
+    * license/LICENSE.jsr166y.txt (Public Domain)
+  * HOMEPAGE:
+    * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+    * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+
+This product contains a modified version of Robert Harder's Public Domain
+Base64 Encoder and Decoder, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.base64.txt (Public Domain)
+  * HOMEPAGE:
+    * http://iharder.sourceforge.net/current/java/base64/
+
+This product contains a modified portion of 'Webbit', an event based
+WebSocket and HTTP server, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.webbit.txt (BSD License)
+  * HOMEPAGE:
+    * https://github.com/joewalnes/webbit
+
+This product contains a modified portion of 'SLF4J', a simple logging
+facade for Java, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.slf4j.txt (MIT License)
+  * HOMEPAGE:
+    * http://www.slf4j.org/
+
+This product contains a modified portion of 'ArrayDeque', written by Josh
+Bloch of Google, Inc:
+
+  * LICENSE:
+    * license/LICENSE.deque.txt (Public Domain)
+
+This product contains a modified portion of 'Apache Harmony', an open source
+Java SE, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.harmony.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://archive.apache.org/dist/harmony/
+
+This product contains a modified version of Roland Kuhn's ASL2
+AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue.
+It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.abstractnodequeue.txt (Public Domain)
+  * HOMEPAGE:
+    * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java
+
+This product contains a modified portion of 'jbzip2', a Java bzip2 compression
+and decompression library written by Matthew J. Francis. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jbzip2.txt (MIT License)
+  * HOMEPAGE:
+    * https://code.google.com/p/jbzip2/
+
+This product contains a modified portion of 'libdivsufsort', a C API library to construct
+the suffix array and the Burrows-Wheeler transformed string for any input string of
+a constant-size alphabet written by Yuta Mori. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.libdivsufsort.txt (MIT License)
+  * HOMEPAGE:
+    * https://code.google.com/p/libdivsufsort/
+
+This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM,
+ which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jctools.txt (ASL2 License)
+  * HOMEPAGE:
+    * https://github.com/JCTools/JCTools
+
+This product optionally depends on 'JZlib', a re-implementation of zlib in
+pure Java, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jzlib.txt (BSD style License)
+  * HOMEPAGE:
+    * http://www.jcraft.com/jzlib/
+
+This product optionally depends on 'Compress-LZF', a Java library for encoding and
+decoding data in LZF format, written by Tatu Saloranta. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.compress-lzf.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/ning/compress
+
+This product optionally depends on 'lz4', a LZ4 Java compression
+and decompression library written by Adrien Grand. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.lz4.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/jpountz/lz4-java
+
+This product optionally depends on 'lzma-java', a LZMA Java compression
+and decompression library, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.lzma-java.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/jponge/lzma-java
+
+This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression
+and decompression library written by William Kinney. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jfastlz.txt (MIT License)
+  * HOMEPAGE:
+    * https://code.google.com/p/jfastlz/
+
+This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data
+interchange format, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.protobuf.txt (New BSD License)
+  * HOMEPAGE:
+    * http://code.google.com/p/protobuf/
+
+This product optionally depends on 'Bouncy Castle Crypto APIs' to generate
+a temporary self-signed X.509 certificate when the JVM does not provide the
+equivalent functionality.  It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.bouncycastle.txt (MIT License)
+  * HOMEPAGE:
+    * http://www.bouncycastle.org/
+
+This product optionally depends on 'Snappy', a compression library produced
+by Google Inc, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.snappy.txt (New BSD License)
+  * HOMEPAGE:
+    * http://code.google.com/p/snappy/
+
+This product contains a modified portion of UnsignedBytes LexicographicalComparator
+from Guava v21 project by Google Inc, which can be obtained at:
+
+  * LICENSE:
+    * license/COPYING (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/google/guava
+
+This product optionally depends on 'JBoss Marshalling', an alternative Java
+serialization API, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1)
+  * HOMEPAGE:
+    * http://www.jboss.org/jbossmarshalling
+
+This product optionally depends on 'Caliper', Google's micro-
+benchmarking framework, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.caliper.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://code.google.com/p/caliper/
+
+This product optionally depends on 'Apache Commons Logging', a logging
+framework, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.commons-logging.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://commons.apache.org/logging/
+
+This product optionally depends on 'Apache Log4J', a logging framework, which
+can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.log4j.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://logging.apache.org/log4j/
+
+This product optionally depends on 'Aalto XML', an ultra-high performance
+non-blocking XML processor, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.aalto-xml.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * http://wiki.fasterxml.com/AaltoHome
+
+This product contains a modified version of 'HPACK', a Java implementation of
+the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.hpack.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * https://github.com/twitter/hpack
+
+This product contains a modified portion of 'Apache Commons Lang', a Java library
+provides utilities for the java.lang API, which can be obtained at:
+
+  * LICENSE:
+    * license/LICENSE.commons-lang.txt (Apache License 2.0)
+  * HOMEPAGE:
+    * https://commons.apache.org/proper/commons-lang/
+
+This product contains a modified portion of 'JDOM 1.1', which can be obtained at:
+
+  * LICENSE:
+    * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt
+  * HOMEPAGE:
+    * http://www.jdom.org/
+
+The binary distribution of this product bundles binaries of
+Commons Codec 1.4,
+which has the following notices:
+ * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+  ===============================================================================
+  The content of package org.apache.commons.codec.language.bm has been translated
+  from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+  with permission from the original authors.
+  Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
+
+The binary distribution of this product bundles binaries of
+Commons Lang 2.6,
+which has the following notices:
+ * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace())
+
+The binary distribution of this product bundles binaries of
+Apache Log4j 1.2.17,
+which has the following notices:
+ * ResolverUtil.java
+    Copyright 2005-2006 Tim Fennell
+  Dumbster SMTP test server
+    Copyright 2004 Jason Paul Kitchen
+  TypeUtil.java
+    Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams
+
+The binary distribution of this product bundles binaries of
+"Java Concurrency in Practice" book annotations 1.0,
+which has the following notices:
+ * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative
+  Commons Attribution License (http://creativecommons.org/licenses/by/2.5)
+  Official home: http://www.jcip.net Any republication or derived work
+  distributed in source code form must include this copyright and license
+  notice.
+
+The binary distribution of this product bundles binaries of
+Jetty :: Http Utility 9.3.19.,
+Jetty :: IO Utility 9.3.19.,
+Jetty :: Security 9.3.19.,
+Jetty :: Server Core 9.3.19.,
+Jetty :: Servlet Handling 9.3.19.,
+Jetty :: Utilities 9.3.19.,
+Jetty :: Utilities :: Ajax,
+Jetty :: Webapp Application Support 9.3.19.,
+Jetty :: XML utilities 9.3.19.,
+which has the following notices:
+ * ==============================================================
+    Jetty Web Container
+    Copyright 1995-2016 Mort Bay Consulting Pty Ltd.
+   ==============================================================
+
+   The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd
+   unless otherwise noted.
+
+   Jetty is dual licensed under both
+
+     * The Apache 2.0 License
+       http://www.apache.org/licenses/LICENSE-2.0.html
+
+         and
+
+     * The Eclipse Public 1.0 License
+       http://www.eclipse.org/legal/epl-v10.html
+
+   Jetty may be distributed under either license.
+
+   ------
+   Eclipse
+
+   The following artifacts are EPL.
+    * org.eclipse.jetty.orbit:org.eclipse.jdt.core
+
+   The following artifacts are EPL and ASL2.
+    * org.eclipse.jetty.orbit:javax.security.auth.message
+
+
+   The following artifacts are EPL and CDDL 1.0.
+    * org.eclipse.jetty.orbit:javax.mail.glassfish
+
+
+   ------
+   Oracle
+
+   The following artifacts are CDDL + GPLv2 with classpath exception.
+   https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
+
+    * javax.servlet:javax.servlet-api
+    * javax.annotation:javax.annotation-api
+    * javax.transaction:javax.transaction-api
+    * javax.websocket:javax.websocket-api
+
+   ------
+   Oracle OpenJDK
+
+   If ALPN is used to negotiate HTTP/2 connections, then the following
+   artifacts may be included in the distribution or downloaded when ALPN
+   module is selected.
+
+    * java.sun.security.ssl
+
+   These artifacts replace/modify OpenJDK classes.  The modififications
+   are hosted at github and both modified and original are under GPL v2 with
+   classpath exceptions.
+   http://openjdk.java.net/legal/gplv2+ce.html
+
+
+   ------
+   OW2
+
+   The following artifacts are licensed by the OW2 Foundation according to the
+   terms of http://asm.ow2.org/license.html
+
+   org.ow2.asm:asm-commons
+   org.ow2.asm:asm
+
+
+   ------
+   Apache
+
+   The following artifacts are ASL2 licensed.
+
+   org.apache.taglibs:taglibs-standard-spec
+   org.apache.taglibs:taglibs-standard-impl
+
+
+   ------
+   MortBay
+
+   The following artifacts are ASL2 licensed.  Based on selected classes from
+   following Apache Tomcat jars, all ASL2 licensed.
+
+   org.mortbay.jasper:apache-jsp
+     org.apache.tomcat:tomcat-jasper
+     org.apache.tomcat:tomcat-juli
+     org.apache.tomcat:tomcat-jsp-api
+     org.apache.tomcat:tomcat-el-api
+     org.apache.tomcat:tomcat-jasper-el
+     org.apache.tomcat:tomcat-api
+     org.apache.tomcat:tomcat-util-scan
+     org.apache.tomcat:tomcat-util
+
+   org.mortbay.jasper:apache-el
+     org.apache.tomcat:tomcat-jasper-el
+     org.apache.tomcat:tomcat-el-api
+
+
+   ------
+   Mortbay
+
+   The following artifacts are CDDL + GPLv2 with classpath exception.
+
+   https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html
+
+   org.eclipse.jetty.toolchain:jetty-schemas
+
+   ------
+   Assorted
+
+   The UnixCrypt.java code implements the one way cryptography used by
+   Unix systems for simple password protection.  Copyright 1996 Aki Yoshida,
+   modified April 2001  by Iris Van den Broeke, Daniel Deville.
+   Permission to use, copy, modify and distribute UnixCrypt
+   for non-commercial or commercial purposes and without fee is
+   granted provided that the copyright notice appears in all copies./
+
+The binary distribution of this product bundles binaries of
+Snappy for Java 1.0.4.1,
+which has the following notices:
+ * This product includes software developed by Google
+    Snappy: http://code.google.com/p/snappy/ (New BSD License)
+
+   This product includes software developed by Apache
+    PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/
+    (Apache 2.0 license)
+
+   This library containd statically linked libstdc++. This inclusion is allowed by
+   "GCC RUntime Library Exception"
+   http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html
+
+   == Contributors ==
+     * Tatu Saloranta
+       * Providing benchmark suite
+     * Alec Wysoker
+       * Performance and memory usage improvement
+
+The binary distribution of this product bundles binaries of
+Xerces2 Java Parser 2.9.1,
+which has the following notices:
+ * =========================================================================
+   ==  NOTICE file corresponding to section 4(d) of the Apache License,   ==
+   ==  Version 2.0, in this case for the Apache Xerces Java distribution. ==
+   =========================================================================
+
+   Apache Xerces Java
+   Copyright 1999-2007 The Apache Software Foundation
+
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+
+   Portions of this software were originally based on the following:
+     - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
+     - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
+     - voluntary contributions made by Paul Eng on behalf of the
+       Apache Software Foundation that were originally developed at iClick, Inc.,
+       software copyright (c) 1999.
+
+The binary distribution of this product bundles binaries of
+Logback Classic Module 1.1.2,
+Logback Core Module 1.1.2,
+which has the following notices:
+ * Logback: the reliable, generic, fast and flexible logging framework.
+   Copyright (C) 1999-2012, QOS.ch. All rights reserved.
+
+The binary distribution of this product bundles binaries of
+Apache HBase - Annotations 1.2.6,
+Apache HBase - Client 1.2.6,
+Apache HBase - Common 1.2.6,
+Apache HBase - Hadoop Compatibility 1.2.6,
+Apache HBase - Hadoop Two Compatibility 1.2.6,
+Apache HBase - Prefix Tree 1.2.6,
+Apache HBase - Procedure 1.2.6,
+Apache HBase - Protocol 1.2.6,
+Apache HBase - Server 1.2.6,
+which has the following notices:
+ * Apache HBase
+   Copyright 2007-2015 The Apache Software Foundation
+
+   This product includes software developed at
+   The Apache Software Foundation (http://www.apache.org/).
+
+   --
+   This product incorporates portions of the 'Hadoop' project
+
+   Copyright 2007-2009 The Apache Software Foundation
+
+   Licensed under the Apache License v2.0
+   --
+   Our Orca logo we got here: http://www.vectorfree.com/jumping-orca
+   It is licensed Creative Commons Attribution 3.0.
+   See https://creativecommons.org/licenses/by/3.0/us/
+   We changed the logo by stripping the colored background, inverting
+   it and then rotating it some.
+
+   Later we found that vectorfree.com image is not properly licensed.
+   The original is owned by vectorportal.com. The original was
+   relicensed so we could use it as Creative Commons Attribution 3.0.
+   The license is bundled with the download available here:
+   http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp
+   --
+   This product includes portions of the Bootstrap project v3.0.0
+
+   Copyright 2013 Twitter, Inc.
+
+   Licensed under the Apache License v2.0
+
+   This product uses the Glyphicons Halflings icon set.
+
+   http://glyphicons.com/
+
+   Copyright Jan Kovařík
+
+   Licensed under the Apache License v2.0 as a part of the Bootstrap project.
+
+   --
+   This product includes portions of the Guava project v14, specifically
+   'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java'
+
+   Copyright (C) 2007 The Guava Authors
+
+   Licensed under the Apache License, Version 2.0
+
+The binary distribution of this product bundles binaries of
+Phoenix Core 4.7.0,
+which has the following notices:
+   Apache Phoenix
+   Copyright 2013-2016 The Apache Software Foundation
+
+   This product includes software developed by The Apache Software
+   Foundation (http://www.apache.org/).
+
+   This also includes:
+
+   The phoenix-spark module has been adapted from the phoenix-spark library
+   distributed under the terms of the Apache 2 license. Original source copyright:
+   Copyright 2014 Simply Measured, Inc.
+   Copyright 2015 Interset Software Inc.
+
+   The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5
+   (https://pypi.python.org/pypi/python-daemon/). Original source copyright:
+   # Copyright © 2008–2015 Ben Finney <ben+python@benfinney.id.au>
+   # Copyright © 2007–2008 Robert Niederreiter, Jens Klein
+   # Copyright © 2004–2005 Chad J. Schroeder
+   # Copyright © 2003 Clark Evans
+   # Copyright © 2002 Noah Spurrier
+   # Copyright © 2001 Jürgen Hermann
+
+The binary distribution of this product bundles binaries of
+Plexus Cipher: encryption/decryption Component 1.4,
+which has the following notices:
+ * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java
+   which is Apache license: http://www.apache.org/licenses/LICENSE-2.0
+
+   The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java
+    which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0
+
+The binary distribution of this product bundles binaries of
+software.amazon.ion:ion-java 1.0.1,
+which has the following notices:
+ * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+The binary distribution of this product bundles binaries of
+joda-time:joda-time:2.9.9
+which has the following notices:
+ * =============================================================================
+   = NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
+   =============================================================================
+   This product includes software developed by
+   Joda.org (http://www.joda.org/).
+
+The binary distribution of this product bundles binaries of
+Ehcache 3.3.1,
+which has the following notices:
+ * Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software
+
+The binary distribution of this product bundles binaries of
+metrics-core 3.2.4
+which has the following notices:
+ * Copyright 2010-2013 Coda Hale and Yammer, Inc.
+
+   This product includes software developed by Coda Hale and Yammer, Inc.
+
+   This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64,
+   LongAdder), which was released with the following comments:
+
+   Written by Doug Lea with assistance from members of JCP JSR-166
+   Expert Group and released to the public domain, as explained at
+   http://creativecommons.org/publicdomain/zero/1.0/
+
+The source and binary distribution of this product bundles binaries of
+xterm.js 3.8.0
+which has the following notices:
+Copyright (c) 2017-2018, The xterm.js authors (https://github.com/xtermjs/xterm.js)
+Copyright (c) 2014-2016, SourceLair Private Company (https://www.sourcelair.com)
+Copyright (c) 2012-2013, Christopher Jeffrey (https://github.com/chjj/)
+
+The source and binary distribution of this product bundles modified version of
+  github.com/awslabs/aws-js-s3-explorer licensed under Apache 2.0 license
+  with the following notice:
+
+AWS JavaScript S3 Explorer
+Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+The binary distribution of this product bundles binaries of
+jline 3.9.0 (https://github.com/jline/jline3)
+
+  * LICENSE:
+    * license/LICENSE.jline3.txt (BSD License)
+  * HOMEPAGE:
+    * https://github.com/jline/jline3

README.txt

@@ -4,4 +4,30 @@ For the latest information about Hadoop, please visit our website at:
 
 and our wiki, at:
 
-   https://cwiki.apache.org/confluence/display/HADOOP/
+   http://wiki.apache.org/hadoop/
+
+This distribution includes cryptographic software.  The country in 
+which you currently reside may have restrictions on the import, 
+possession, use, and/or re-export to another country, of 
+encryption software.  BEFORE using any encryption software, please 
+check your country's laws, regulations and policies concerning the
+import, possession, or use, and re-export of encryption software, to 
+see if this is permitted.  See <http://www.wassenaar.org/> for more
+information.
+
+The U.S. Government Department of Commerce, Bureau of Industry and
+Security (BIS), has classified this software as Export Commodity 
+Control Number (ECCN) 5D002.C.1, which includes information security
+software using or performing cryptographic functions with asymmetric
+algorithms.  The form and manner of this Apache Software Foundation
+distribution makes it eligible for export under the License Exception
+ENC Technology Software Unrestricted (TSU) exception (see the BIS 
+Export Administration Regulations, Section 740.13) for both object 
+code and source code.
+
+The following provides more details on the included cryptographic
+software:
+  Hadoop Core uses the SSL libraries from the Jetty project written 
+by mortbay.org.
+  Hadoop Yarn Server Web Proxy uses the BouncyCastle Java
+cryptography APIs written by the Legion of the Bouncy Castle Inc.

dev-support/Jenkinsfile

@@ -1,338 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-def getGithubCreds() {
-    return [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
-                                passwordVariable: 'GITHUB_TOKEN',
-                                usernameVariable: 'GITHUB_USER')]
-}
-
-// Publish JUnit results only if there are XML files under surefire-reports
-def publishJUnitResults() {
-    def findCmdExitCode = sh script: "find ${SOURCEDIR} -wholename */target/surefire-reports/*.xml | egrep .", returnStatus: true
-    boolean surefireReportsExist = findCmdExitCode == 0
-    if (surefireReportsExist) {
-        echo "XML files found under surefire-reports, running junit"
-        // The path should be relative to WORKSPACE for the junit.
-        SRC = "${SOURCEDIR}/**/target/surefire-reports/*.xml".replace("$WORKSPACE/","")
-        try {
-            junit "${SRC}"
-        } catch(e) {
-            echo 'junit processing: ' + e.toString()
-        }
-    } else {
-        echo "No XML files found under surefire-reports, skipping junit"
-    }
-}
-
-pipeline {
-
-    agent {
-        label 'Hadoop'
-    }
-
-    options {
-        buildDiscarder(logRotator(numToKeepStr: '5'))
-        timeout (time: 48, unit: 'HOURS')
-        timestamps()
-        checkoutToSubdirectory('src')
-    }
-
-    environment {
-        YETUS='yetus'
-        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
-        YETUS_VERSION='rel/0.14.0'
-    }
-
-    parameters {
-        string(name: 'JIRA_ISSUE_KEY',
-               defaultValue: '',
-               description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
-    }
-
-    stages {
-        stage ('install yetus') {
-            steps {
-                dir("${WORKSPACE}/${YETUS}") {
-                    checkout([
-                        $class: 'GitSCM',
-                        branches: [[name: "${env.YETUS_VERSION}"]],
-                        userRemoteConfigs: [[ url: 'https://github.com/apache/yetus.git']]]
-                    )
-                }
-            }
-        }
-
-        // Setup codebase so that each platform's build happens in its own exclusive copy of the
-        // codebase.
-        // Primarily because YETUS messes up the git branch information and affects the subsequent
-        // optional stages after the first one.
-        stage ('setup sources') {
-            steps {
-                dir("${WORKSPACE}/centos-7") {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7
-                    '''
-                }
-
-                dir("${WORKSPACE}/centos-8") {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
-                    '''
-                }
-
-                dir("${WORKSPACE}/debian-10") {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
-                    '''
-                }
-
-                dir("${WORKSPACE}/ubuntu-focal") {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
-                    '''
-                }
-            }
-        }
-
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 7.
-        stage ('precommit-run Centos 7') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/centos-7/src"
-                PATCHDIR = "${WORKSPACE}/centos-7/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(getGithubCreds()) {
-                    sh '''#!/usr/bin/env bash
-
-                    chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                    "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                    '''
-                }
-            }
-
-            post {
-                // Since this is an optional platform, we want to copy the artifacts
-                // and archive it only if the build fails, to help with debugging.
-                failure {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}"
-                    '''
-                    archiveArtifacts "out/**"
-                }
-
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Centos 8.
-        stage ('precommit-run Centos 8') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/centos-8/src"
-                PATCHDIR = "${WORKSPACE}/centos-8/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(getGithubCreds()) {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                        '''
-                }
-            }
-
-            post {
-                // Since this is an optional platform, we want to copy the artifacts
-                // and archive it only if the build fails, to help with debugging.
-                failure {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
-                    '''
-                    archiveArtifacts "out/**"
-                }
-
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-
-        // This is an optional stage which runs only when there's a change in
-        // C++/C++ build/platform.
-        // This stage serves as a means of cross platform validation, which is
-        // really needed to ensure that any C++ related/platform change doesn't
-        // break the Hadoop build on Debian 10.
-        stage ('precommit-run Debian 10') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/debian-10/src"
-                PATCHDIR = "${WORKSPACE}/debian-10/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
-                IS_OPTIONAL = 1
-            }
-
-            steps {
-                withCredentials(getGithubCreds()) {
-                    sh '''#!/usr/bin/env bash
-
-                    chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                    "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                    '''
-                }
-            }
-
-            post {
-                // Since this is an optional platform, we want to copy the artifacts
-                // and archive it only if the build fails, to help with debugging.
-                failure {
-                    sh '''#!/usr/bin/env bash
-
-                    cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
-                    '''
-                    archiveArtifacts "out/**"
-                }
-
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-
-        // We want to use Ubuntu Focal as our main CI and thus, this stage
-        // isn't optional (runs for all the PRs).
-        stage ('precommit-run Ubuntu focal') {
-            environment {
-                SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
-                PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
-                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
-                IS_OPTIONAL = 0
-            }
-
-            steps {
-                withCredentials(getGithubCreds()) {
-                    sh '''#!/usr/bin/env bash
-
-                    chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                    "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
-                    '''
-                }
-            }
-
-            post {
-                always {
-                    script {
-                        // Publish status if it was missed (YETUS-1059)
-                        withCredentials(
-                            [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
-                                              passwordVariable: 'GITHUB_TOKEN',
-                                              usernameVariable: 'GITHUB_USER')]) {
-                            sh '''#!/usr/bin/env bash
-
-                            # Copy the artifacts of Ubuntu focal build to workspace
-                            cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
-
-                            # Send Github status
-                            chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                            "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
-                            '''
-                        }
-
-                        // YETUS output
-                        archiveArtifacts "out/**"
-
-                        // Publish the HTML report so that it can be looked at
-                        // Has to be relative to WORKSPACE.
-                        publishHTML (target: [
-                                      allowMissing: true,
-                                      keepAll: true,
-                                      alwaysLinkToLastBuild: true,
-                                      // Has to be relative to WORKSPACE
-                                      reportDir: "out",
-                                      reportFiles: 'report.html',
-                                      reportName: 'Yetus Report'
-                        ])
-
-                        publishJUnitResults()
-                    }
-                }
-
-                cleanup() {
-                    script {
-                        sh '''#!/usr/bin/env bash
-
-                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
-                        '''
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        // Jenkins pipeline jobs fill slaves on PRs without this :(
-        cleanup() {
-            script {
-                sh '''#!/usr/bin/env bash
-
-                # See HADOOP-13951
-                chmod -R u+rxw "${WORKSPACE}"
-                '''
-                deleteDir()
-            }
-        }
-    }
-}

dev-support/bin/checkcompatibility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -30,16 +30,33 @@
 import shutil
 import subprocess
 import sys
-import urllib.request
-import argparse
+import urllib2
+try:
+  import argparse
+except ImportError:
+  sys.stderr.write("Please install argparse, e.g. via `pip install argparse`.")
+  sys.exit(2)
 
 # Various relative paths
 REPO_DIR = os.getcwd()
 
 def check_output(*popenargs, **kwargs):
-  """ Run command with arguments and return its output as a string. """
-  return subprocess.check_output(*popenargs, **kwargs, encoding='utf-8')
-
+  r"""Run command with arguments and return its output as a byte string.
+  Backported from Python 2.7 as it's implemented as pure python on stdlib.
+  >>> check_output(['/usr/bin/python', '--version'])
+  Python 2.6.2
+  """
+  process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+  output, _ = process.communicate()
+  retcode = process.poll()
+  if retcode:
+    cmd = kwargs.get("args")
+    if cmd is None:
+      cmd = popenargs[0]
+    error = subprocess.CalledProcessError(retcode, cmd)
+    error.output = output
+    raise error
+  return output
 
 def get_repo_dir():
   """ Return the path to the top of the repo. """
@@ -122,7 +139,7 @@ def checkout_java_acc(force):
   url = "https://github.com/lvc/japi-compliance-checker/archive/1.8.tar.gz"
   scratch_dir = get_scratch_dir()
   path = os.path.join(scratch_dir, os.path.basename(url))
-  jacc = urllib.request.urlopen(url)
+  jacc = urllib2.urlopen(url)
   with open(path, 'wb') as w:
     w.write(jacc.read())
 
@@ -175,9 +192,9 @@ def run_java_acc(src_name, src_jars, dst_name, dst_jars, annotations):
 
   if annotations is not None:
     annotations_path = os.path.join(get_scratch_dir(), "annotations.txt")
-    with open(annotations_path, "w") as f:
+    with file(annotations_path, "w") as f:
       for ann in annotations:
-        print(ann, file=f)
+        print >>f, ann
     args += ["-annotations-list", annotations_path]
 
   subprocess.check_call(args)
@@ -247,8 +264,8 @@ def main():
   parser.add_argument("--skip-build",
                       action="store_true",
                       help="Skip building the projects.")
-  parser.add_argument("src_rev", nargs=1, type=str, help="Source revision.")
-  parser.add_argument("dst_rev", nargs="?", type=str, default="HEAD",
+  parser.add_argument("src_rev", nargs=1, help="Source revision.")
+  parser.add_argument("dst_rev", nargs="?", default="HEAD",
                       help="Destination revision. " +
                       "If not specified, will use HEAD.")
 

dev-support/bin/create-release

@@ -204,11 +204,6 @@ function set_defaults
   DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile"
   DOCKERRAN=false
 
-  CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
-  if [ "$CPU_ARCH" = "aarch64" ]; then
-    DOCKERFILE="${BASEDIR}/dev-support/docker/Dockerfile_aarch64"
-  fi
-
   # Extract Java version from ${BASEDIR}/pom.xml
   # doing this outside of maven means we can do this before
   # the docker container comes up...
@@ -254,9 +249,7 @@ function startgpgagent
       eval $("${GPGAGENT}" --daemon \
         --options "${LOGDIR}/gpgagent.conf" \
         --log-file="${LOGDIR}/create-release-gpgagent.log")
-      GPGAGENTPID=$(pgrep "${GPGAGENT}")
-      GPG_AGENT_INFO="$HOME/.gnupg/S.gpg-agent:$GPGAGENTPID:1"
-      export GPG_AGENT_INFO
+      GPGAGENTPID=$(echo "${GPG_AGENT_INFO}" | cut -f 2 -d:)
     fi
 
     if [[ -n "${GPG_AGENT_INFO}" ]]; then
@@ -293,7 +286,6 @@ function usage
   echo "--security              Emergency security release"
   echo "--sign                  Use .gnupg dir to sign the artifacts and jars"
   echo "--version=[version]     Use an alternative version string"
-  echo "--mvnargs=[args]        Extra Maven args to be provided when running mvn commands"
 }
 
 function option_parse
@@ -348,9 +340,6 @@ function option_parse
       --version=*)
         HADOOP_VERSION=${i#*=}
       ;;
-      --mvnargs=*)
-        MVNEXTRAARGS=${i#*=}
-      ;;
     esac
   done
 
@@ -417,9 +406,6 @@ function option_parse
       MVN_ARGS=("-Dmaven.repo.local=${MVNCACHE}")
     fi
   fi
-  if [ -n "$MVNEXTRAARGS" ]; then
-    MVN_ARGS+=("$MVNEXTRAARGS")
-  fi
 
   if [[ "${SECURITYRELEASE}" = true ]]; then
     if [[ ! -d "${BASEDIR}/hadoop-common-project/hadoop-common/src/site/markdown/release/${HADOOP_VERSION}" ]]; then
@@ -513,15 +499,10 @@ function dockermode
 
     # we always force build with the OpenJDK JDK
     # but with the correct version
-    if [ "$CPU_ARCH" = "aarch64" ]; then
-      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
-    else
-      echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
-    fi
-
+    echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
     echo "USER ${user_name}"
     printf "\n\n"
-  ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
+  ) | docker build -t "${imgname}" -
 
   run docker run -i -t \
     --privileged \
@@ -542,10 +523,6 @@ function makearelease
 
   big_console_header "Cleaning the Source Tree"
 
-  # Since CVE-2022-24765 in April 2022, git refuses to work in directories
-  # whose owner != the current user, unless explicitly told to trust it.
-  git config --global --add safe.directory /build/source
-
   # git clean to clear any remnants from previous build
   run "${GIT}" clean -xdf -e /patchprocess
 
@@ -662,12 +639,10 @@ function signartifacts
 
   big_console_header "Signing the release"
 
-  run cd "${ARTIFACTS_DIR}"
-  for i in *; do
+  for i in ${ARTIFACTS_DIR}/*; do
     ${GPG} --use-agent --armor --output "${i}.asc" --detach-sig "${i}"
-    sha512sum --tag "${i}" > "${i}.sha512"
+    ${GPG} --print-mds "${i}" > "${i}.mds"
   done
-  run cd "${BASEDIR}"
 
   if [[ "${ASFRELEASE}" = true ]]; then
     echo "Fetching the Apache Hadoop KEYS file..."

dev-support/bin/dist-copynativelibs

@@ -72,7 +72,7 @@ function bundle_native_bin
   fi
 
   cd "${libdir}" || exit 1
-  ${TAR} ./*"${binpattern}"* | (cd "${TARGET_BIN_DIR}"/  || exit 1 ; ${UNTAR})
+  ${TAR} ./*"${libpattern}"* | (cd "${TARGET_BIN_DIR}"/  || exit 1 ; ${UNTAR})
   if [[ $? -ne 0 ]]; then
     echo "Bundling bin files for ${binoption} failed"
     exit 1
@@ -111,6 +111,9 @@ for i in "$@"; do
     --openssllibbundle=*)
       OPENSSLLIBBUNDLE=${i#*=}
     ;;
+    --snappybinbundle=*)
+      SNAPPYBINBUNDLE=${i#*=}
+    ;;
     --snappylib=*)
       SNAPPYLIB=${i#*=}
     ;;
@@ -149,13 +152,13 @@ if [[ -d "${LIB_DIR}" ]]; then
     exit 1
   fi
 
-  bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "libsnappy." "${SNAPPYLIB}"
+  bundle_native_lib "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
 
-  bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "libzstd." "${ZSTDLIB}"
+  bundle_native_lib "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
 
-  bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "libcrypto." "${OPENSSLLIB}"
+  bundle_native_lib "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"
 
-  bundle_native_lib "${ISALBUNDLE}" "isal.lib" "libisal." "${ISALLIB}"
+  bundle_native_lib "${ISALBUNDLE}" "isal.lib" "isa" "${ISALLIB}"
 
   bundle_native_lib "${PMDKBUNDLE}" "pmdk.lib" "pmdk" "${PMDKLIB}"
 fi
@@ -164,7 +167,7 @@ fi
 
 # Windows doesn't have a LIB_DIR, everything goes into bin
 
-if [[ -d "${BIN_DIR}" && $(ls -A "${BIN_DIR}") ]] ; then
+if [[ -d "${BIN_DIR}" ]] ; then
   mkdir -p "${TARGET_BIN_DIR}"
   cd "${BIN_DIR}"  || exit 1
   ${TAR} ./* | (cd "${TARGET_BIN_DIR}"/ || exit 1; ${UNTAR})
@@ -173,6 +176,8 @@ if [[ -d "${BIN_DIR}" && $(ls -A "${BIN_DIR}") ]] ; then
     exit 1
   fi
 
+  bundle_native_bin "${SNAPPYBINBUNDLE}" "${SNAPPYLIBBUNDLE}" "snappy.lib" "snappy" "${SNAPPYLIB}"
+
   bundle_native_bin "${ZSTDBINBUNDLE}" "${ZSTDLIBBUNDLE}" "zstd.lib" "zstd" "${ZSTDLIB}"
 
   bundle_native_bin "${OPENSSLBINBUNDLE}" "${OPENSSLLIBBUNDLE}" "openssl.lib" "crypto" "${OPENSSLLIB}"

dev-support/bin/dist-layout-stitching

@@ -21,6 +21,9 @@ VERSION=$1
 # project.build.directory
 BASEDIR=$2
 
+#hdds.version
+HDDS_VERSION=$3
+
 function run()
 {
   declare res
@@ -119,10 +122,6 @@ run mkdir "hadoop-${VERSION}"
 run cd "hadoop-${VERSION}"
 run cp -p "${ROOT}/LICENSE.txt" .
 run cp -p "${ROOT}/NOTICE.txt" .
-run cp -p "${ROOT}/LICENSE-binary" .
-run mkdir licenses-binary
-run cp -p "${ROOT}/licenses-binary"/* licenses-binary/
-run cp -p "${ROOT}/NOTICE-binary" .
 run cp -p "${ROOT}/README.txt" .
 
 # Copy hadoop-common first so that it have always have all dependencies.
@@ -149,7 +148,6 @@ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-cli
 run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
 
 run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
-run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
 
 
 echo

dev-support/bin/hadoop.sh

@@ -1,615 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# SHELLDOC-IGNORE
-#
-# Override these to match Apache Hadoop's requirements
-personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
-
-# These flags are needed to run Yetus against Hadoop on Windows.
-WINDOWS_FLAGS="-Pnative-win
-  -Dhttps.protocols=TLSv1.2
-  -Drequire.openssl
-  -Drequire.test.libhadoop
-  -Dshell-executable=${BASH_EXECUTABLE}
-  -Dopenssl.prefix=${VCPKG_INSTALLED_PACKAGES}
-  -Dcmake.prefix.path=${VCPKG_INSTALLED_PACKAGES}
-  -Dwindows.cmake.toolchain.file=${CMAKE_TOOLCHAIN_FILE}
-  -Dwindows.cmake.build.type=RelWithDebInfo
-  -Dwindows.build.hdfspp.dll=off
-  -Dwindows.no.sasl=on
-  -Duse.platformToolsetVersion=v142"
-
-## @description  Globals specific to this personality
-## @audience     private
-## @stability    evolving
-function personality_globals
-{
-  # shellcheck disable=SC2034
-  BUILDTOOL=maven
-  #shellcheck disable=SC2034
-  PATCH_BRANCH_DEFAULT=trunk
-  #shellcheck disable=SC2034
-  PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/HADOOP/How+To+Contribute"
-  #shellcheck disable=SC2034
-  JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-  #shellcheck disable=SC2034
-  GITHUB_REPO_DEFAULT="apache/hadoop"
-
-  HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)}
-  if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then
-    HADOOP_HOMEBREW_DIR=/usr/local
-  fi
-}
-
-function personality_parse_args
-{
-  declare i
-
-  for i in "$@"; do
-    case ${i} in
-      --hadoop-isal-prefix=*)
-        delete_parameter "${i}"
-        ISAL_HOME=${i#*=}
-      ;;
-      --hadoop-openssl-prefix=*)
-        delete_parameter "${i}"
-        OPENSSL_HOME=${i#*=}
-      ;;
-      --hadoop-snappy-prefix=*)
-        delete_parameter "${i}"
-        SNAPPY_HOME=${i#*=}
-      ;;
-    esac
-  done
-}
-
-## @description  Calculate the actual module ordering
-## @audience     private
-## @stability    evolving
-## @param        ordering
-function hadoop_order
-{
-  declare ordering=$1
-  declare hadoopm
-
-  if [[ ${ordering} = normal ]]; then
-    hadoopm="${CHANGED_MODULES[*]}"
-  elif [[ ${ordering} = union ]]; then
-    hadoopm="${CHANGED_UNION_MODULES}"
-  elif [[ ${ordering} = mvnsrc ]]; then
-    hadoopm="${MAVEN_SRC_MODULES[*]}"
-  elif [[ ${ordering} = mvnsrctest ]]; then
-    hadoopm="${MAVEN_SRCTEST_MODULES[*]}"
-  else
-    hadoopm="${ordering}"
-  fi
-  echo "${hadoopm}"
-}
-
-## @description  Retrieves the Hadoop project version defined in the root pom.xml
-## @audience     private
-## @stability    evolving
-## @returns      0 on success, 1 on failure
-function load_hadoop_version
-{
-  if [[ -f "${BASEDIR}/pom.xml" ]]; then
-      HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
-          | head -1 \
-          | "${SED}"  -e 's|^ *<version>||' -e 's|</version>.*$||' \
-          | cut -f1 -d- )
-      return 0
-    else
-      return 1
-    fi
-}
-
-## @description  Determine if it is safe to run parallel tests
-## @audience     private
-## @stability    evolving
-## @param        ordering
-function hadoop_test_parallel
-{
-  if load_hadoop_version; then
-    export HADOOP_VERSION
-  else
-    return 1
-  fi
-
-  hmajor=${HADOOP_VERSION%%\.*}
-  hmajorminor=${HADOOP_VERSION%\.*}
-  hminor=${hmajorminor##*\.}
-  # ... and just for reference
-  #hmicro=${HADOOP_VERSION##*\.}
-
-  # Apache Hadoop v2.8.0 was the first one to really
-  # get working parallel unit tests
-  if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then
-    return 1
-  fi
-
-  return 0
-}
-
-## @description  Install extra modules for unit tests
-## @audience     private
-## @stability    evolving
-## @param        ordering
-function hadoop_unittest_prereqs
-{
-  declare input=$1
-  declare mods
-  declare need_common=0
-  declare building_common=0
-  declare module
-  declare flags
-  declare fn
-
-  # prior to running unit tests, hdfs needs libhadoop.so built
-  # if we're building root, then this extra work is moot
-
-  #shellcheck disable=SC2086
-  mods=$(hadoop_order ${input})
-
-  for module in ${mods}; do
-    if [[ ${module} = hadoop-hdfs-project* ]]; then
-      need_common=1
-    elif [[ ${module} = hadoop-common-project/hadoop-common
-      || ${module} = hadoop-common-project ]]; then
-      building_common=1
-    elif [[ ${module} = . ]]; then
-      return
-    fi
-  done
-
-  # Windows builds *ALWAYS* need hadoop-common compiled
-  case ${OSTYPE} in
-    Windows_NT|CYGWIN*|MINGW*|MSYS*)
-      need_common=1
-    ;;
-  esac
-
-  if [[ ${need_common} -eq 1
-      && ${building_common} -eq 0 ]]; then
-    echo "unit test pre-reqs:"
-    module="hadoop-common-project/hadoop-common"
-    fn=$(module_file_fragment "${module}")
-    flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
-    pushd "${BASEDIR}/${module}" >/dev/null || return 1
-    # shellcheck disable=SC2086
-    echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
-      "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
-    popd >/dev/null || return 1
-  fi
-}
-
-## @description  Calculate the flags/settings for yarn-ui v2 build
-## @description  based upon the OS
-## @audience     private
-## @stability    evolving
-function yarn_ui2_flag
-{
-
-  if [[ ${BUILD_NATIVE} != true ]]; then
-    return
-  fi
-
-  # Now it only tested on Linux/OSX, don't enable the profile on
-  # windows until it get verified
-  case ${OSTYPE} in
-    Linux)
-      # shellcheck disable=SC2086
-      echo -Pyarn-ui
-    ;;
-    Darwin)
-      echo -Pyarn-ui
-    ;;
-    *)
-      # Do nothing
-    ;;
-  esac
-}
-
-## @description  Calculate the flags/settings for native code
-## @description  based upon the OS
-## @audience     private
-## @stability    evolving
-function hadoop_native_flags
-{
-  if [[ ${BUILD_NATIVE} != true ]]; then
-    return
-  fi
-
-  declare -a args
-
-  # Based upon HADOOP-11937
-  #
-  # Some notes:
-  #
-  # - getting fuse to compile on anything but Linux
-  #   is always tricky.
-  # - Darwin assumes homebrew is in use.
-  # - HADOOP-12027 required for bzip2 on OS X.
-  # - bzip2 is broken in lots of places
-  #   (the shared library is considered experimental)
-  #   e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
-  #
-
-  args=("-Drequire.test.libhadoop")
-
-  if [[ -d "${ISAL_HOME}/include" ]]; then
-    args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}")
-  fi
-
-  if [[ -d "${OPENSSL_HOME}/include" ]]; then
-    args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}")
-  elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then
-    args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/")
-  fi
-
-  if [[ -d "${SNAPPY_HOME}/include" ]]; then
-    args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}")
-  elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then
-    args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy")
-  fi
-
-  case ${OSTYPE} in
-    Linux)
-      # shellcheck disable=SC2086
-      echo \
-        -Pnative \
-        -Drequire.fuse \
-        -Drequire.openssl \
-        -Drequire.snappy \
-        -Drequire.valgrind \
-        -Drequire.zstd \
-        "${args[@]}"
-    ;;
-    Darwin)
-      echo \
-        "${args[@]}" \
-        -Pnative \
-        -Drequire.snappy  \
-        -Drequire.openssl
-    ;;
-    Windows_NT|CYGWIN*|MINGW*|MSYS*)
-      echo \
-        "${args[@]}" \
-        -Drequire.snappy \
-        -Pdist \
-        -Dtar \
-        "${WINDOWS_FLAGS}"
-    ;;
-    *)
-      echo \
-        "${args[@]}"
-    ;;
-  esac
-}
-
-## @description  Queue up modules for this personality
-## @audience     private
-## @stability    evolving
-## @param        repostatus
-## @param        testtype
-function personality_modules
-{
-  declare repostatus=$1
-  declare testtype=$2
-  declare extra=""
-  declare ordering="normal"
-  declare needflags=false
-  declare foundbats=false
-  declare flags
-  declare fn
-  declare i
-  declare hadoopm
-
-  yetus_debug "Personality: ${repostatus} ${testtype}"
-
-  clear_personality_queue
-
-  case ${testtype} in
-    asflicense)
-      # this is very fast and provides the full path if we do it from
-      # the root of the source
-      personality_enqueue_module .
-      return
-    ;;
-    checkstyle)
-      ordering="union"
-      extra="-DskipTests"
-    ;;
-    compile)
-      ordering="union"
-      extra="-DskipTests"
-      needflags=true
-
-      # if something in common changed, we build the whole world
-      if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then
-        yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
-        ordering="."
-      fi
-    ;;
-    distclean)
-      ordering="."
-      extra="-DskipTests"
-    ;;
-    javadoc)
-      if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
-        ordering=.
-      fi
-
-      if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then
-        echo "javadoc pre-reqs:"
-        for i in hadoop-project \
-          hadoop-common-project/hadoop-annotations; do
-            fn=$(module_file_fragment "${i}")
-            pushd "${BASEDIR}/${i}" >/dev/null || return 1
-            echo "cd ${i}"
-            echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
-              "${MAVEN}" "${MAVEN_ARGS[@]}" install
-            popd >/dev/null || return 1
-        done
-      fi
-      extra="-Pdocs -DskipTests"
-    ;;
-    mvneclipse)
-      if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
-        ordering=.
-      fi
-    ;;
-    mvninstall)
-      extra="-DskipTests"
-      if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then
-        ordering=.
-      fi
-    ;;
-    mvnsite)
-      if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
-        ordering=.
-      fi
-    ;;
-    unit)
-      extra="-Dsurefire.rerunFailingTestsCount=2"
-      if [[ "${BUILDMODE}" = full ]]; then
-        ordering=mvnsrc
-      elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
-        ordering=.
-      fi
-
-      if [[ ${TEST_PARALLEL} = "true" ]] ; then
-        if hadoop_test_parallel; then
-          extra="${extra} -Pparallel-tests"
-          if [[ -n ${TEST_THREADS:-} ]]; then
-            extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
-          fi
-        fi
-      fi
-      needflags=true
-      hadoop_unittest_prereqs "${ordering}"
-
-      if ! verify_needed_test javac; then
-        yetus_debug "hadoop: javac not requested"
-        if ! verify_needed_test native; then
-          yetus_debug "hadoop: native not requested"
-          yetus_debug "hadoop: adding -DskipTests to unit test"
-          extra="-DskipTests"
-        fi
-      fi
-
-      for i in "${CHANGED_FILES[@]}"; do
-        if [[ "${i}" =~ \.bats ]]; then
-          foundbats=true
-        fi
-      done
-
-      if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then
-        yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
-        extra="${extra} -P!shelltest"
-      else
-        extra="${extra} -Pshelltest"
-      fi
-    ;;
-    *)
-      extra="-DskipTests"
-    ;;
-  esac
-
-  if [[ ${needflags} = true ]]; then
-    flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
-    extra="${extra} ${flags}"
-  fi
-
-  if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-    extra="-Ptest-patch -Pdist -Dtar ${WINDOWS_FLAGS} ${extra}"
-  fi
-
-  for module in $(hadoop_order ${ordering}); do
-    # shellcheck disable=SC2086
-    personality_enqueue_module ${module} ${extra}
-  done
-}
-
-## @description  Add tests based upon personality needs
-## @audience     private
-## @stability    evolving
-## @param        filename
-function personality_file_tests
-{
-  declare filename=$1
-
-  yetus_debug "Using Hadoop-specific personality_file_tests"
-
-  if [[ ${filename} =~ src/main/webapp ]]; then
-    yetus_debug "tests/webapp: ${filename}"
-    add_test shadedclient
-  elif [[ ${filename} =~ \.sh
-       || ${filename} =~ \.cmd
-       || ${filename} =~ src/scripts
-       || ${filename} =~ src/test/scripts
-       || ${filename} =~ src/main/bin
-       || ${filename} =~ shellprofile\.d
-       || ${filename} =~ src/main/conf
-       ]]; then
-    yetus_debug "tests/shell: ${filename}"
-    add_test mvnsite
-    add_test unit
-  elif [[ ${filename} =~ \.md$
-       || ${filename} =~ \.md\.vm$
-       || ${filename} =~ src/site
-       ]]; then
-    yetus_debug "tests/site: ${filename}"
-    add_test mvnsite
-  elif [[ ${filename} =~ \.c$
-       || ${filename} =~ \.cc$
-       || ${filename} =~ \.h$
-       || ${filename} =~ \.hh$
-       || ${filename} =~ \.proto$
-       || ${filename} =~ \.cmake$
-       || ${filename} =~ CMakeLists.txt
-       ]]; then
-    yetus_debug "tests/units: ${filename}"
-    add_test compile
-    add_test cc
-    add_test mvnsite
-    add_test javac
-    add_test unit
-  elif [[ ${filename} =~ build.xml$
-       || ${filename} =~ pom.xml$
-       || ${filename} =~ \.java$
-       || ${filename} =~ src/main
-       ]]; then
-      yetus_debug "tests/javadoc+units: ${filename}"
-      add_test compile
-      add_test javac
-      add_test javadoc
-      add_test mvninstall
-      add_test mvnsite
-      add_test unit
-      add_test shadedclient
-  fi
-
-  # if we change anything in here, e.g. the test scripts
-  # then run the client artifact tests
-  if [[ ${filename} =~ hadoop-client-modules ]]; then
-    add_test shadedclient
-  fi
-
-  if [[ ${filename} =~ src/test ]]; then
-    yetus_debug "tests: src/test"
-    add_test unit
-  fi
-
-  if [[ ${filename} =~ \.java$ ]]; then
-    add_test spotbugs
-  fi
-}
-
-## @description  Image to print on success
-## @audience     private
-## @stability    evolving
-function hadoop_console_success
-{
-  printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
-  printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
-  printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
-  printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
-  printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
-  printf "ICB8bXwgICB8bXwgIAo"
-}
-
-###################################################
-# Hadoop project specific check of IT for shaded artifacts
-
-add_test_type shadedclient
-
-## @description check for test modules and add test/plugins as needed
-## @audience private
-## @stability evolving
-function shadedclient_initialize
-{
-  maven_add_install shadedclient
-}
-
-## @description build client facing shaded and non-shaded artifacts and test them
-## @audience private
-## @stability evolving
-## @param repostatus
-function shadedclient_rebuild
-{
-  declare repostatus=$1
-  declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt"
-  declare module
-  declare -a modules=()
-
-  if [[ ${OSTYPE} = Windows_NT ||
-        ${OSTYPE} =~ ^CYGWIN.* ||
-        ${OSTYPE} =~ ^MINGW32.* ||
-        ${OSTYPE} =~ ^MSYS.* ]]; then
-    echo "hadoop personality: building on windows, skipping check of client artifacts."
-    return 0
-  fi
-
-  yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
-  for module in hadoop-client-modules/hadoop-client-check-invariants \
-                hadoop-client-modules/hadoop-client-check-test-invariants \
-                hadoop-client-modules/hadoop-client-integration-tests; do
-    if [ -d "${module}" ]; then
-      yetus_debug "hadoop personality: test module '${module}' is present."
-      modules+=(-pl "${module}")
-    fi
-  done
-  if [ ${#modules[@]} -eq 0 ]; then
-    echo "hadoop personality: no test modules present, skipping check of client artifacts."
-    return 0
-  fi
-
-  big_console_header "Checking client artifacts on ${repostatus} with shaded clients"
-
-  extra="-Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dspotbugs.skip=true"
-
-  if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-    if load_hadoop_version; then
-      export HADOOP_HOME="${SOURCEDIR}/hadoop-dist/target/hadoop-${HADOOP_VERSION}-SNAPSHOT"
-    else
-      yetus_error "[WARNING] Unable to extract the Hadoop version and thus HADOOP_HOME is not set. Some tests may fail."
-    fi
-
-    extra="${WINDOWS_FLAGS} ${extra}"
-  fi
-
-  echo_and_redirect "${logfile}" \
-    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am "${modules[@]}" "${extra}"
-
-  big_console_header "Checking client artifacts on ${repostatus} with non-shaded clients"
-
-  echo_and_redirect "${logfile}" \
-    "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
-      "${modules[@]}" \
-      -DskipShade -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true \
-      -Dspotbugs.skip=true "${extra}"
-
-  count=$("${GREP}" -c '\[ERROR\]' "${logfile}")
-  if [[ ${count} -gt 0 ]]; then
-    add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."
-    return 1
-  fi
-
-  add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts."
-  return 0
-}

dev-support/bin/test-patch

@@ -15,4 +15,4 @@
 # limitations under the License.
 
 BINDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-0}")" >/dev/null && pwd -P)
-exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dirs=dev-support "$@"
+exec "${BINDIR}/yetus-wrapper" test-patch --project=hadoop --skip-dir=dev-support "$@"

dev-support/bin/yetus-wrapper

@@ -77,7 +77,7 @@ WANTED="$1"
 shift
 ARGV=("$@")
 
-HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.14.0}
+HADOOP_YETUS_VERSION=${HADOOP_YETUS_VERSION:-0.8.0}
 BIN=$(yetus_abs "${BASH_SOURCE-$0}")
 BINDIR=$(dirname "${BIN}")
 
@@ -144,7 +144,7 @@ else
   exit 1
 fi
 
-if [[ -n "${GPGBIN}" && ! "${HADOOP_SKIP_YETUS_VERIFICATION}" = true ]]; then
+if [[ -n "${GPGBIN}" ]]; then
   if ! mkdir -p .gpg; then
     yetus_error "ERROR: yetus-dl: Unable to create ${HADOOP_PATCHPROCESS}/.gpg"
     exit 1
@@ -171,17 +171,7 @@ if [[ -n "${GPGBIN}" && ! "${HADOOP_SKIP_YETUS_VERIFICATION}" = true ]]; then
    fi
 fi
 
-if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-  gunzip -c "${TARBALL}.gz" | tar xpf -
-
-  # One of the entries in the Yetus tarball unzips a symlink qbt.sh.
-  # The symlink creation fails on Windows, unless this CI is run as Admin or Developer mode is
-  # enabled.
-  # Thus, we create the qbt.sh symlink ourselves and move it to the target.
-  YETUS_PRECOMMIT_DIR="${YETUS_PREFIX}-${HADOOP_YETUS_VERSION}/lib/precommit"
-  ln -s "${YETUS_PRECOMMIT_DIR}/test-patch.sh" qbt.sh
-  mv qbt.sh "${YETUS_PRECOMMIT_DIR}"
-elif ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then
+if ! (gunzip -c "${TARBALL}.gz" | tar xpf -); then
   yetus_error "ERROR: ${TARBALL}.gz is corrupt. Investigate and then remove ${HADOOP_PATCHPROCESS} to try again."
   exit 1
 fi

dev-support/code-formatter/hadoop_idea_formatter.xml

@@ -1,75 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<code_scheme name="Hadoop" version="173">
-  <option name="RIGHT_MARGIN" value="100" />
-  <JavaCodeStyleSettings>
-    <option name="CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
-    <option name="NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND" value="999" />
-    <option name="PACKAGES_TO_USE_IMPORT_ON_DEMAND">
-      <value />
-    </option>
-    <option name="IMPORT_LAYOUT_TABLE">
-      <value>
-        <package name="java" withSubpackages="true" static="false" />
-        <package name="javax" withSubpackages="true" static="false" />
-        <emptyLine />
-        <package name="org.apache.hadoop.thirdparty" withSubpackages="true" static="false" />
-        <package name="" withSubpackages="true" static="false" />
-        <emptyLine />
-        <package name="org.apache" withSubpackages="true" static="false" />
-        <emptyLine />
-        <package name="" withSubpackages="true" static="true" />
-      </value>
-    </option>
-  </JavaCodeStyleSettings>
-  <codeStyleSettings language="JAVA">
-    <option name="RIGHT_MARGIN" value="100" />
-    <option name="KEEP_LINE_BREAKS" value="false" />
-    <option name="KEEP_FIRST_COLUMN_COMMENT" value="false" />
-    <option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="false" />
-    <option name="KEEP_BLANK_LINES_IN_DECLARATIONS" value="1" />
-    <option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
-    <option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="1" />
-    <option name="INDENT_CASE_FROM_SWITCH" value="false" />
-    <option name="ALIGN_MULTILINE_PARAMETERS" value="false" />
-    <option name="ALIGN_MULTILINE_RESOURCES" value="false" />
-    <option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="true" />
-    <option name="CALL_PARAMETERS_WRAP" value="1" />
-    <option name="METHOD_PARAMETERS_WRAP" value="1" />
-    <option name="RESOURCE_LIST_WRAP" value="5" />
-    <option name="EXTENDS_LIST_WRAP" value="1" />
-    <option name="THROWS_LIST_WRAP" value="1" />
-    <option name="EXTENDS_KEYWORD_WRAP" value="1" />
-    <option name="THROWS_KEYWORD_WRAP" value="1" />
-    <option name="METHOD_CALL_CHAIN_WRAP" value="1" />
-    <option name="BINARY_OPERATION_WRAP" value="1" />
-    <option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="true" />
-    <option name="TERNARY_OPERATION_WRAP" value="5" />
-    <option name="ARRAY_INITIALIZER_WRAP" value="1" />
-    <option name="ASSIGNMENT_WRAP" value="1" />
-    <option name="METHOD_ANNOTATION_WRAP" value="2" />
-    <option name="CLASS_ANNOTATION_WRAP" value="2" />
-    <option name="FIELD_ANNOTATION_WRAP" value="2" />
-    <option name="VARIABLE_ANNOTATION_WRAP" value="2" />
-    <indentOptions>
-      <option name="INDENT_SIZE" value="2" />
-      <option name="CONTINUATION_INDENT_SIZE" value="4" />
-      <option name="TAB_SIZE" value="2" />
-    </indentOptions>
-  </codeStyleSettings>
-</code_scheme>

dev-support/determine-flaky-tests-hadoop.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Given a jenkins test job, this script examines all runs of the job done
+# within specified period of time (number of days prior to the execution
+# time of this script), and reports all failed tests.
+#
+# The output of this script includes a section for each run that has failed
+# tests, with each failed test name listed.
+#
+# More importantly, at the end, it outputs a summary section to list all failed
+# tests within all examined runs, and indicate how many runs a same test
+# failed, and sorted all failed tests by how many runs each test failed.
+#
+# This way, when we see failed tests in PreCommit build, we can quickly tell
+# whether a failed test is a new failure, or it failed before and how often it
+# failed, so to have idea whether it may just be a flaky test.
+#
+# Of course, to be 100% sure about the reason of a test failure, closer look
+# at the failed test for the specific run is necessary.
+#
+import sys
+import platform
+sysversion = sys.hexversion
+onward30 = False
+if sysversion < 0x020600F0:
+  sys.exit("Minimum supported python version is 2.6, the current version is " +
+      "Python" + platform.python_version())
+
+if sysversion == 0x030000F0:
+  sys.exit("There is a known bug with Python" + platform.python_version() +
+      ", please try a different version");
+
+if sysversion < 0x03000000:
+  import urllib2
+else:
+  onward30 = True
+  import urllib.request
+
+import datetime
+import json as simplejson
+import logging
+from optparse import OptionParser
+import time
+
+# Configuration
+DEFAULT_JENKINS_URL = "https://builds.apache.org"
+DEFAULT_JOB_NAME = "Hadoop-Common-trunk"
+DEFAULT_NUM_PREVIOUS_DAYS = 14
+DEFAULT_TOP_NUM_FAILED_TEST = -1
+
+SECONDS_PER_DAY = 86400
+
+# total number of runs to examine
+numRunsToExamine = 0
+
+#summary mode
+summary_mode = False
+
+#total number of errors
+error_count = 0
+
+""" Parse arguments """
+def parse_args():
+  parser = OptionParser()
+  parser.add_option("-J", "--jenkins-url", type="string",
+                    dest="jenkins_url", help="Jenkins URL",
+                    default=DEFAULT_JENKINS_URL)
+  parser.add_option("-j", "--job-name", type="string",
+                    dest="job_name", help="Job name to look at",
+                    default=DEFAULT_JOB_NAME)
+  parser.add_option("-n", "--num-days", type="int",
+                    dest="num_prev_days", help="Number of days to examine",
+                    default=DEFAULT_NUM_PREVIOUS_DAYS)
+  parser.add_option("-t", "--top", type="int",
+                    dest="num_failed_tests",
+                    help="Summary Mode, only show top number of failed tests",
+                    default=DEFAULT_TOP_NUM_FAILED_TEST)
+
+  (options, args) = parser.parse_args()
+  if args:
+    parser.error("unexpected arguments: " + repr(args))
+  return options
+
+""" Load data from specified url """
+def load_url_data(url):
+  if onward30:
+    ourl = urllib.request.urlopen(url)
+    codec = ourl.info().get_param('charset')
+    content = ourl.read().decode(codec)
+    data = simplejson.loads(content, strict=False)
+  else:
+    ourl = urllib2.urlopen(url)
+    data = simplejson.load(ourl, strict=False)
+  return data
+ 
+""" List all builds of the target project. """
+def list_builds(jenkins_url, job_name):
+  global summary_mode
+  url = "%(jenkins)s/job/%(job_name)s/api/json?tree=builds[url,result,timestamp]" % dict(
+      jenkins=jenkins_url,
+      job_name=job_name)
+
+  try:
+    data = load_url_data(url)
+
+  except:
+    if not summary_mode:
+      logging.error("Could not fetch: %s" % url)
+    error_count += 1
+    raise
+  return data['builds']
+
+""" Find the names of any tests which failed in the given build output URL. """
+def find_failing_tests(testReportApiJson, jobConsoleOutput):
+  global summary_mode
+  global error_count
+  ret = set()
+  try:
+    data = load_url_data(testReportApiJson)
+
+  except:
+    if not summary_mode:
+      logging.error("    Could not open testReport, check " +
+        jobConsoleOutput + " for why it was reported failed")
+    error_count += 1
+    return ret
+
+  for suite in data['suites']:
+    for cs in suite['cases']:
+      status = cs['status']
+      errDetails = cs['errorDetails']
+      if (status == 'REGRESSION' or status == 'FAILED' or (errDetails is not None)):
+        ret.add(cs['className'] + "." + cs['name'])
+
+  if len(ret) == 0 and (not summary_mode):
+    logging.info("    No failed tests in testReport, check " +
+        jobConsoleOutput + " for why it was reported failed.")
+  return ret
+
+""" Iterate runs of specfied job within num_prev_days and collect results """
+def find_flaky_tests(jenkins_url, job_name, num_prev_days):
+  global numRunsToExamine
+  global summary_mode
+  all_failing = dict()
+  # First list all builds
+  builds = list_builds(jenkins_url, job_name)
+
+  # Select only those in the last N days
+  min_time = int(time.time()) - SECONDS_PER_DAY * num_prev_days
+  builds = [b for b in builds if (int(b['timestamp']) / 1000) > min_time]
+
+  # Filter out only those that failed
+  failing_build_urls = [(b['url'] , b['timestamp']) for b in builds
+      if (b['result'] in ('UNSTABLE', 'FAILURE'))]
+
+  tnum = len(builds)
+  num = len(failing_build_urls)
+  numRunsToExamine = tnum
+  if not summary_mode:
+    logging.info("    THERE ARE " + str(num) + " builds (out of " + str(tnum)
+      + ") that have failed tests in the past " + str(num_prev_days) + " days"
+      + ((".", ", as listed below:\n")[num > 0]))
+
+  for failed_build_with_time in failing_build_urls:
+    failed_build = failed_build_with_time[0];
+    jobConsoleOutput = failed_build + "Console";
+    testReport = failed_build + "testReport";
+    testReportApiJson = testReport + "/api/json";
+
+    ts = float(failed_build_with_time[1]) / 1000.
+    st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
+    if not summary_mode:
+      logging.info("===>%s" % str(testReport) + " (" + st + ")")
+    failing = find_failing_tests(testReportApiJson, jobConsoleOutput)
+    if failing:
+      for ftest in failing:
+        if not summary_mode:
+          logging.info("    Failed test: %s" % ftest)
+        all_failing[ftest] = all_failing.get(ftest,0)+1
+
+  return all_failing
+
+def main():
+  global numRunsToExamine
+  global summary_mode
+  logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
+
+  # set up logger to write to stdout
+  soh = logging.StreamHandler(sys.stdout)
+  soh.setLevel(logging.INFO)
+  logger = logging.getLogger()
+  logger.removeHandler(logger.handlers[0])
+  logger.addHandler(soh)
+
+  opts = parse_args()
+  logging.info("****Recently FAILED builds in url: " + opts.jenkins_url
+      + "/job/" + opts.job_name + "")
+
+  if opts.num_failed_tests != -1:
+    summary_mode = True
+
+  all_failing = find_flaky_tests(opts.jenkins_url, opts.job_name,
+      opts.num_prev_days)
+  if len(all_failing) == 0:
+    raise SystemExit(0)
+
+  if summary_mode and opts.num_failed_tests < len(all_failing):
+    logging.info("\nAmong " + str(numRunsToExamine) +
+                 " runs examined, top " + str(opts.num_failed_tests) +
+                 " failed tests <#failedRuns: testName>:")
+  else:
+      logging.info("\nAmong " + str(numRunsToExamine) +
+                   " runs examined, all failed tests <#failedRuns: testName>:")
+
+  # print summary section: all failed tests sorted by how many times they failed
+  line_count = 0
+  for tn in sorted(all_failing, key=all_failing.get, reverse=True):
+    logging.info("    " + str(all_failing[tn])+ ": " + tn)
+    if summary_mode:
+      line_count += 1
+      if line_count == opts.num_failed_tests:
+        break
+
+  if summary_mode and error_count > 0:
+    logging.info("\n" + str(error_count) + " errors found, you may "
+                 + "re-run in non summary mode to see error details.");
+
+if __name__ == "__main__":
+  main()

dev-support/docker/Dockerfile

@@ -1,3 +1,4 @@
+
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -17,7 +18,7 @@
 # Dockerfile for installing the necessary dependencies for building Hadoop.
 # See BUILDING.txt.
 
-FROM ubuntu:focal
+FROM ubuntu:xenial
 
 WORKDIR /root
 
@@ -33,61 +34,172 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
 ######
-# Platform package dependency resolver
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
+#
+# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
+# Ubuntu Java.  See Java section below!
 ######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
-    && chmod a+r pkg-resolver/*.json
-
-######
-# Install packages from apt
-######
-# hadolint ignore=DL3008,SC2046
+# hadolint ignore=DL3008
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends python3 \
-    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
+    && apt-get -q install -y --no-install-recommends \
+        apt-utils \
+        build-essential \
+        bzip2 \
+        clang \
+        curl \
+        doxygen \
+        fuse \
+        g++ \
+        gcc \
+        git \
+        gnupg-agent \
+        libbz2-dev \
+        libcurl4-openssl-dev \
+        libfuse-dev \
+        libprotobuf-dev \
+        libprotoc-dev \
+        libsasl2-dev \
+        libsnappy-dev \
+        libssl-dev \
+        libtool \
+        libzstd1-dev \
+        locales \
+        make \
+        pinentry-curses \
+        pkg-config \
+        python \
+        python2.7 \
+        python-pip \
+        python-pkg-resources \
+        python-setuptools \
+        python-wheel \
+        rsync \
+        software-properties-common \
+        snappy \
+        sudo \
+        valgrind \
+        zlib1g-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+#######
+# OpenJDK 8
+#######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV PYTHONIOENCODING=utf-8
 
 ######
-# Set env vars required to build Hadoop
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
 ######
-ENV MAVEN_HOME /usr
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+RUN mkdir -p /opt/cmake \
+    && curl -L -s -S \
+      https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
+      -o /opt/cmake.tar.gz \
+    && tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
+ENV CMAKE_HOME /opt/cmake
+ENV PATH "${PATH}:/opt/cmake/bin"
 
-#######
-# Set env vars for SpotBugs 4.2.2
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf 3.7.1
-#######
+######
+# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/protobuf-src \
+    && curl -L -s -S \
+      https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+      -o /opt/protobuf.tar.gz \
+    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+    && cd /opt/protobuf-src \
+    && ./configure --prefix=/opt/protobuf \
+    && make install \
+    && cd /root \
+    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends maven \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+ENV MAVEN_HOME /usr
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+# Ant is needed for findbugs
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends findbugs ant \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+ENV FINDBUGS_HOME /usr
+
+####
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
+####
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 \
+    && apt-get -q update \
+    && apt-get -q install -y --no-install-recommends shellcheck \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+####
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
+####
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+    && apt-get -q install -y --no-install-recommends bats \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+####
+# Install pylint at fixed version (2.0.0 removed python2 support)
+# https://github.com/PyCQA/pylint/issues/2294
+####
+RUN pip2 install pylint==1.9.2
+
+####
+# Install dateutil.parser
+####
+RUN pip2 install python-dateutil==2.7.3
+
+###
+# Install node.js for web UI framework (4.2.6 ships with Xenial)
+###
+# hadolint ignore=DL3008, DL3016
+RUN apt-get -q update \
+    && apt-get install -y --no-install-recommends nodejs npm \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
+    && ln -s /usr/bin/nodejs /usr/bin/node \
+    && npm install npm@latest -g \
+    && npm install -g jshint
+
+###
+# Install hadolint
+####
+RUN curl -L -s -S \
+        https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
+        -o /bin/hadolint \
+   && chmod a+rx /bin/hadolint \
+   && shasum -a 512 /bin/hadolint | \
+        awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
+
 ###
 # Avoid out of memory errors in builds
 ###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
+ENV MAVEN_OPTS -Xms256m -Xmx1536m
 
-# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
-
-####
-# Install packages
-####
-RUN pkg-resolver/install-common-pkgs.sh
-RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
-RUN pkg-resolver/install-boost.sh ubuntu:focal
-RUN pkg-resolver/install-protobuf.sh ubuntu:focal
-RUN pkg-resolver/install-hadolint.sh ubuntu:focal
-RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
 
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
@@ -95,6 +207,11 @@ RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
 # YETUS CUT HERE
 ###
 
+# Hugo static website generator (for new hadoop site and Ozone docs)
+RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
+    && dpkg --install hugo.deb \
+    && rm hugo.deb
+
 # Add a welcome message and environment checks.
 COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
 RUN chmod 755 /root/hadoop_env_checks.sh

dev-support/docker/Dockerfile_aarch64

@@ -1,103 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM ubuntu:focal
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-#####
-# Disable suggests/recommends
-#####
-RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
-RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
-    && chmod a+r pkg-resolver/*.json
-
-######
-# Install packages from apt
-######
-# hadolint ignore=DL3008,SC2046
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends python3 \
-    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV PYTHONIOENCODING=utf-8
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /usr
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
-
-#######
-# Set env vars for SpotBugs 4.2.2
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf 3.7.1
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
-
-# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
-
-# Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
-ENV QT_QPA_PLATFORM offscreen
-
-####
-# Install packages
-####
-RUN pkg-resolver/install-common-pkgs.sh
-RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
-RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
-RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
-
-###
-# Everything past this point is either not needed for testing or breaks Yetus.
-# So tell Yetus not to read the rest of the file:
-# YETUS CUT HERE
-###
-
-# Add a welcome message and environment checks.
-COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
-RUN chmod 755 /root/hadoop_env_checks.sh
-# hadolint ignore=SC2016
-RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

dev-support/docker/Dockerfile_centos_7

@@ -1,96 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM centos:7
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
-    && chmod a+r pkg-resolver/*.json
-
-######
-# Install packages from yum
-######
-# hadolint ignore=DL3008,SC2046
-RUN yum update -y \
-    && yum groupinstall -y "Development Tools" \
-    && yum install -y \
-        centos-release-scl \
-        python3 \
-    && yum install -y $(pkg-resolver/resolve.py centos:7)
-
-# Set GCC 9 as the default C/C++ compiler
-RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
-SHELL ["/bin/bash", "--login", "-c"]
-
-######
-# Set the environment variables needed for CMake
-# to find and use GCC 9 for compilation
-######
-ENV GCC_HOME "/opt/rh/devtoolset-9"
-ENV CC "${GCC_HOME}/root/usr/bin/gcc"
-ENV CXX "${GCC_HOME}/root/usr/bin/g++"
-ENV SHLVL 1
-ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
-ENV PCP_DIR "${GCC_HOME}/root"
-ENV MANPATH "${GCC_HOME}/root/usr/share/man:"
-ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
-ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
-
-# TODO: Set locale
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:${MAVEN_HOME}/bin"
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
-
-#######
-# Set env vars for SpotBugs
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-######
-# Install packages
-######
-RUN pkg-resolver/install-maven.sh centos:7
-RUN pkg-resolver/install-cmake.sh centos:7
-RUN pkg-resolver/install-zstandard.sh centos:7
-RUN pkg-resolver/install-yasm.sh centos:7
-RUN pkg-resolver/install-protobuf.sh centos:7
-RUN pkg-resolver/install-boost.sh centos:7
-RUN pkg-resolver/install-spotbugs.sh centos:7
-RUN pkg-resolver/install-nodejs.sh centos:7
-RUN pkg-resolver/install-git.sh centos:7
-RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/Dockerfile_centos_8

@@ -1,118 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM centos:8
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
-    && chmod a+r pkg-resolver/*.json
-
-######
-# Centos 8 has reached its EOL and the packages
-# are no longer available on mirror.centos.org site.
-# Please see https://www.centos.org/centos-linux-eol/
-######
-RUN pkg-resolver/set-vault-as-baseurl-centos.sh centos:8
-
-######
-# Install packages from yum
-######
-# hadolint ignore=DL3008,SC2046
-RUN yum update -y \
-    && yum install -y python3 \
-    && yum install -y $(pkg-resolver/resolve.py centos:8)
-
-####
-# Install EPEL
-####
-RUN pkg-resolver/install-epel.sh centos:8
-
-RUN dnf --enablerepo=powertools install -y \
-    doxygen \
-    snappy-devel \
-    yasm
-
-RUN dnf install -y \
-    bouncycastle \
-    gcc-toolset-9-gcc \
-    gcc-toolset-9-gcc-c++ \
-    libpmem-devel
-
-# Set GCC 9 as the default C/C++ compiler
-RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc
-SHELL ["/bin/bash", "--login", "-c"]
-
-######
-# Set the environment variables needed for CMake
-# to find and use GCC 9 for compilation
-######
-ENV GCC_HOME "/opt/rh/gcc-toolset-9"
-ENV CC "${GCC_HOME}/root/usr/bin/gcc"
-ENV CXX "${GCC_HOME}/root/usr/bin/g++"
-ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
-ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
-ENV SHLVL 1
-ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
-ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
-ENV MODULESHOME "/usr/share/Modules"
-ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
-ENV PCP_DIR "${GCC_HOME}/root"
-ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
-ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
-ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
-
-# TODO: Set locale
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /opt/maven
-ENV PATH "${PATH}:${MAVEN_HOME}/bin"
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
-
-#######
-# Set env vars for SpotBugs
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-######
-# Install packages
-######
-RUN pkg-resolver/install-maven.sh centos:8
-RUN pkg-resolver/install-cmake.sh centos:8
-RUN pkg-resolver/install-boost.sh centos:8
-RUN pkg-resolver/install-spotbugs.sh centos:8
-RUN pkg-resolver/install-protobuf.sh centos:8
-RUN pkg-resolver/install-zstandard.sh centos:8
-RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/Dockerfile_debian_10

@@ -1,102 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM debian:10
-
-WORKDIR /root
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-#####
-# Disable suggests/recommends
-#####
-RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
-RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV DEBCONF_TERSE true
-
-######
-# Platform package dependency resolver
-######
-COPY pkg-resolver pkg-resolver
-RUN chmod a+x pkg-resolver/install-pkg-resolver.sh
-RUN pkg-resolver/install-pkg-resolver.sh debian:10
-
-######
-# Install packages from apt
-######
-# hadolint ignore=DL3008,SC2046
-RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py debian:10) \
-    && echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list \
-    && apt-get -q update \
-    && apt-get -q install -y --no-install-recommends -t bullseye $(pkg-resolver/resolve.py --release=bullseye debian:10) \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-# TODO : Set locale
-
-######
-# Set env vars required to build Hadoop
-######
-ENV MAVEN_HOME /usr
-# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
-ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
-
-#######
-# Set env vars for SpotBugs 4.2.2
-#######
-ENV SPOTBUGS_HOME /opt/spotbugs
-
-#######
-# Set env vars for Google Protobuf 3.7.1
-#######
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
-
-###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx3072m
-
-# Skip gpg verification when downloading Yetus via yetus-wrapper
-ENV HADOOP_SKIP_YETUS_VERIFICATION true
-
-####
-# Install packages
-####
-RUN pkg-resolver/install-cmake.sh debian:10
-RUN pkg-resolver/install-spotbugs.sh debian:10
-RUN pkg-resolver/install-boost.sh debian:10
-RUN pkg-resolver/install-protobuf.sh debian:10
-RUN pkg-resolver/install-hadolint.sh debian:10
-RUN pkg-resolver/install-intel-isa-l.sh debian:10
-
-###
-# Everything past this point is either not needed for testing or breaks Yetus.
-# So tell Yetus not to read the rest of the file:
-# YETUS CUT HERE
-###
-
-# Add a welcome message and environment checks.
-COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
-RUN chmod 755 /root/hadoop_env_checks.sh
-# hadolint ignore=SC2016
-RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc

dev-support/docker/Dockerfile_windows_10

@@ -1,124 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
-
-FROM mcr.microsoft.com/windows:ltsc2019
-
-# Need to disable the progress bar for speeding up the downloads.
-# hadolint ignore=SC2086
-RUN powershell $Global:ProgressPreference = 'SilentlyContinue'
-
-# Restore the default Windows shell for correct batch processing.
-SHELL ["cmd", "/S", "/C"]
-
-# Install Visual Studio 2019 Build Tools.
-RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe \
-    && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache \
-    --installPath "%ProgramFiles(x86)%\Microsoft Visual Studio\2019\BuildTools" \
-    --add Microsoft.VisualStudio.Workload.VCTools \
-    --add Microsoft.VisualStudio.Component.VC.ASAN \
-    --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \
-    --add Microsoft.VisualStudio.Component.Windows10SDK.19041 \
-    || IF "%ERRORLEVEL%"=="3010" EXIT 0) \
-    && del /q vs_buildtools.exe
-
-# Install Chocolatey.
-RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
-RUN setx PATH "%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
-
-# Install git.
-RUN choco install git.install -y
-RUN powershell Copy-Item -Recurse -Path 'C:\Program Files\Git' -Destination C:\Git
-
-# Install vcpkg.
-# hadolint ignore=DL3003
-RUN powershell git clone https://github.com/microsoft/vcpkg.git \
-    && cd vcpkg \
-    && git checkout 7ffa425e1db8b0c3edf9c50f2f3a0f25a324541d \
-    && .\bootstrap-vcpkg.bat
-RUN powershell .\vcpkg\vcpkg.exe install boost:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install protobuf:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install openssl:x64-windows
-RUN powershell .\vcpkg\vcpkg.exe install zlib:x64-windows
-ENV PROTOBUF_HOME "C:\vcpkg\installed\x64-windows"
-
-# Install Azul Java 8 JDK.
-RUN powershell Invoke-WebRequest -URI https://cdn.azul.com/zulu/bin/zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -OutFile $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\zulu8.62.0.19-ca-jdk8.0.332-win_x64.zip -DestinationPath "C:\Java"
-ENV JAVA_HOME "C:\Java\zulu8.62.0.19-ca-jdk8.0.332-win_x64"
-RUN setx PATH "%PATH%;%JAVA_HOME%\bin"
-
-# Install Apache Maven.
-RUN powershell Invoke-WebRequest -URI https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/apache-maven-3.8.6-bin.zip -OutFile $Env:TEMP\apache-maven-3.8.6-bin.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\apache-maven-3.8.6-bin.zip -DestinationPath "C:\Maven"
-RUN setx PATH "%PATH%;C:\Maven\apache-maven-3.8.6\bin"
-ENV MAVEN_OPTS '-Xmx2048M -Xss128M'
-
-# Install CMake 3.19.0.
-RUN powershell Invoke-WebRequest -URI https://cmake.org/files/v3.19/cmake-3.19.0-win64-x64.zip -OutFile $Env:TEMP\cmake-3.19.0-win64-x64.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\cmake-3.19.0-win64-x64.zip -DestinationPath "C:\CMake"
-RUN setx PATH "%PATH%;C:\CMake\cmake-3.19.0-win64-x64\bin"
-
-# Install zstd 1.5.4.
-RUN powershell Invoke-WebRequest -Uri https://github.com/facebook/zstd/releases/download/v1.5.4/zstd-v1.5.4-win64.zip -OutFile $Env:TEMP\zstd-v1.5.4-win64.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\zstd-v1.5.4-win64.zip -DestinationPath "C:\ZStd"
-RUN setx PATH "%PATH%;C:\ZStd"
-
-# Install libopenssl 3.1.0 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libopenssl-3.1.0-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar.zst -o $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar
-RUN powershell mkdir "C:\LibOpenSSL"
-RUN powershell tar -xvf $Env:TEMP\libopenssl-3.1.0-1-x86_64.pkg.tar -C "C:\LibOpenSSL"
-
-# Install libxxhash 0.8.1 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libxxhash-0.8.1-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar.zst -o $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar
-RUN powershell mkdir "C:\LibXXHash"
-RUN powershell tar -xvf $Env:TEMP\libxxhash-0.8.1-1-x86_64.pkg.tar -C "C:\LibXXHash"
-
-# Install libzstd 1.5.4 needed for rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/libzstd-1.5.4-1-x86_64.pkg.tar.zst -OutFile $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar.zst -o $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar
-RUN powershell mkdir "C:\LibZStd"
-RUN powershell tar -xvf $Env:TEMP\libzstd-1.5.4-1-x86_64.pkg.tar -C "C:\LibZStd"
-
-# Install rsync 3.2.7.
-RUN powershell Invoke-WebRequest -Uri https://repo.msys2.org/msys/x86_64/rsync-3.2.7-2-x86_64.pkg.tar.zst -OutFile $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst
-RUN powershell zstd -d $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar.zst -o $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar
-RUN powershell mkdir "C:\RSync"
-RUN powershell tar -xvf $Env:TEMP\rsync-3.2.7-2-x86_64.pkg.tar -C "C:\RSync"
-# Copy the dependencies of rsync 3.2.7.
-RUN powershell Copy-Item -Path "C:\LibOpenSSL\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
-RUN powershell Copy-Item -Path "C:\LibXXHash\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
-RUN powershell Copy-Item -Path "C:\LibZStd\usr\bin\*.dll" -Destination "C:\Program` Files\Git\usr\bin"
-RUN powershell Copy-Item -Path "C:\RSync\usr\bin\*" -Destination "C:\Program` Files\Git\usr\bin"
-
-# Install Python 3.10.11.
-RUN powershell Invoke-WebRequest -Uri https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip -OutFile $Env:TEMP\python-3.10.11-embed-amd64.zip
-RUN powershell Expand-Archive -Path $Env:TEMP\python-3.10.11-embed-amd64.zip -DestinationPath "C:\Python3"
-RUN powershell New-Item -ItemType HardLink -Value "C:\Python3\python.exe" -Path "C:\Python3\python3.exe"
-RUN setx path "%PATH%;C:\Python3"
-
-# We get strange Javadoc errors without this.
-RUN setx classpath ""
-
-RUN git config --global core.longpaths true
-RUN setx PATH "%PATH%;C:\Program Files\Git\usr\bin"
-
-# Define the entry point for the docker container.
-ENTRYPOINT ["C:\\Program Files (x86)\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvars64.bat", "&&", "cmd.exe"]

dev-support/docker/README.md

@@ -1,114 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-# Docker images for building Hadoop
-
-This folder contains the Dockerfiles for building Hadoop on various platforms.
-
-# Dependency management
-
-The mode of installation of the dependencies needed for building Hadoop varies from one platform to
-the other. Different platforms have different toolchains. Some packages tend to be polymorphic
-across platforms and most commonly, a package that's readily available in one platform's toolchain
-isn't available on another. We thus, resort to building and installing the package from source,
-causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all
-the platforms. We need a system to track a dependency - for a package - for a platform
-
-- (and optionally) for a release. Thus, there's a lot of diversity that needs to be handled for
-  managing package dependencies and
-  `pkg-resolver` caters to that.
-
-## Supported platforms
-
-`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management.
-
-## Package dependencies
-
-`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON.
-
-```json
-{
-  "dependency_1": {
-    "platform_1": "package_1",
-    "platform_2": [
-      "package_1",
-      "package_2"
-    ]
-  },
-  "dependency_2": {
-    "platform_1": [
-      "package_1",
-      "package_2",
-      "package_3"
-    ]
-  },
-  "dependency_3": {
-    "platform_1": {
-      "release_1": "package_1_1_1",
-      "release_2": [
-        "package_1_2_1",
-        "package_1_2_2"
-      ]
-    },
-    "platform_2": [
-      "package_2_1",
-      {
-        "release_1": "package_2_1_1"
-      }
-    ]
-  }
-}
-```
-
-The root JSON element contains unique _dependency_ children. This in turn contains the name of the _
-platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of
-how to interpret the above JSON -
-
-1. For `dependency_1`, `package_1` needs to be installed for `platform_1`.
-2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`.
-3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for
-   `platform_1`.
-4. For `dependency_3`, `package_1_1_1` gets installed only if `release_1` has been specified
-   for `platform_1`.
-5. For `dependency_3`, the packages `package_1_2_1` and `package_1_2_2` gets installed only
-   if `release_2` has been specified for `platform_1`.
-6. For `dependency_3`, for `platform_2`, `package_2_1` is always installed, but `package_2_1_1` gets
-   installed only if `release_1` has been specified.
-
-### Tool help
-
-```shell
-$ pkg-resolver/resolve.py -h
-usage: resolve.py [-h] [-r RELEASE] platform
-
-Platform package dependency resolver for building Apache Hadoop
-
-positional arguments:
-  platform              The name of the platform to resolve the dependencies for
-
-optional arguments:
-  -h, --help            show this help message and exit
-  -r RELEASE, --release RELEASE
-                        The release label to filter the packages for the given platform
-```
-
-## Standalone packages
-
-Most commonly, some packages are not available across the toolchains in various platforms. Thus, we
-would need to build and install them. Since we need to do this across all the Dockerfiles for all
-the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put
-the build steps in a `pkg-resolver/install-<package>.sh` and invoke this in all the Dockerfiles.

dev-support/docker/pkg-resolver/check_platform.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Checks whether the given platform is supported for building Apache Hadoop
-"""
-
-import json
-import sys
-
-
-def get_platforms():
-    """
-    :return: A list of the supported platforms managed by pkg-resolver.
-    """
-
-    with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file:
-        return json.loads(platforms_file.read())
-
-
-def is_supported_platform(platform):
-    """
-    :param platform: The name of the platform
-    :return: Whether the platform is supported
-    """
-    return platform in get_platforms()
-
-
-if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1),
-              file=sys.stderr)
-        sys.exit(1)
-
-    sys.exit(0 if is_supported_platform(sys.argv[1]) else 1)

dev-support/docker/pkg-resolver/install-boost.sh

@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="1.72.0"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "1.72.0" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "1.72.0" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /opt/boost-library &&
-    curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 &&
-    mv boost_1_72_0.tar.bz2 /opt/boost-library &&
-    cd /opt/boost-library &&
-    tar --bzip2 -xf boost_1_72_0.tar.bz2 &&
-    cd /opt/boost-library/boost_1_72_0 &&
-    ./bootstrap.sh --prefix=/usr/ &&
-    ./b2 --without-python install &&
-    cd /root &&
-    rm -rf /opt/boost-library
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-cmake.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="3.19.0"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "3.19.0" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "3.19.0" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /tmp/cmake /opt/cmake &&
-    curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz &&
-    tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake &&
-    cd /opt/cmake || exit && ./bootstrap &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root || exit
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-common-pkgs.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-######
-# Install pylint and python-dateutil
-######
-pip3 install pylint==2.6.0 python-dateutil==2.8.1

dev-support/docker/pkg-resolver/install-epel.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="8"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "8" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "8" ]; then
-  mkdir -p /tmp/epel &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
-      -o /tmp/epel/epel-release-latest-8.noarch.rpm &&
-    rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-git.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="2.9.5"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "2.9.5" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "2.9.5" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /tmp/git /opt/git &&
-    curl -L -s -S https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.9.5.tar.gz >/tmp/git/git-2.9.5.tar.gz &&
-    tar xzf /tmp/git/git-2.9.5.tar.gz --strip-components 1 -C /opt/git &&
-    cd /opt/git || exit &&
-    make configure &&
-    ./configure --prefix=/usr/local &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root || exit
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-hadolint.sh

@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-curl -L -s -S \
-  https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-  -o /bin/hadolint &&
-  chmod a+rx /bin/hadolint &&
-  shasum -a 512 /bin/hadolint |
-  awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'

dev-support/docker/pkg-resolver/install-intel-isa-l.sh

@@ -1,58 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="2.29.0"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "2.29.0" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "2.29.0" ]; then
-  # hadolint ignore=DL3003,DL3008
-  mkdir -p /opt/isa-l-src &&
-    curl -L -s -S \
-      https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
-      -o /opt/isa-l.tar.gz &&
-    tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src &&
-    cd /opt/isa-l-src &&
-    ./autogen.sh &&
-    ./configure &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root &&
-    rm -rf /opt/isa-l-src
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-maven.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="3.6.3"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "3.6.3" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "3.6.3" ]; then
-  mkdir -p /opt/maven /tmp/maven &&
-    curl -L -s -S https://dlcdn.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-      -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
-    tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-nodejs.sh

@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="14.16.1"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "14.16.1" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "14.16.1" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /tmp/node &&
-    curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz &&
-    tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node &&
-    cd /tmp/node || exit &&
-    ./configure &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root || exit
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-pkg-resolver.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: No platform specified, please specify one"
-  exit 1
-fi
-
-chmod a+x pkg-resolver/*.sh pkg-resolver/*.py
-chmod a+r pkg-resolver/*.json
-
-if [ "$1" == "debian:10" ]; then
-  apt-get -q update
-  apt-get -q install -y --no-install-recommends python3 \
-    python3-pip \
-    python3-pkg-resources \
-    python3-setuptools \
-    python3-wheel
-  pip3 install pylint==2.6.0 python-dateutil==2.8.1
-else
-  # Need to add the code for the rest of the platforms - HADOOP-17920
-  echo "ERROR: The given platform $1 is not yet supported or is invalid"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="3.7.1"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "3.7.1" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "3.7.1" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /opt/protobuf-src &&
-    curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz &&
-    tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
-    cd /opt/protobuf-src &&
-    ./configure --prefix=/opt/protobuf &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root &&
-    rm -rf /opt/protobuf-src
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-spotbugs.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="4.2.2"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "4.2.2" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "4.2.2" ]; then
-  mkdir -p /opt/spotbugs &&
-    curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz &&
-    tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs &&
-    chmod +x /opt/spotbugs/bin/*
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="1.2.0-4"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "1.2.0-4" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "1.2.0-4" ]; then
-  mkdir -p /tmp/yasm &&
-    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
-      -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
-    rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/install-zstandard.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-pkg-resolver/check_platform.py "$1"
-if [ $? -eq 1 ]; then
-  echo "ERROR: Unsupported platform $1"
-  exit 1
-fi
-
-default_version="1.4.9"
-version_to_install=$default_version
-if [ -n "$2" ]; then
-  version_to_install="$2"
-fi
-
-if [ "$version_to_install" != "1.4.9" ]; then
-  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
-  version_to_install=$default_version
-fi
-
-if [ "$version_to_install" == "1.4.9" ]; then
-  # hadolint ignore=DL3003
-  mkdir -p /opt/zstd /tmp/zstd &&
-    curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz &&
-    tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd &&
-    cd /opt/zstd || exit &&
-    make "-j$(nproc)" &&
-    make install &&
-    cd /root || exit
-else
-  echo "ERROR: Don't know how to install version $version_to_install"
-  exit 1
-fi

dev-support/docker/pkg-resolver/packages.json

@@ -1,360 +0,0 @@
-{
-  "ant": {
-    "debian:10": "ant",
-    "ubuntu:focal": "ant",
-    "ubuntu:focal::arch64": "ant",
-    "centos:7": "ant",
-    "centos:8": "ant"
-  },
-  "apt-utils": {
-    "debian:10": "apt-utils",
-    "ubuntu:focal": "apt-utils",
-    "ubuntu:focal::arch64": "apt-utils"
-  },
-  "automake": {
-    "debian:10": "automake",
-    "ubuntu:focal": "automake",
-    "ubuntu:focal::arch64": "automake",
-    "centos:7": "automake",
-    "centos:8": "automake"
-  },
-  "autoconf": {
-    "centos:7": "autoconf"
-  },
-  "bats": {
-    "debian:10": "bats",
-    "ubuntu:focal": "bats",
-    "ubuntu:focal::arch64": "bats"
-  },
-  "build-essential": {
-    "debian:10": "build-essential",
-    "ubuntu:focal": "build-essential",
-    "ubuntu:focal::arch64": "build-essential",
-    "centos:7": "build-essential"
-  },
-  "bzip2": {
-    "debian:10": [
-      "bzip2",
-      "libbz2-dev"
-    ],
-    "ubuntu:focal": [
-      "bzip2",
-      "libbz2-dev"
-    ],
-    "ubuntu:focal::arch64": [
-      "bzip2",
-      "libbz2-dev"
-    ],
-    "centos:7": [
-      "bzip2",
-      "bzip2-devel"
-    ],
-    "centos:8": [
-      "bzip2",
-      "bzip2-devel"
-    ]
-  },
-  "clang": {
-    "debian:10": "clang",
-    "ubuntu:focal": "clang",
-    "ubuntu:focal::arch64": "clang",
-    "centos:7": "clang",
-    "centos:8": "clang"
-  },
-  "cmake": {
-    "ubuntu:focal": "cmake",
-    "ubuntu:focal::arch64": "cmake"
-  },
-  "curl": {
-    "debian:10": [
-      "curl",
-      "libcurl4-openssl-dev"
-    ],
-    "ubuntu:focal": [
-      "curl",
-      "libcurl4-openssl-dev"
-    ],
-    "ubuntu:focal::arch64": [
-      "curl",
-      "libcurl4-openssl-dev"
-    ],
-    "centos:7": [
-      "curl",
-      "libcurl-devel"
-    ],
-    "centos:8": [
-      "curl",
-      "libcurl-devel"
-    ]
-  },
-  "doxygen": {
-    "debian:10": "doxygen",
-    "ubuntu:focal": "doxygen",
-    "ubuntu:focal::arch64": "doxygen",
-    "centos:7": "doxygen"
-  },
-  "dnf": {
-    "centos:8": "dnf"
-  },
-  "fuse": {
-    "debian:10": [
-      "fuse",
-      "libfuse-dev"
-    ],
-    "ubuntu:focal": [
-      "fuse",
-      "libfuse-dev"
-    ],
-    "ubuntu:focal::arch64": [
-      "fuse",
-      "libfuse-dev"
-    ],
-    "centos:7": [
-      "fuse",
-      "fuse-libs",
-      "fuse-devel"
-    ],
-    "centos:8": [
-      "fuse",
-      "fuse-libs",
-      "fuse-devel"
-    ]
-  },
-  "gcc": {
-    "debian:10": {
-      "bullseye": [
-        "gcc",
-        "g++"
-      ]
-    },
-    "ubuntu:focal": [
-      "gcc",
-      "g++"
-    ],
-    "ubuntu:focal::arch64": [
-      "gcc",
-      "g++"
-    ],
-    "centos:7": [
-      "centos-release-scl",
-      "devtoolset-9"
-    ]
-  },
-  "gettext": {
-    "centos:7": "gettext-devel"
-  },
-  "git": {
-    "debian:10": "git",
-    "ubuntu:focal": "git",
-    "ubuntu:focal::arch64": "git",
-    "centos:8": "git"
-  },
-  "gnupg-agent": {
-    "debian:10": "gnupg-agent",
-    "ubuntu:focal": "gnupg-agent",
-    "ubuntu:focal::arch64": "gnupg-agent"
-  },
-  "hugo": {
-    "debian:10": "hugo",
-    "ubuntu:focal": "hugo",
-    "ubuntu:focal::arch64": "hugo"
-  },
-  "libbcprov-java": {
-    "debian:10": "libbcprov-java",
-    "ubuntu:focal": "libbcprov-java",
-    "ubuntu:focal::arch64": "libbcprov-java"
-  },
-  "libtool": {
-    "debian:10": "libtool",
-    "ubuntu:focal": "libtool",
-    "ubuntu:focal::arch64": "libtool",
-    "centos:7": "libtool",
-    "centos:8": "libtool"
-  },
-  "openssl": {
-    "debian:10": "libssl-dev",
-    "ubuntu:focal": "libssl-dev",
-    "ubuntu:focal::arch64": "libssl-dev",
-    "centos:7": "openssl-devel",
-    "centos:8": "openssl-devel"
-  },
-  "perl": {
-    "centos:7": [
-      "perl-CPAN",
-      "perl-devel"
-    ]
-  },
-  "protocol-buffers": {
-    "debian:10": [
-      "libprotobuf-dev",
-      "libprotoc-dev"
-    ],
-    "ubuntu:focal": [
-      "libprotobuf-dev",
-      "libprotoc-dev"
-    ],
-    "ubuntu:focal::arch64": [
-      "libprotobuf-dev",
-      "libprotoc-dev"
-    ]
-  },
-  "sasl": {
-    "debian:10": "libsasl2-dev",
-    "ubuntu:focal": "libsasl2-dev",
-    "ubuntu:focal::arch64": "libsasl2-dev",
-    "centos:7": "cyrus-sasl-devel",
-    "centos:8": "cyrus-sasl-devel"
-  },
-  "snappy": {
-    "debian:10": "libsnappy-dev",
-    "ubuntu:focal": "libsnappy-dev",
-    "ubuntu:focal::arch64": "libsnappy-dev",
-    "centos:7": "snappy-devel"
-  },
-  "zlib": {
-    "debian:10": [
-      "libzstd-dev",
-      "zlib1g-dev"
-    ],
-    "ubuntu:focal": [
-      "libzstd-dev",
-      "zlib1g-dev"
-    ],
-    "ubuntu:focal::arch64": [
-      "libzstd-dev",
-      "zlib1g-dev"
-    ],
-    "centos:7": [
-      "zlib-devel",
-      "lz4-devel"
-    ],
-    "centos:8": [
-      "zlib-devel",
-      "lz4-devel"
-    ]
-  },
-  "locales": {
-    "debian:10": "locales",
-    "ubuntu:focal": "locales",
-    "ubuntu:focal::arch64": "locales"
-  },
-  "libtirpc-devel": {
-    "centos:7": "libtirpc-devel",
-    "centos:8": "libtirpc-devel"
-  },
-  "libpmem": {
-    "centos:7": "libpmem-devel"
-  },
-  "make": {
-    "debian:10": "make",
-    "ubuntu:focal": "make",
-    "ubuntu:focal::arch64": "make",
-    "centos:7": "make",
-    "centos:8": "make"
-  },
-  "maven": {
-    "debian:10": "maven",
-    "ubuntu:focal": "maven",
-    "ubuntu:focal::arch64": "maven"
-  },
-  "java": {
-    "debian:10": "openjdk-11-jdk",
-    "ubuntu:focal": [
-      "openjdk-8-jdk",
-      "openjdk-11-jdk"
-    ],
-    "ubuntu:focal::arch64": [
-      "openjdk-8-jdk",
-      "openjdk-11-jdk"
-    ]
-  },
-  "pinentry-curses": {
-    "debian:10": "pinentry-curses",
-    "ubuntu:focal": "pinentry-curses",
-    "ubuntu:focal::arch64": "pinentry-curses",
-    "centos:7": "pinentry-curses",
-    "centos:8": "pinentry-curses"
-  },
-  "pkg-config": {
-    "debian:10": "pkg-config",
-    "ubuntu:focal": "pkg-config",
-    "ubuntu:focal::arch64": "pkg-config",
-    "centos:8": "pkg-config"
-  },
-  "python": {
-    "debian:10": [
-      "python3",
-      "python3-pip",
-      "python3-pkg-resources",
-      "python3-setuptools",
-      "python3-wheel"
-    ],
-    "ubuntu:focal": [
-      "python3",
-      "python3-pip",
-      "python3-pkg-resources",
-      "python3-setuptools",
-      "python3-wheel"
-    ],
-    "ubuntu:focal::arch64": [
-      "python2.7",
-      "python3",
-      "python3-pip",
-      "python3-pkg-resources",
-      "python3-setuptools",
-      "python3-wheel"
-    ],
-    "centos:7": [
-      "python3",
-      "python3-pip",
-      "python3-setuptools",
-      "python3-wheel"
-    ],
-    "centos:8": [
-      "python3",
-      "python3-pip",
-      "python3-setuptools",
-      "python3-wheel"
-    ]
-  },
-  "rsync": {
-    "debian:10": "rsync",
-    "ubuntu:focal": "rsync",
-    "ubuntu:focal::arch64": "rsync",
-    "centos:7": "rsync",
-    "centos:8": "rsync"
-  },
-  "shellcheck": {
-    "debian:10": "shellcheck",
-    "ubuntu:focal": "shellcheck",
-    "ubuntu:focal::arch64": "shellcheck"
-  },
-  "shasum": {
-    "centos:7": "perl-Digest-SHA",
-    "centos:8": "perl-Digest-SHA"
-  },
-  "software-properties-common": {
-    "debian:10": "software-properties-common",
-    "ubuntu:focal": "software-properties-common",
-    "ubuntu:focal::arch64": "software-properties-common"
-  },
-  "sudo": {
-    "debian:10": "sudo",
-    "ubuntu:focal": "sudo",
-    "ubuntu:focal::arch64": "sudo",
-    "centos:7": "sudo",
-    "centos:8": "sudo"
-  },
-  "valgrind": {
-    "debian:10": "valgrind",
-    "ubuntu:focal": "valgrind",
-    "ubuntu:focal::arch64": "valgrind",
-    "centos:7": "valgrind",
-    "centos:8": "valgrind"
-  },
-  "yasm": {
-    "debian:10": "yasm",
-    "ubuntu:focal": "yasm",
-    "ubuntu:focal::arch64": "yasm"
-  }
-}

dev-support/docker/pkg-resolver/platforms.json

@@ -1,7 +0,0 @@
-[
-  "ubuntu:focal",
-  "ubuntu:focal::arch64",
-  "centos:7",
-  "centos:8",
-  "debian:10"
-]

dev-support/docker/pkg-resolver/resolve.py

@@ -1,98 +0,0 @@
-#!/usr/bin/env python3
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Platform package dependency resolver for building Apache Hadoop.
-"""
-
-import argparse
-import json
-import sys
-from check_platform import is_supported_platform
-
-
-def get_packages(platform, release=None):
-    """
-    Resolve and get the list of packages to install for the given platform.
-
-    :param platform: The platform for which the packages needs to be resolved.
-    :param release: An optional parameter that filters the packages of the given platform for the
-    specified release.
-    :return: A list of resolved packages to install.
-    """
-    with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file:
-        pkgs = json.loads(pkg_file.read())
-    packages = []
-
-    def process_package(package, in_release=False):
-        """
-        Processes the given package object that belongs to a platform and adds it to the packages
-        list variable in the parent scope.
-        In essence, this method recursively traverses the JSON structure defined in packages.json
-        and performs the core filtering.
-
-        :param package: The package object to process.
-        :param in_release: A boolean that indicates whether the current travels belongs to a package
-        that needs to be filtered for the given release label.
-        """
-        if isinstance(package, list):
-            for entry in package:
-                process_package(entry, in_release)
-        elif isinstance(package, dict):
-            if release is None:
-                return
-            for entry in package.get(release, []):
-                process_package(entry, in_release=True)
-        elif isinstance(package, str):
-            # Filter out the package that doesn't belong to this release,
-            # if a release label has been specified.
-            if release is not None and not in_release:
-                return
-            packages.append(package)
-        else:
-            raise Exception('Unknown package of type: {}'.format(type(package)))
-
-    for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()):
-        process_package(platforms.get(platform))
-    return packages
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1),
-              file=sys.stderr)
-        sys.exit(1)
-
-    arg_parser = argparse.ArgumentParser(
-        description='Platform package dependency resolver for building Apache Hadoop')
-    arg_parser.add_argument('-r', '--release', nargs=1, type=str,
-                            help='The release label to filter the packages for the given platform')
-    arg_parser.add_argument('platform', nargs=1, type=str,
-                            help='The name of the platform to resolve the dependencies for')
-    args = arg_parser.parse_args()
-
-    if not is_supported_platform(args.platform[0]):
-        print(
-            'ERROR: The given platform {} is not supported. '
-            'Please refer to platforms.json for a list of supported platforms'.format(
-                args.platform), file=sys.stderr)
-        sys.exit(1)
-
-    packages_to_install = get_packages(args.platform[0],
-                                       args.release[0] if args.release is not None else None)
-    print(' '.join(packages_to_install))

dev-support/docker/pkg-resolver/set-vault-as-baseurl-centos.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ $# -lt 1 ]; then
-  echo "ERROR: Need at least 1 argument, $# were provided"
-  exit 1
-fi
-
-if [ "$1" == "centos:7" ] || [ "$1" == "centos:8" ]; then
-  cd /etc/yum.repos.d/ || exit &&
-    sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* &&
-    sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* &&
-    yum update -y &&
-    cd /root || exit
-else
-  echo "ERROR: Setting the archived baseurl is only supported for centos 7 and 8 environments"
-  exit 1
-fi

dev-support/git-jira-validation/README.md

@@ -1,134 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements.  See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License.  You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-Apache Hadoop Git/Jira FixVersion validation
-============================================================
-
-Git commits in Apache Hadoop contains Jira number of the format
-HADOOP-XXXX or HDFS-XXXX or YARN-XXXX or MAPREDUCE-XXXX.
-While creating a release candidate, we also include changelist
-and this changelist can be identified based on Fixed/Closed Jiras
-with the correct fix versions. However, sometimes we face few
-inconsistencies between fixed Jira and Git commit message.
-
-git_jira_fix_version_check.py script takes care of
-identifying all git commits with commit
-messages with any of these issues:
-
-1. commit is reverted as per commit message
-2. commit does not contain Jira number format in message
-3. Jira does not have expected fixVersion
-4. Jira has expected fixVersion, but it is not yet resolved
-
-Moreover, this script also finds any resolved Jira with expected
-fixVersion but without any corresponding commit present.
-
-This should be useful as part of RC preparation.
-
-git_jira_fix_version_check supports python3 and it required
-installation of jira:
-
-```
-$ python3 --version
-Python 3.9.7
-
-$ python3 -m venv ./venv
-
-$ ./venv/bin/pip install -r dev-support/git-jira-validation/requirements.txt
-
-$ ./venv/bin/python dev-support/git-jira-validation/git_jira_fix_version_check.py
-
-```
-
-The script also requires below inputs:
-```
-1. First commit hash to start excluding commits from history:
-   Usually we can provide latest commit hash from last tagged release
-   so that the script will only loop through all commits in git commit
-   history before this commit hash. e.g for 3.3.2 release, we can provide
-   git hash: fa4915fdbbbec434ab41786cb17b82938a613f16
-   because this commit bumps up hadoop pom versions to 3.3.2:
-   https://github.com/apache/hadoop/commit/fa4915fdbbbec434ab41786cb17b82938a613f16
-
-2. Fix Version:
-   Exact fixVersion that we would like to compare all Jira's fixVersions
-   with. e.g for 3.3.2 release, it should be 3.3.2.
-
-3. JIRA Project Name:
-   The exact name of Project as case-sensitive e.g HADOOP / OZONE
-
-4. Path of project's working dir with release branch checked-in:
-   Path of project from where we want to compare git hashes from. Local fork
-   of the project should be up-to date with upstream and expected release
-   branch should be checked-in.
-
-5. Jira server url (default url: https://issues.apache.org/jira):
-   Default value of server points to ASF Jiras but this script can be
-   used outside of ASF Jira too.
-```
-
-
-Example of script execution:
-```
-JIRA Project Name (e.g HADOOP / OZONE etc): HADOOP
-First commit hash to start excluding commits from history: fa4915fdbbbec434ab41786cb17b82938a613f16
-Fix Version: 3.3.2
-Jira server url (default: https://issues.apache.org/jira):
-Path of project's working dir with release branch checked-in: /Users/vjasani/Documents/src/hadoop-3.3/hadoop
-
-Check git status output and verify expected branch
-
-On branch branch-3.3.2
-Your branch is up to date with 'origin/branch-3.3.2'.
-
-nothing to commit, working tree clean
-
-
-Jira/Git commit message diff starting: ##############################################
-Jira not present with version: 3.3.2. 	 Commit: 8cd8e435fb43a251467ca74fadcb14f21a3e8163 HADOOP-17198. Support S3 Access Points  (#3260) (branch-3.3.2) (#3955)
-WARN: Jira not found. 			 Commit: 8af28b7cca5c6020de94e739e5373afc69f399e5 Updated the index as per 3.3.2 release
-WARN: Jira not found. 			 Commit: e42e483d0085aa46543ebcb1196dd155ddb447d0 Make upstream aware of 3.3.1 release
-Commit seems reverted. 			 Commit: 6db1165380cd308fb74c9d17a35c1e57174d1e09 Revert "HDFS-14099. Unknown frame descriptor when decompressing multiple frames (#3836)"
-Commit seems reverted. 			 Commit: 1e3f94fa3c3d4a951d4f7438bc13e6f008f228f4 Revert "HDFS-16333. fix balancer bug when transfer an EC block (#3679)"
-Jira not present with version: 3.3.2. 	 Commit: ce0bc7b473a62a580c1227a4de6b10b64b045d3a HDFS-16344. Improve DirectoryScanner.Stats#toString (#3695)
-Jira not present with version: 3.3.2. 	 Commit: 30f0629d6e6f735c9f4808022f1a1827c5531f75 HDFS-16339. Show the threshold when mover threads quota is exceeded (#3689)
-Jira not present with version: 3.3.2. 	 Commit: e449daccf486219e3050254d667b74f92e8fc476 YARN-11007. Correct words in YARN documents (#3680)
-Commit seems reverted. 			 Commit: 5c189797828e60a3329fd920ecfb99bcbccfd82d Revert "HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686)"
-Jira not present with version: 3.3.2. 	 Commit: 544dffd179ed756bc163e4899e899a05b93d9234 HDFS-16171. De-flake testDecommissionStatus (#3280)
-Jira not present with version: 3.3.2. 	 Commit: c6914b1cb6e4cab8263cd3ae5cc00bc7a8de25de HDFS-16350. Datanode start time should be set after RPC server starts successfully (#3711)
-Jira not present with version: 3.3.2. 	 Commit: 328d3b84dfda9399021ccd1e3b7afd707e98912d HDFS-16336. Addendum: De-flake TestRollingUpgrade#testRollback (#3686)
-Jira not present with version: 3.3.2. 	 Commit: 3ae8d4ccb911c9ababd871824a2fafbb0272c016 HDFS-16336. De-flake TestRollingUpgrade#testRollback (#3686)
-Jira not present with version: 3.3.2. 	 Commit: 15d3448e25c797b7d0d401afdec54683055d4bb5 HADOOP-17975. Fallback to simple auth does not work for a secondary DistributedFileSystem instance. (#3579)
-Jira not present with version: 3.3.2. 	 Commit: dd50261219de71eaa0a1ad28529953e12dfb92e0 YARN-10991. Fix to ignore the grouping "[]" for resourcesStr in parseResourcesString method (#3592)
-Jira not present with version: 3.3.2. 	 Commit: ef462b21bf03b10361d2f9ea7b47d0f7360e517f HDFS-16332. Handle invalid token exception in sasl handshake (#3677)
-WARN: Jira not found. 			 Commit: b55edde7071419410ea5bea4ce6462b980e48f5b Also update hadoop.version to 3.3.2
-...
-...
-...
-Found first commit hash after which git history is redundant. commit: fa4915fdbbbec434ab41786cb17b82938a613f16
-Exiting successfully
-Jira/Git commit message diff completed: ##############################################
-
-Any resolved Jira with fixVersion 3.3.2 but corresponding commit not present
-Starting diff: ##############################################
-HADOOP-18066 is marked resolved with fixVersion 3.3.2 but no corresponding commit found
-HADOOP-17936 is marked resolved with fixVersion 3.3.2 but no corresponding commit found
-Completed diff: ##############################################
-
-
-```
-

dev-support/git-jira-validation/git_jira_fix_version_check.py

@@ -1,117 +0,0 @@
-#!/usr/bin/env python3
-############################################################################
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-############################################################################
-"""An application to assist Release Managers with ensuring that histories in
-Git and fixVersions in JIRA are in agreement. See README.md for a detailed
-explanation.
-"""
-
-
-import os
-import re
-import subprocess
-
-from jira import JIRA
-
-jira_project_name = input("JIRA Project Name (e.g HADOOP / OZONE etc): ") \
-                    or "HADOOP"
-# Define project_jira_keys with - appended. e.g for HADOOP Jiras,
-# project_jira_keys should include HADOOP-, HDFS-, YARN-, MAPREDUCE-
-project_jira_keys = [jira_project_name + '-']
-if jira_project_name == 'HADOOP':
-    project_jira_keys.append('HDFS-')
-    project_jira_keys.append('YARN-')
-    project_jira_keys.append('MAPREDUCE-')
-
-first_exclude_commit_hash = input("First commit hash to start excluding commits from history: ")
-fix_version = input("Fix Version: ")
-
-jira_server_url = input(
-    "Jira server url (default: https://issues.apache.org/jira): ") \
-        or "https://issues.apache.org/jira"
-
-jira = JIRA(server=jira_server_url)
-
-local_project_dir = input("Path of project's working dir with release branch checked-in: ")
-os.chdir(local_project_dir)
-
-GIT_STATUS_MSG = subprocess.check_output(['git', 'status']).decode("utf-8")
-print('\nCheck git status output and verify expected branch\n')
-print(GIT_STATUS_MSG)
-
-print('\nJira/Git commit message diff starting: ##############################################')
-
-issue_set_from_commit_msg = set()
-
-for commit in subprocess.check_output(['git', 'log', '--pretty=oneline']).decode(
-        "utf-8").splitlines():
-    if commit.startswith(first_exclude_commit_hash):
-        print("Found first commit hash after which git history is redundant. commit: "
-              + first_exclude_commit_hash)
-        print("Exiting successfully")
-        break
-    if re.search('revert', commit, re.IGNORECASE):
-        print("Commit seems reverted. \t\t\t Commit: " + commit)
-        continue
-    ACTUAL_PROJECT_JIRA = None
-    matches = re.findall('|'.join(project_jira_keys), commit)
-    if matches:
-        ACTUAL_PROJECT_JIRA = matches[0]
-    if not ACTUAL_PROJECT_JIRA:
-        print("WARN: Jira not found. \t\t\t Commit: " + commit)
-        continue
-    JIRA_NUM = ''
-    for c in commit.split(ACTUAL_PROJECT_JIRA)[1]:
-        if c.isdigit():
-            JIRA_NUM = JIRA_NUM + c
-        else:
-            break
-    issue = jira.issue(ACTUAL_PROJECT_JIRA + JIRA_NUM)
-    EXPECTED_FIX_VERSION = False
-    for version in issue.fields.fixVersions:
-        if version.name == fix_version:
-            EXPECTED_FIX_VERSION = True
-            break
-    if not EXPECTED_FIX_VERSION:
-        print("Jira not present with version: " + fix_version + ". \t Commit: " + commit)
-        continue
-    if issue.fields.status is None or issue.fields.status.name not in ('Resolved', 'Closed'):
-        print("Jira is not resolved yet? \t\t Commit: " + commit)
-    else:
-        # This means Jira corresponding to current commit message is resolved with expected
-        # fixVersion.
-        # This is no-op by default, if needed, convert to print statement.
-        issue_set_from_commit_msg.add(ACTUAL_PROJECT_JIRA + JIRA_NUM)
-
-print('Jira/Git commit message diff completed: ##############################################')
-
-print('\nAny resolved Jira with fixVersion ' + fix_version
-      + ' but corresponding commit not present')
-print('Starting diff: ##############################################')
-all_issues_with_fix_version = jira.search_issues(
-    'project=' + jira_project_name + ' and status in (Resolved,Closed) and fixVersion='
-    + fix_version)
-
-for issue in all_issues_with_fix_version:
-    if issue.key not in issue_set_from_commit_msg:
-        print(issue.key + ' is marked resolved with fixVersion ' + fix_version
-            + ' but no corresponding commit found')
-
-print('Completed diff: ##############################################')

dev-support/git-jira-validation/requirements.txt

@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-jira==3.1.1

dev-support/hadoop-vote.sh

@@ -1,204 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This script is useful to perform basic sanity tests for the given
-# Hadoop RC. It checks for the Checksum, Signature, Rat check,
-# Build from source and building tarball from the source.
-
-set -e -o pipefail
-
-usage() {
-  SCRIPT=$(basename "${BASH_SOURCE[@]}")
-
-  cat << __EOF
-hadoop-vote. A script for standard vote which verifies the following items
-1. Checksum of sources and binaries
-2. Signature of sources and binaries
-3. Rat check
-4. Built from source
-5. Built tar from source
-
-Usage: ${SCRIPT} -s | --source <url> [-k | --key <signature>] [-f | --keys-file-url <url>] [-o | --output-dir </path/to/use>] [-D property[=value]] [-P profiles]
-       ${SCRIPT} -h | --help
-
-  -h | --help                   Show this screen.
-  -s | --source '<url>'         A URL pointing to the release candidate sources and binaries
-                                e.g. https://dist.apache.org/repos/dist/dev/hadoop/hadoop-<version>RC0/
-  -k | --key '<signature>'      A signature of the public key, e.g. 9AD2AE49
-  -f | --keys-file-url '<url>'   the URL of the key file, default is
-                                https://downloads.apache.org/hadoop/common/KEYS
-  -o | --output-dir '</path>'   directory which has the stdout and stderr of each verification target
-  -D |                          list of maven properties to set for the mvn invocations, e.g. <-D hbase.profile=2.0 -D skipTests> Defaults to unset
-  -P |                          list of maven profiles to set for the build from source, e.g. <-P native -P yarn-ui>
-__EOF
-}
-
-MVN_PROPERTIES=()
-MVN_PROFILES=()
-
-while ((${#})); do
-  case "${1}" in
-    -h | --help )
-      usage; exit 0 ;;
-    -s | --source )
-      SOURCE_URL="${2}"; shift 2 ;;
-    -k | --key )
-      SIGNING_KEY="${2}"; shift 2 ;;
-    -f | --keys-file-url )
-      KEY_FILE_URL="${2}"; shift 2 ;;
-    -o | --output-dir )
-      OUTPUT_DIR="${2}"; shift 2 ;;
-    -D )
-      MVN_PROPERTIES+=("-D ${2}"); shift 2 ;;
-    -P )
-      MVN_PROFILES+=("-P ${2}"); shift 2 ;;
-    * )
-      usage >&2; exit 1             ;;
-  esac
-done
-
-# Source url must be provided
-if [ -z "${SOURCE_URL}" ]; then
-  usage;
-  exit 1
-fi
-
-cat << __EOF
-Although This tool helps verifying Hadoop RC build and unit tests,
-operator may still consider verifying the following manually:
-1. Verify the API compatibility report
-2. Integration/performance/benchmark tests
-3. Object store specific Integration tests against an endpoint
-4. Verify overall unit test stability from Jenkins builds or locally
-5. Other concerns if any
-__EOF
-
-[[ "${SOURCE_URL}" != */ ]] && SOURCE_URL="${SOURCE_URL}/"
-HADOOP_RC_VERSION=$(tr "/" "\n" <<< "${SOURCE_URL}" | tail -n2)
-HADOOP_VERSION=$(echo "${HADOOP_RC_VERSION}" | sed -e 's/-RC[0-9]//g' | sed -e 's/hadoop-//g')
-JAVA_VERSION=$(java -version 2>&1 | cut -f3 -d' ' | head -n1 | sed -e 's/"//g')
-OUTPUT_DIR="${OUTPUT_DIR:-$(pwd)}"
-
-if [ ! -d "${OUTPUT_DIR}" ]; then
-    echo "Output directory ${OUTPUT_DIR} does not exist, please create it before running this script."
-    exit 1
-fi
-
-OUTPUT_PATH_PREFIX="${OUTPUT_DIR}"/"${HADOOP_RC_VERSION}"
-
-# default value for verification targets, 0 = failed
-SIGNATURE_PASSED=0
-CHECKSUM_PASSED=0
-RAT_CHECK_PASSED=0
-BUILD_FROM_SOURCE_PASSED=0
-BUILD_TAR_FROM_SOURCE_PASSED=0
-
-function download_and_import_keys() {
-    KEY_FILE_URL="${KEY_FILE_URL:-https://downloads.apache.org/hadoop/common/KEYS}"
-    echo "Obtain and import the publisher key(s) from ${KEY_FILE_URL}"
-    # download the keys file into file KEYS
-    wget -O KEYS "${KEY_FILE_URL}"
-    gpg --import KEYS
-    if [ -n "${SIGNING_KEY}" ]; then
-        gpg --list-keys "${SIGNING_KEY}"
-    fi
-}
-
-function download_release_candidate () {
-    # get all files from release candidate repo
-    wget -r -np -N -nH --cut-dirs 4 "${SOURCE_URL}"
-}
-
-function verify_signatures() {
-    rm -f "${OUTPUT_PATH_PREFIX}"_verify_signatures
-    for file in *.tar.gz; do
-        gpg --verify "${file}".asc "${file}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_signatures && SIGNATURE_PASSED=1 || SIGNATURE_PASSED=0
-    done
-}
-
-function verify_checksums() {
-    rm -f "${OUTPUT_PATH_PREFIX}"_verify_checksums
-    SHA_EXT=$(find . -name "*.sha*" | awk -F '.' '{ print $NF }' | head -n 1)
-    for file in *.tar.gz; do
-        sha512sum --tag "${file}" > "${file}"."${SHA_EXT}".tmp
-        diff "${file}"."${SHA_EXT}".tmp "${file}"."${SHA_EXT}" 2>&1 | tee -a "${OUTPUT_PATH_PREFIX}"_verify_checksums && CHECKSUM_PASSED=1 || CHECKSUM_PASSED=0
-        rm -f "${file}"."${SHA_EXT}".tmp
-    done
-}
-
-function unzip_from_source() {
-    tar -zxvf hadoop-"${HADOOP_VERSION}"-src.tar.gz
-    cd hadoop-"${HADOOP_VERSION}"-src
-}
-
-function rat_test() {
-    rm -f "${OUTPUT_PATH_PREFIX}"_rat_test
-    mvn clean apache-rat:check "${MVN_PROPERTIES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_rat_test && RAT_CHECK_PASSED=1
-}
-
-function build_from_source() {
-    rm -f "${OUTPUT_PATH_PREFIX}"_build_from_source
-    # No unit test run.
-    mvn clean install "${MVN_PROPERTIES[@]}" -DskipTests "${MVN_PROFILES[@]}" 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_from_source && BUILD_FROM_SOURCE_PASSED=1
-}
-
-function build_tar_from_source() {
-    rm -f "${OUTPUT_PATH_PREFIX}"_build_tar_from_source
-    # No unit test run.
-    mvn clean package "${MVN_PROPERTIES[@]}" -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true 2>&1 | tee "${OUTPUT_PATH_PREFIX}"_build_tar_from_source && BUILD_TAR_FROM_SOURCE_PASSED=1
-}
-
-function execute() {
-   ${1} || print_when_exit
-}
-
-function print_when_exit() {
-  cat << __EOF
-        * Signature: $( ((SIGNATURE_PASSED)) && echo "ok" || echo "failed" )
-        * Checksum : $( ((CHECKSUM_PASSED)) && echo "ok" || echo "failed" )
-        * Rat check (${JAVA_VERSION}): $( ((RAT_CHECK_PASSED)) && echo "ok" || echo "failed" )
-         - mvn clean apache-rat:check ${MVN_PROPERTIES[@]}
-        * Built from source (${JAVA_VERSION}): $( ((BUILD_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
-         - mvn clean install ${MVN_PROPERTIES[@]} -DskipTests ${MVN_PROFILES[@]}
-        * Built tar from source (${JAVA_VERSION}): $( ((BUILD_TAR_FROM_SOURCE_PASSED)) && echo "ok" || echo "failed" )
-         - mvn clean package ${MVN_PROPERTIES[@]} -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true
-__EOF
-  if ((CHECKSUM_PASSED)) && ((SIGNATURE_PASSED)) && ((RAT_CHECK_PASSED)) && ((BUILD_FROM_SOURCE_PASSED)) && ((BUILD_TAR_FROM_SOURCE_PASSED)) ; then
-    exit 0
-  fi
-  exit 1
-}
-
-pushd "${OUTPUT_DIR}"
-
-download_and_import_keys
-download_release_candidate
-
-pushd "${HADOOP_RC_VERSION}"
-
-execute verify_signatures
-execute verify_checksums
-execute unzip_from_source
-execute rat_test
-execute build_from_source
-execute build_tar_from_source
-
-popd
-popd
-
-print_when_exit

dev-support/jenkins.sh

@@ -1,259 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script is called from the Jenkinsfile, which ultimately runs
-# the CI through Yetus.
-# We use Ubuntu Focal as the main platform for building Hadoop, thus
-# it runs for all the PRs. Additionally, we also ensure that
-# Hadoop builds across the supported platforms whenever there's a change
-# in any of the C/C++ files, C/C++ build files or platform changes.
-
-## @description  Check if the given extension is related to C/C++
-## @param        seeking
-## @return       0 if yes
-## @return       1 if no
-is_c_cpp_extension() {
-  local c_cpp_extension=("c" "cc" "cpp" "h" "hpp")
-  local seeking=$1
-
-  for element in "${c_cpp_extension[@]}"; do
-    if [[ $element == "$seeking" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-## @description  Check if the given relative path corresponds to
-##               change in platform files
-## @param        in_path
-## @return       0 if yes
-## @return       1 if no
-is_platform_change() {
-  declare in_path
-  in_path="${SOURCEDIR}"/"${1}"
-
-  for path in "${DOCKERFILE}" "${SOURCEDIR}"/dev-support/docker/pkg-resolver/*.json; do
-    if [ "${in_path}" == "${path}" ]; then
-      echo "Found C/C++ platform related changes in ${in_path}"
-      return 0
-    fi
-  done
-  return 1
-}
-
-## @description  Checks if the given path corresponds to a change
-##               in C/C++ files or related to C/C++ build system
-## @param        path
-## @return       0 if yes
-## @return       1 if no
-is_c_cpp_change() {
-  shopt -s nocasematch
-
-  local path=$1
-  declare filename
-  filename=$(basename -- "${path}")
-  extension=${filename##*.}
-
-  if is_c_cpp_extension "${extension}"; then
-    echo "Found C/C++ changes in ${path}"
-    return 0
-  fi
-
-  if [[ $filename =~ CMakeLists\.txt ]]; then
-    echo "Found C/C++ build related changes in ${path}"
-    return 0
-  fi
-  return 1
-}
-
-## @description  Check if the CI needs to be run - CI will always run if
-##               IS_OPTIONAL is 0, or if there's any change in
-##               C/C++ files or C/C++ build or platform
-## @return       0 if yes
-## @return       1 if no
-function check_ci_run() {
-  # Get the first commit of this PR relative to the trunk branch
-  firstCommitOfThisPr=$(git --git-dir "${SOURCEDIR}/.git" rev-parse origin/trunk)
-
-  # Loop over the paths of all the changed files and check if the criteria
-  # to run the CI has been satisfied
-  for path in $(git --git-dir "${SOURCEDIR}/.git" diff --name-only "${firstCommitOfThisPr}" HEAD); do
-    if is_c_cpp_change "${path}"; then
-      return 0
-    fi
-
-    if is_platform_change "${path}"; then
-      return 0
-    fi
-  done
-
-  # We must run the CI if it's not optional
-  if [ "$IS_OPTIONAL" -eq 0 ]; then
-    return 0
-  fi
-  return 1
-}
-
-## @description  Run the CI using YETUS
-function run_ci() {
-  TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
-
-  if [[ "$IS_WINDOWS" && "$IS_WINDOWS" == 1 ]]; then
-    echo "Building in a Windows environment, skipping some Yetus related settings"
-  else
-    # run in docker mode and specifically point to our
-    # Dockerfile since we don't want to use the auto-pulled version.
-    YETUS_ARGS+=("--docker")
-    YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
-    YETUS_ARGS+=("--mvn-custom-repos")
-    YETUS_ARGS+=("--dockermemlimit=22g")
-
-    # test with Java 8 and 11
-    YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
-    YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
-    YETUS_ARGS+=("--multijdktests=compile")
-  fi
-
-  if [[ "$IS_NIGHTLY_BUILD" && "$IS_NIGHTLY_BUILD" == 1 ]]; then
-    YETUS_ARGS+=("--empty-patch")
-    YETUS_ARGS+=("--branch=${BRANCH_NAME}")
-  else
-    # this must be clean for every run
-    if [[ -d "${PATCHDIR}" ]]; then
-      rm -rf "${PATCHDIR:?}"
-    fi
-    mkdir -p "${PATCHDIR}"
-
-    # if given a JIRA issue, process it. If CHANGE_URL is set
-    # (e.g., Github Branch Source plugin), process it.
-    # otherwise exit, because we don't want Hadoop to do a
-    # full build.  We wouldn't normally do this check for smaller
-    # projects. :)
-    if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
-      YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
-    elif [[ -z "${CHANGE_URL}" ]]; then
-      echo "Full build skipped" >"${PATCHDIR}/report.html"
-      exit 0
-    fi
-
-    # write Yetus report as GitHub comment (YETUS-1102)
-    YETUS_ARGS+=("--github-write-comment")
-    YETUS_ARGS+=("--github-use-emoji-vote")
-  fi
-
-  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
-
-  # where the source is located
-  YETUS_ARGS+=("--basedir=${SOURCEDIR}")
-
-  # our project defaults come from a personality file
-  YETUS_ARGS+=("--project=hadoop")
-  YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")
-
-  # lots of different output formats
-  YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
-  YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
-  YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
-
-  # enable writing back to Github
-  YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
-
-  # auto-kill any surefire stragglers during unit test runs
-  YETUS_ARGS+=("--reapermode=kill")
-
-  # set relatively high limits for ASF machines
-  # changing these to higher values may cause problems
-  # with other jobs on systemd-enabled machines
-  YETUS_ARGS+=("--proclimit=5500")
-
-  # -1 spotbugs issues that show up prior to the patch being applied
-  YETUS_ARGS+=("--spotbugs-strict-precheck")
-
-  # rsync these files back into the archive dir
-  YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
-
-  # URL for user-side presentation in reports and such to our artifacts
-  # (needs to match the archive bits below)
-  YETUS_ARGS+=("--build-url-artifacts=artifact/out")
-
-  # plugins to enable
-  YETUS_ARGS+=("--plugins=all,-jira")
-
-  # don't let these tests cause -1s because we aren't really paying that
-  # much attention to them
-  YETUS_ARGS+=("--tests-filter=checkstyle")
-
-  # effectively treat dev-suport as a custom maven module
-  YETUS_ARGS+=("--skip-dirs=dev-support")
-
-  # help keep the ASF boxes clean
-  YETUS_ARGS+=("--sentinel")
-
-  # custom javadoc goals
-  YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
-
-  "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
-}
-
-## @description  Cleans up the processes started by YETUS
-function cleanup_ci_proc() {
-  # See YETUS-764
-  if [ -f "${PATCHDIR}/pidfile.txt" ]; then
-    echo "test-patch process appears to still be running: killing"
-    kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
-    sleep 10
-  fi
-  if [ -f "${PATCHDIR}/cidfile.txt" ]; then
-    echo "test-patch container appears to still be running: killing"
-    docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
-  fi
-}
-
-## @description  Invokes github_status_recovery in YETUS's precommit
-function github_status_recovery() {
-  YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
-  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
-  TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
-  /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
-}
-
-if [ -z "$1" ]; then
-  echo "Must specify an argument for jenkins.sh"
-  echo "run_ci                  - Runs the CI based on platform image as defined by DOCKERFILE"
-  echo "cleanup_ci_proc         - Cleans up the processes spawned for running the CI"
-  echo "github_status_recovery  - Sends Github status (refer to YETUS precommit for more details)"
-  exit 1
-fi
-
-# Process arguments to jenkins.sh
-if [ "$1" == "run_ci" ]; then
-  # Check if the CI needs to be run, if so, do so :)
-  if check_ci_run; then
-    run_ci
-  else
-    echo "No C/C++ file or C/C++ build or platform changes found, will not run CI for this platform"
-  fi
-elif [ "$1" == "cleanup_ci_proc" ]; then
-  cleanup_ci_proc
-elif [ "$1" == "github_status_recovery" ]; then
-  github_status_recovery
-else
-  echo "Don't know how to process $1"
-  exit 1
-fi

hadoop-assemblies/pom.xml

@@ -23,11 +23,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-assemblies</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <name>Apache Hadoop Assemblies</name>
   <description>Apache Hadoop Assemblies</description>
 

hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml

@@ -131,19 +131,6 @@
       </includes>
       <outputDirectory>/include</outputDirectory>
     </fileSet>
-    <fileSet>
-      <directory>.</directory>
-      <includes>
-        <include>LICENSE-binary</include>
-        <include>NOTICE-binary</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>./license-binary</directory>
-      <includes>
-        <include>*</include>
-      </includes>
-    </fileSet>
   </fileSets>
   <dependencySets>
     <dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-blockgen.xml

@@ -1,35 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
-  <id>hadoop-dynamometer-blockgen</id>
-  <formats>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-
-  <fileSets>
-    <fileSet>
-      <directory>${basedir}/src/main/bash</directory>
-      <outputDirectory>dynamometer-blockgen/bin</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
-  </fileSets>
-
-</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-infra.xml

@@ -1,35 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
-  <id>hadoop-dynamometer-infra</id>
-  <formats>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-
-  <fileSets>
-    <fileSet>
-      <directory>${basedir}/src/main/bash</directory>
-      <outputDirectory>dynamometer-infra/bin</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
-  </fileSets>
-
-</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer-workload.xml

@@ -1,35 +0,0 @@
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
-  <id>hadoop-dynamometer-workload</id>
-  <formats>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-
-  <fileSets>
-    <fileSet>
-      <directory>${basedir}/src/main/bash</directory>
-      <outputDirectory>dynamometer-workload/bin</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
-  </fileSets>
-
-</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml

@@ -1,73 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the Li2cense is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
-  <id>hadoop-dynamometer</id>
-  <formats>
-    <format>dir</format>
-  </formats>
-  <includeBaseDirectory>false</includeBaseDirectory>
-  <fileSets>
-    <fileSet>
-      <directory>../hadoop-dynamometer-blockgen/target</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
-      <includes>
-        <include>*-sources.jar</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>../hadoop-dynamometer-blockgen/target/hadoop-dynamometer-blockgen-${project.version}/dynamometer-blockgen</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-blockgen</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>../hadoop-dynamometer-workload/target</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
-      <includes>
-        <include>*-sources.jar</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>../hadoop-dynamometer-workload/target/hadoop-dynamometer-workload-${project.version}/dynamometer-workload</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-workload</outputDirectory>
-    </fileSet>
-    <fileSet>
-      <directory>../hadoop-dynamometer-infra/target</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
-      <includes>
-        <include>*-sources.jar</include>
-      </includes>
-    </fileSet>
-    <fileSet>
-      <directory>../hadoop-dynamometer-infra/target/hadoop-dynamometer-infra-${project.version}/dynamometer-infra</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/dynamometer/dynamometer-infra</outputDirectory>
-    </fileSet>
-  </fileSets>
-  <dependencySets>
-    <dependencySet>
-      <outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
-      <unpack>false</unpack>
-      <scope>runtime</scope>
-      <useProjectArtifact>false</useProjectArtifact>
-      <excludes>
-        <!-- use slf4j from common to avoid multiple binding warnings -->
-        <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
-      </excludes>
-    </dependencySet>
-  </dependencySets>
-</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-src-submarine.xml

@@ -0,0 +1,56 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+  <id>hadoop-src</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <includeBaseDirectory>true</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>.</directory>
+      <includes>
+        <include>LICENCE.txt</include>
+        <include>README.txt</include>
+        <include>NOTICE.txt</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>.</directory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>.git/**</exclude>
+        <exclude>**/.gitignore</exclude>
+        <exclude>**/.svn</exclude>
+        <exclude>**/*.iws</exclude>
+        <exclude>**/*.ipr</exclude>
+        <exclude>**/*.iml</exclude>
+        <exclude>**/.classpath</exclude>
+        <exclude>**/.project</exclude>
+        <exclude>**/.settings</exclude>
+        <exclude>**/target/**</exclude>
+        <!-- until the code that does this is fixed -->
+        <exclude>**/*.log</exclude>
+        <exclude>**/build/**</exclude>
+        <exclude>**/file:/**</exclude>
+        <exclude>**/SecurityAuth.audit*</exclude>
+      </excludes>
+    </fileSet>
+  </fileSets>
+</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-src-with-hdds.xml

@@ -0,0 +1,56 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
+  <id>hadoop-src</id>
+  <formats>
+    <format>tar.gz</format>
+  </formats>
+  <includeBaseDirectory>true</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>.</directory>
+      <includes>
+        <include>LICENCE.txt</include>
+        <include>README.txt</include>
+        <include>NOTICE.txt</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>.</directory>
+      <useDefaultExcludes>true</useDefaultExcludes>
+      <excludes>
+        <exclude>.git/**</exclude>
+        <exclude>**/.gitignore</exclude>
+        <exclude>**/.svn</exclude>
+        <exclude>**/*.iws</exclude>
+        <exclude>**/*.ipr</exclude>
+        <exclude>**/*.iml</exclude>
+        <exclude>**/.classpath</exclude>
+        <exclude>**/.project</exclude>
+        <exclude>**/.settings</exclude>
+        <exclude>**/target/**</exclude>
+        <!-- until the code that does this is fixed -->
+        <exclude>**/*.log</exclude>
+        <exclude>**/build/**</exclude>
+        <exclude>**/file:/**</exclude>
+        <exclude>**/SecurityAuth.audit*</exclude>
+      </excludes>
+    </fileSet>
+  </fileSets>
+</assembly>

hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml

@@ -31,12 +31,6 @@
         <include>NOTICE.txt</include>
       </includes>
     </fileSet>
-    <fileSet>
-      <directory>./licenses</directory>
-      <includes>
-        <include>*</include>
-      </includes>
-    </fileSet>
     <fileSet>
       <directory>.</directory>
       <useDefaultExcludes>true</useDefaultExcludes>
@@ -56,7 +50,8 @@
         <exclude>**/build/**</exclude>
         <exclude>**/file:/**</exclude>
         <exclude>**/SecurityAuth.audit*</exclude>
-        <exclude>patchprocess/**</exclude>
+        <exclude>hadoop-ozone/**</exclude>
+        <exclude>hadoop-hdds/**</exclude>
       </excludes>
     </fileSet>
   </fileSets>

hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml

@@ -47,14 +47,6 @@
       <outputDirectory>/libexec/shellprofile.d</outputDirectory>
       <fileMode>0755</fileMode>
     </fileSet>
-    <fileSet>
-      <directory>../hadoop-federation-balance/src/main/shellprofile.d</directory>
-      <includes>
-        <include>*</include>
-      </includes>
-      <outputDirectory>/libexec/shellprofile.d</outputDirectory>
-      <fileMode>0755</fileMode>
-    </fileSet>
     <fileSet>
       <directory>../hadoop-extras/src/main/shellprofile.d</directory>
       <includes>
@@ -119,13 +111,6 @@
         <include>*-sources.jar</include>
       </includes>
     </fileSet>
-    <fileSet>
-      <directory>../hadoop-federation-balance/target</directory>
-      <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
-      <includes>
-        <include>*-sources.jar</include>
-      </includes>
-    </fileSet>
     <fileSet>
       <directory>../hadoop-extras/target</directory>
       <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
@@ -224,7 +209,7 @@
       <excludes>
         <exclude>org.apache.hadoop:hadoop-common</exclude>
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
-        <exclude>org.apache.hadoop:hadoop-client</exclude>
+        <exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
         <!-- pipes is native stuff, this just keeps pom from being package-->
         <exclude>org.apache.hadoop:hadoop-pipes</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->

hadoop-build-tools/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <artifactId>hadoop-main</artifactId>
     <groupId>org.apache.hadoop</groupId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>hadoop-build-tools</artifactId>

hadoop-build-tools/src/main/resources/checkstyle/checkstyle.xml

@@ -67,12 +67,6 @@
     <!--<module name="FileLength">-->
     <module name="FileTabCharacter"/>
 
-    <!-- Checks for line length violations.                                 -->
-    <!-- See https://checkstyle.sourceforge.io/config_sizes.html#LineLength -->
-    <module name="LineLength">
-        <property name="max" value="100"/>
-    </module>
-
     <module name="TreeWalker">
 
         <module name="SuppressWarningsHolder"/>
@@ -121,16 +115,14 @@
 
         <!-- Checks for imports                              -->
         <!-- See http://checkstyle.sf.net/config_import.html -->
-        <module name="IllegalImport">
-            <property name="regexp" value="true"/>
-            <property name="illegalPkgs" value="sun"/>
-        </module>
+        <module name="IllegalImport"/> <!-- defaults to sun.* packages -->
         <module name="RedundantImport"/>
         <module name="UnusedImports"/>
 
 
         <!-- Checks for Size Violations.                    -->
         <!-- See http://checkstyle.sf.net/config_sizes.html -->
+        <module name="LineLength"/>
         <module name="MethodLength"/>
         <module name="ParameterNumber">
           <property name="ignoreOverriddenMethods" value="true"/>
@@ -159,9 +151,7 @@
 
         <!-- Checks for blocks. You know, those {}'s         -->
         <!-- See http://checkstyle.sf.net/config_blocks.html -->
-        <module name="AvoidNestedBlocks">
-          <property name="allowInSwitchCase" value="true"/>
-        </module>
+        <module name="AvoidNestedBlocks"/>
         <module name="EmptyBlock"/>
         <module name="LeftCurly"/>
         <module name="NeedBraces"/>

hadoop-client-modules/hadoop-client-api/pom.xml

@@ -18,11 +18,11 @@
 <parent>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-project</artifactId>
-   <version>3.4.0-SNAPSHOT</version>
+   <version>3.3.0-SNAPSHOT</version>
    <relativePath>../../hadoop-project</relativePath>
 </parent>
   <artifactId>hadoop-client-api</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <description>Apache Hadoop Client</description>
@@ -67,13 +67,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <!-- snappy-java is native library and cannot be relocated. So we explicitly exclude it
-         from shaded jar to prevent possible conflict. Make it as transitive dependency to
-         make the downstream pull it. -->
-    <dependency>
-      <groupId>org.xerial.snappy</groupId>
-      <artifactId>snappy-java</artifactId>
-    </dependency>
   </dependencies>
   <profiles>
     <profile>
@@ -94,10 +87,13 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
-            <configuration>
-              <createSourcesJar>true</createSourcesJar>
-              <shadeSourcesContent>true</shadeSourcesContent>
-            </configuration>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-maven-plugins</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -109,10 +105,6 @@
                     <includes>
                       <include>org.apache.hadoop:*</include>
                     </includes>
-                    <excludes>
-                      <!-- Leave snappy that includes native methods which cannot be relocated. -->
-                      <exclude>org.xerial.snappy:*</exclude>
-                    </excludes>
                   </artifactSet>
                   <filters>
                     <!-- We get these package level classes from various yarn api jars -->
@@ -134,7 +126,9 @@
                       <excludes>
                         <exclude>org/apache/hadoop/*</exclude>
                         <exclude>org/apache/hadoop/**/*</exclude>
-                        <!-- Our non-shaded logging libraries -->
+                        <!-- Our non-shaded htrace and logging libraries -->
+                        <exclude>org/apache/htrace/*</exclude>
+                        <exclude>org/apache/htrace/**/*</exclude>
                         <exclude>org/slf4j/*</exclude>
                         <exclude>org/slf4j/**/*</exclude>
                         <exclude>org/apache/commons/logging/*</exclude>
@@ -149,14 +143,6 @@
                         <exclude>org/w3c/dom/**/*</exclude>
                         <exclude>org/xml/sax/*</exclude>
                         <exclude>org/xml/sax/**/*</exclude>
-                        <exclude>org/bouncycastle/*</exclude>
-                        <exclude>org/bouncycastle/**/*</exclude>
-                        <!-- Exclude snappy-java -->
-                        <exclude>org/xerial/snappy/*</exclude>
-                        <exclude>org/xerial/snappy/**/*</exclude>
-                        <!-- Exclude org.widlfly.openssl -->
-                        <exclude>org/wildfly/openssl/*</exclude>
-                        <exclude>org/wildfly/openssl/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -175,8 +161,6 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
-                        <exclude>com/ibm/security/*</exclude>
-                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -235,9 +219,6 @@
                         <!-- Exclude config keys for Hadoop that look like package names -->
                         <exclude>net/topology/*</exclude>
                         <exclude>net/topology/**/*</exclude>
-                        <!-- Exclude lz4-java -->
-                        <exclude>net/jpountz/*</exclude>
-                        <exclude>net/jpountz/**/*</exclude>
                       </excludes>
                     </relocation>
                     <!-- okio declares a top level package instead of nested -->
@@ -247,7 +228,8 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                    <!-- Needed until MSHADE-182 -->
+                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resource>NOTICE.txt</resource>

hadoop-client-modules/hadoop-client-check-invariants/pom.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -18,11 +18,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-client-check-invariants</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <description>
@@ -56,7 +56,7 @@
           <dependency>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.5.1</version>
+            <version>1.0-beta-3</version>
           </dependency>
         </dependencies>
         <executions>
@@ -80,6 +80,8 @@
                          but enforcer still sees it.
                     -->
                     <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+                    <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+                    <exclude>org.apache.htrace:htrace-core4</exclude>
                     <!-- Leave slf4j unshaded so downstream users can configure logging. -->
                     <exclude>org.slf4j:slf4j-api</exclude>
                     <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
@@ -90,8 +92,6 @@
                     <exclude>com.google.code.findbugs:jsr305</exclude>
                     <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                     <exclude>org.bouncycastle:*</exclude>
-                    <!-- Leave snappy that includes native methods which cannot be relocated. -->
-                    <exclude>org.xerial.snappy:*</exclude>
                   </excludes>
                 </banTransitiveDependencies>
                 <banDuplicateClasses>
@@ -154,7 +154,6 @@
             </goals>
             <configuration>
               <excludeTransitive>true</excludeTransitive>
-              <pathSeparator>;</pathSeparator>
               <outputProperty>hadoop-client-artifacts</outputProperty>
             </configuration>
           </execution>

hadoop-client-modules/hadoop-client-check-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -15,24 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Usage: $0 [/path/to/some/example.jar;/path/to/another/example/created.jar]
+# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar]
 #
 # accepts a single command line argument with a colon separated list of
 # paths to jars to check. Iterates through each such passed jar and checks
 # all the contained paths to make sure they follow the below constructed
 # safe list.
 
-# We use +=, which is a bash 3.1+ feature
-if [[ -z "${BASH_VERSINFO[0]}" ]] \
-   || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
-   || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 1 ]]; then
-  echo "bash v3.1+ is required. Sorry."
-  exit 1
-fi
-
-set -e
-set -o pipefail
-
 # we have to allow the directories that lead to the org/apache/hadoop dir
 allowed_expr="(^org/$|^org/apache/$"
 # We allow the following things to exist in our client artifacts:
@@ -67,29 +56,13 @@ allowed_expr+="|^krb5_udp-template.conf$"
 # Jetty uses this style sheet for directory listings. TODO ensure our
 # internal use of jetty disallows directory listings and remove this.
 allowed_expr+="|^jetty-dir.css$"
-# Snappy java is native library. We cannot relocate it to under org/apache/hadoop.
-allowed_expr+="|^org/xerial/"
 
 allowed_expr+=")"
 declare -i bad_artifacts=0
 declare -a bad_contents
-declare -a artifact_list
-while IFS='' read -r -d ';' line; do artifact_list+=("$line"); done < <(printf '%s;' "$1")
-if [ "${#artifact_list[@]}" -eq 0 ]; then
-  echo "[ERROR] No artifacts passed in."
-  exit 1
-fi
-
-jar_list_failed ()
-{
-    echo "[ERROR] Listing jar contents for file '${artifact}' failed."
-    exit 1
-}
-trap jar_list_failed SIGUSR1
-
+IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
 for artifact in "${artifact_list[@]}"; do
-  # Note: On Windows the output from jar tf may contain \r\n's.  Normalize to \n.
-  while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${allowed_expr}" )
+  bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}"))
   if [ ${#bad_contents[@]} -gt 0 ]; then
     echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
     echo "    Please check the following and either correct the build or update"

hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -18,11 +18,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-client-check-test-invariants</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>pom</packaging>
 
   <description>
@@ -60,7 +60,7 @@
           <dependency>
             <groupId>org.codehaus.mojo</groupId>
             <artifactId>extra-enforcer-rules</artifactId>
-            <version>1.5.1</version>
+            <version>1.0-beta-3</version>
           </dependency>
         </dependencies>
         <executions>
@@ -84,6 +84,8 @@
                          but enforcer still sees it.
                     -->
                     <exclude>org.apache.hadoop:hadoop-annotations</exclude>
+                    <!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
+                    <exclude>org.apache.htrace:htrace-core4</exclude>
                     <!-- Leave slf4j unshaded so downstream users can configure logging. -->
                     <exclude>org.slf4j:slf4j-api</exclude>
                     <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
@@ -98,8 +100,6 @@
                     <exclude>com.google.code.findbugs:jsr305</exclude>
                     <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                     <exclude>org.bouncycastle:*</exclude>
-                    <!-- Leave snappy that includes native methods which cannot be relocated. -->
-                    <exclude>org.xerial.snappy:*</exclude>
                   </excludes>
                 </banTransitiveDependencies>
                 <banDuplicateClasses>
@@ -164,7 +164,6 @@
               <!-- these two get covered in our non-test invariant check -->
               <excludeArtifactIds>hadoop-client-api,hadoop-client-runtime</excludeArtifactIds>
               <excludeTransitive>true</excludeTransitive>
-              <pathSeparator>;</pathSeparator>
               <outputProperty>hadoop-client-artifacts</outputProperty>
             </configuration>
           </execution>

hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh

@@ -15,24 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Usage: $0 [/path/to/some/example.jar;/path/to/another/example/created.jar]
+# Usage: $0 [/path/to/some/example.jar:/path/to/another/example/created.jar]
 #
 # accepts a single command line argument with a colon separated list of
 # paths to jars to check. Iterates through each such passed jar and checks
 # all the contained paths to make sure they follow the below constructed
 # safe list.
 
-# We use +=, which is a bash 3.1+ feature
-if [[ -z "${BASH_VERSINFO[0]}" ]] \
-   || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
-   || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 1 ]]; then
-  echo "bash v3.1+ is required. Sorry."
-  exit 1
-fi
-
-set -e
-set -o pipefail
-
 # we have to allow the directories that lead to the org/apache/hadoop dir
 allowed_expr="(^org/$|^org/apache/$"
 # We allow the following things to exist in our client artifacts:
@@ -41,10 +30,8 @@ allowed_expr="(^org/$|^org/apache/$"
 allowed_expr+="|^org/apache/hadoop/"
 #   * whatever in the "META-INF" directory
 allowed_expr+="|^META-INF/"
-#   * whatever under the "webapps" directory; for things shipped by yarn
+#   * whatever under the "webapps" directory; for minicluster UIs
 allowed_expr+="|^webapps/"
-#   * Resources files used by Hadoop YARN mini cluster
-allowed_expr+="|^TERMINAL/"
 #   * Hadoop's default configuration files, which have the form
 #     "_module_-default.xml"
 allowed_expr+="|^[^-]*-default.xml$"
@@ -56,29 +43,20 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
 #   * Used by JavaSandboxLinuxContainerRuntime as a default, loaded
 #     from root, so can't relocate. :(
 allowed_expr+="|^java.policy$"
-#   * Used by javax.annotation
-allowed_expr+="|^jndi.properties$"
+# * allowing native libraries from rocksdb. Leaving native libraries as it is.
+allowed_expr+="|^librocksdbjni-linux32.so"
+allowed_expr+="|^librocksdbjni-linux64.so"
+allowed_expr+="|^librocksdbjni-osx.jnilib"
+allowed_expr+="|^librocksdbjni-win64.dll"
+allowed_expr+="|^librocksdbjni-linux-ppc64le.so"
+
 
 allowed_expr+=")"
 declare -i bad_artifacts=0
 declare -a bad_contents
-declare -a artifact_list
-while IFS='' read -r -d ';' line; do artifact_list+=("$line"); done < <(printf '%s;' "$1")
-if [ "${#artifact_list[@]}" -eq 0 ]; then
-  echo "[ERROR] No artifacts passed in."
-  exit 1
-fi
-
-jar_list_failed ()
-{
-    echo "[ERROR] Listing jar contents for file '${artifact}' failed."
-    exit 1
-}
-trap jar_list_failed SIGUSR1
-
+IFS=: read -r -d '' -a artifact_list < <(printf '%s\0' "$1")
 for artifact in "${artifact_list[@]}"; do
-  # Note: On Windows the output from jar tf may contain \r\n's.  Normalize to \n.
-  while IFS='' read -r line; do bad_contents+=("$line"); done < <( ( jar tf "${artifact}" | sed 's/\\r//' || kill -SIGUSR1 $$ ) | grep -v -E "${allowed_expr}" )
+  bad_contents=($(jar tf "${artifact}" | grep -v -E "${allowed_expr}"))
   if [ ${#bad_contents[@]} -gt 0 ]; then
     echo "[ERROR] Found artifact with unexpected contents: '${artifact}'"
     echo "    Please check the following and either correct the build or update"

hadoop-client-modules/hadoop-client-integration-tests/pom.xml

@@ -18,11 +18,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-client-integration-tests</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
 
   <description>Checks that we can use the generated artifacts</description>
   <name>Apache Hadoop Client Packaging Integration Tests</name>
@@ -52,11 +52,6 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.lz4</groupId>
-      <artifactId>lz4-java</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
   <profiles>
     <profile>
@@ -80,27 +75,6 @@
           <artifactId>hadoop-client-minicluster</artifactId>
           <scope>test</scope>
         </dependency>
-        <dependency>
-          <groupId>org.bouncycastle</groupId>
-          <artifactId>bcprov-jdk15on</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>org.bouncycastle</groupId>
-          <artifactId>bcpkix-jdk15on</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>javax.xml.bind</groupId>
-          <artifactId>jaxb-api</artifactId>
-          <scope>test</scope>
-        </dependency>
-        <dependency>
-          <groupId>javax.activation</groupId>
-          <artifactId>activation</artifactId>
-          <version>1.1.1</version>
-          <scope>test</scope>
-        </dependency>
       </dependencies>
       <build>
         <plugins>
@@ -184,24 +158,6 @@
           <artifactId>hadoop-hdfs</artifactId>
           <scope>test</scope>
           <type>test-jar</type>
-          <exclusions>
-            <exclusion>
-              <groupId>org.ow2.asm</groupId>
-              <artifactId>asm-commons</artifactId>
-            </exclusion>
-          </exclusions>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-yarn-server-tests</artifactId>
-          <scope>test</scope>
-          <type>test-jar</type>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.hadoop</groupId>
-          <artifactId>hadoop-common</artifactId>
-          <scope>test</scope>
-          <type>test-jar</type>
         </dependency>
       </dependencies>
     </profile>

hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseHadoopCodecs.java

@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.hadoop.example;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
-import java.io.*;
-import java.util.Arrays;
-import java.util.Random;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.RandomDatum;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionInputStream;
-import org.apache.hadoop.io.compress.CompressionOutputStream;
-import org.apache.hadoop.io.compress.zlib.ZlibFactory;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Ensure that we can perform codec operations given the API and runtime jars
- * by performing some simple smoke tests.
- */
-public class ITUseHadoopCodecs {
-
-  private static final Logger LOG = LoggerFactory.getLogger(ITUseHadoopCodecs.class);
-
-  private Configuration haddopConf = new Configuration();
-  private int dataCount = 100;
-  private int dataSeed = new Random().nextInt();
-
-  @Test
-  public void testGzipCodec() throws IOException {
-    ZlibFactory.setNativeZlibLoaded(false);
-    assertFalse(ZlibFactory.isNativeZlibLoaded(haddopConf));
-    codecTest(haddopConf, dataSeed, 0, "org.apache.hadoop.io.compress.GzipCodec");
-    codecTest(haddopConf, dataSeed, dataCount, "org.apache.hadoop.io.compress.GzipCodec");
-  }
-
-  @Test
-  public void testSnappyCodec() throws IOException {
-    codecTest(haddopConf, dataSeed, 0, "org.apache.hadoop.io.compress.SnappyCodec");
-    codecTest(haddopConf, dataSeed, dataCount, "org.apache.hadoop.io.compress.SnappyCodec");
-  }
-
-  @Test
-  public void testLz4Codec() {
-    Arrays.asList(false, true).forEach(config -> {
-      haddopConf.setBoolean(
-          CommonConfigurationKeys.IO_COMPRESSION_CODEC_LZ4_USELZ4HC_KEY,
-          config);
-      try {
-        codecTest(haddopConf, dataSeed, 0, "org.apache.hadoop.io.compress.Lz4Codec");
-        codecTest(haddopConf, dataSeed, dataCount, "org.apache.hadoop.io.compress.Lz4Codec");
-      } catch (IOException e) {
-        throw new RuntimeException("failed when running codecTest", e);
-      }
-    });
-  }
-
-  private void codecTest(Configuration conf, int seed, int count, String codecClass)
-      throws IOException {
-
-    // Create the codec
-    CompressionCodec codec = null;
-    try {
-      codec = (CompressionCodec)
-              ReflectionUtils.newInstance(conf.getClassByName(codecClass), conf);
-    } catch (ClassNotFoundException cnfe) {
-      throw new IOException("Illegal codec!");
-    }
-    LOG.info("Created a Codec object of type: " + codecClass);
-
-    // Generate data
-    DataOutputBuffer data = new DataOutputBuffer();
-    RandomDatum.Generator generator = new RandomDatum.Generator(seed);
-    for(int i = 0; i < count; ++i) {
-      generator.next();
-      RandomDatum key = generator.getKey();
-      RandomDatum value = generator.getValue();
-
-      key.write(data);
-      value.write(data);
-    }
-    LOG.info("Generated " + count + " records");
-
-    // Compress data
-    DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
-    try (CompressionOutputStream deflateFilter =
-        codec.createOutputStream(compressedDataBuffer);
-        DataOutputStream deflateOut =
-            new DataOutputStream(new BufferedOutputStream(deflateFilter))) {
-      deflateOut.write(data.getData(), 0, data.getLength());
-      deflateOut.flush();
-      deflateFilter.finish();
-    }
-
-    // De-compress data
-    DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
-    deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0,
-            compressedDataBuffer.getLength());
-    DataInputBuffer originalData = new DataInputBuffer();
-    originalData.reset(data.getData(), 0, data.getLength());
-    try (CompressionInputStream inflateFilter =
-        codec.createInputStream(deCompressedDataBuffer);
-        DataInputStream originalIn =
-            new DataInputStream(new BufferedInputStream(originalData))) {
-
-      // Check
-      int expected;
-      do {
-        expected = originalIn.read();
-        assertEquals("Inflated stream read by byte does not match",
-                expected, inflateFilter.read());
-      } while (expected != -1);
-    }
-
-    LOG.info("SUCCESS! Completed checking " + count + " records");
-  }
-}

hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java

@@ -35,7 +35,6 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
 
 import org.apache.hadoop.conf.Configuration;
 
@@ -44,7 +43,6 @@
 
 import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
 import org.apache.hadoop.hdfs.web.WebHdfsConstants;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
 
 /**
  * Ensure that we can perform operations against the shaded minicluster
@@ -56,7 +54,6 @@ public class ITUseMiniCluster {
       LoggerFactory.getLogger(ITUseMiniCluster.class);
 
   private MiniDFSCluster cluster;
-  private MiniYARNCluster yarnCluster;
 
   private static final String TEST_PATH = "/foo/bar/cats/dee";
   private static final String FILENAME = "test.file";
@@ -76,12 +73,6 @@ public void clusterUp() throws IOException {
         .numDataNodes(3)
         .build();
     cluster.waitActive();
-
-    conf.set("yarn.scheduler.capacity.root.queues", "default");
-    conf.setInt("yarn.scheduler.capacity.root.default.capacity", 100);
-    yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1, 1);
-    yarnCluster.init(conf);
-    yarnCluster.start();
   }
 
   @After
@@ -89,7 +80,6 @@ public void clusterDown() {
     if (cluster != null) {
       cluster.close();
     }
-    IOUtils.cleanupWithLogger(LOG, yarnCluster);
   }
 
   @Test

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -18,11 +18,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-client-minicluster</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <description>Apache Hadoop Minicluster for Clients</description>
@@ -40,12 +40,6 @@
       <artifactId>hadoop-client-api</artifactId>
       <scope>runtime</scope>
     </dependency>
-    <!-- This is the api's compile dependency, but we don't want it to be compile dependency here too. -->
-    <dependency>
-      <groupId>org.xerial.snappy</groupId>
-      <artifactId>snappy-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-runtime</artifactId>
@@ -149,7 +143,7 @@
           <artifactId>hadoop-yarn-common</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>${leveldbjni.group}</groupId>
+          <groupId>org.fusesource.leveldbjni</groupId>
           <artifactId>leveldbjni-all</artifactId>
         </exclusion>
         <exclusion>
@@ -328,14 +322,6 @@
           <groupId>dnsjava</groupId>
           <artifactId>dnsjava</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop.thirdparty</groupId>
-          <artifactId>hadoop-shaded-guava</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.ow2.asm</groupId>
-          <artifactId>asm-commons</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <!-- Add optional runtime dependency on the in-development timeline server module
@@ -407,8 +393,8 @@
     <!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
     <!-- Skip jetty-util because it's in client -->
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-core</artifactId>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
       <optional>true</optional>
       <exclusions>
         <exclusion>
@@ -423,26 +409,30 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>com.github.pjfanning</groupId>
+      <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-json</artifactId>
       <optional>true</optional>
       <exclusions>
-        <exclusion>
-          <groupId>com.fasterxml.jackson.core</groupId>
-          <artifactId>jackson-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.fasterxml.jackson.core</groupId>
-          <artifactId>jackson-databind</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.fasterxml.jackson.jaxrs</groupId>
-          <artifactId>jackson-jaxrs-json-provider</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>javax.xml.bind</groupId>
           <artifactId>jaxb-api</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-core-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-mapper-asl</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-jaxrs</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.codehaus.jackson</groupId>
+          <artifactId>jackson-xc</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -451,23 +441,9 @@
       <optional>true</optional>
     </dependency>
     <dependency>
-      <groupId>com.sun.jersey</groupId>
-      <artifactId>jersey-servlet</artifactId>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-servlet</artifactId>
       <optional>true</optional>
-      <exclusions>
-        <exclusion>
-          <groupId>javax.servlet</groupId>
-          <artifactId>servlet-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>javax.enterprise</groupId>
-          <artifactId>cdi-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>ch.qos.cal10n</groupId>
-          <artifactId>cal10n-api</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
     <dependency>
@@ -508,17 +484,13 @@
           <artifactId>hadoop-yarn-server-common</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>${leveldbjni.group}</groupId>
+          <groupId>org.fusesource.leveldbjni</groupId>
           <artifactId>leveldbjni-all</artifactId>
         </exclusion>
         <exclusion>
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-util</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop.thirdparty</groupId>
-          <artifactId>hadoop-shaded-guava</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.google.guava</groupId>
           <artifactId>guava</artifactId>
@@ -574,10 +546,6 @@
           <groupId>com.google.guava</groupId>
           <artifactId>guava</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.apache.hadoop.thirdparty</groupId>
-          <artifactId>hadoop-shaded-guava</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>
@@ -671,6 +639,13 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-maven-plugins</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -685,18 +660,19 @@
                       <exclude>org.apache.hadoop:hadoop-client-api</exclude>
                       <exclude>org.apache.hadoop:hadoop-client-runtime</exclude>
                       <!-- Fine to expose our purposefully not-shaded deps as dependencies -->
+                      <exclude>org.apache.htrace:htrace-core4</exclude>
                       <exclude>org.slf4j:slf4j-api</exclude>
                       <exclude>commons-logging:commons-logging</exclude>
                       <exclude>junit:junit</exclude>
                       <exclude>com.google.code.findbugs:jsr305</exclude>
                       <exclude>log4j:log4j</exclude>
-                      <exclude>org.eclipse.jetty.websocket:websocket-common</exclude>
-                      <exclude>org.eclipse.jetty.websocket:websocket-api</exclude>
+                      <exclude>org.eclipse.jetty.websocket:*</exclude>
+                      <exclude>javax.websocket:javax.websocket-api</exclude>
+                      <exclude>javax.annotation:javax.annotation-api</exclude>
+                      <exclude>org.eclipse.jetty:jetty-jndi</exclude>
                       <!-- We need a filter that matches just those things that are included in the above artiacts -->
                       <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                       <exclude>org.bouncycastle:*</exclude>
-                      <!-- Leave snappy that includes native methods which cannot be relocated. -->
-                      <exclude>org.xerial.snappy:*</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -743,16 +719,11 @@
                         <exclude>testdata/*</exclude>
                       </excludes>
                     </filter>
+                    <!-- Skip terminal html and javascript -->
                     <filter>
-                      <artifact>com.fasterxml.jackson.*:*</artifact>
+                      <artifact>org.apache.hadoop:hadoop-yarn-server-nodemanager:*</artifact>
                       <excludes>
-                        <exclude>META-INF/versions/11/module-info.class</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>com.google.code.gson:gson</artifact>
-                      <excludes>
-                        <exclude>META-INF/versions/9/module-info.class</exclude>
+                        <exclude>TERMINAL/**/*</exclude>
                       </excludes>
                     </filter>
 
@@ -787,16 +758,16 @@
                         <exclude>xml.xsd</exclude>
                       </excludes>
                     </filter>
+                    <!-- filtering HISTORY-JAVA.md from rocksdb jar -->
                     <filter>
-                      <!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
-                      <artifact>org.eclipse.jetty:*</artifact>
-                      <excludes>
-                        <exclude>about.html</exclude>
-                      </excludes>
+                    <artifact>org.rocksdb:rocksdbjni</artifact>
+                    <excludes>
+                      <exclude>HISTORY-JAVA.md</exclude>
+                    </excludes>
                     </filter>
                     <filter>
                       <!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
-                      <artifact>org.eclipse.jetty.websocket:*</artifact>
+                      <artifact>org.eclipse.jetty:*</artifact>
                       <excludes>
                         <exclude>about.html</exclude>
                       </excludes>
@@ -840,37 +811,6 @@
                         <exclude>*/**</exclude>
                       </excludes>
                     </filter>
-                    <!-- Jetty 9.4.x: jetty-client and jetty-xml are depended by org.eclipse.jetty.websocket:websocket-client.-->
-                    <filter>
-                      <artifact>org.eclipse.jetty:jetty-client</artifact>
-                      <excludes>
-                        <exclude>*/**</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>org.eclipse.jetty:jetty-xml</artifact>
-                      <excludes>
-                        <exclude>*/**</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>org.eclipse.jetty:jetty-http</artifact>
-                      <excludes>
-                        <exclude>*/**</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>org.eclipse.jetty:jetty-util-ajax</artifact>
-                      <excludes>
-                        <exclude>*/**</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>org.eclipse.jetty:jetty-server</artifact>
-                      <excludes>
-                        <exclude>jetty-dir.css</exclude>
-                      </excludes>
-                    </filter>
                   </filters>
 
                   <!-- relocate classes from mssql-jdbc -->
@@ -889,7 +829,9 @@
                       <excludes>
                         <exclude>org/apache/hadoop/*</exclude>
                         <exclude>org/apache/hadoop/**/*</exclude>
-                        <!-- Our non-shaded logging libraries -->
+                        <!-- Our non-shaded htrace and logging libraries -->
+                        <exclude>org/apache/htrace/*</exclude>
+                        <exclude>org/apache/htrace/**/*</exclude>
                         <exclude>org/slf4j/*</exclude>
                         <exclude>org/slf4j/**/*</exclude>
                         <exclude>org/apache/commons/logging/*</exclude>
@@ -908,11 +850,6 @@
                         <exclude>org/w3c/dom/**/*</exclude>
                         <exclude>org/xml/sax/*</exclude>
                         <exclude>org/xml/sax/**/*</exclude>
-                        <exclude>org/bouncycastle/*</exclude>
-                        <exclude>org/bouncycastle/**/*</exclude>
-                        <!-- Exclude snappy-java -->
-                        <exclude>org/xerial/snappy/*</exclude>
-                        <exclude>org/xerial/snappy/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -938,8 +875,6 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
-                        <exclude>com/ibm/security/*</exclude>
-                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -1004,20 +939,6 @@
                         <exclude>**/pom.xml</exclude>
                       </excludes>
                     </relocation>
-                    <relocation>
-                      <pattern>javax/websocket/</pattern>
-                      <shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern>
-                      <excludes>
-                        <exclude>**/pom.xml</exclude>
-                      </excludes>
-                    </relocation>
-                    <relocation>
-                      <pattern>javax/annotation/</pattern>
-                      <shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern>
-                      <excludes>
-                        <exclude>**/pom.xml</exclude>
-                      </excludes>
-                    </relocation>
                     <relocation>
                       <pattern>jersey/</pattern>
                       <shadedPattern>${shaded.dependency.prefix}.jersey.</shadedPattern>
@@ -1033,9 +954,6 @@
                         <!-- Exclude config keys for Hadoop that look like package names -->
                         <exclude>net/topology/*</exclude>
                         <exclude>net/topology/**/*</exclude>
-                        <!-- Exclude lz4-java -->
-                        <exclude>net/jpountz/*</exclude>
-                        <exclude>net/jpountz/**/*</exclude>
                       </excludes>
                     </relocation>
                     <!-- okio declares a top level package instead of nested -->
@@ -1045,7 +963,8 @@
                     </relocation>
                   </relocations>
                   <transformers>
-                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                    <!-- Needed until MSHADE-182 -->
+                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -18,11 +18,11 @@
 <parent>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-project</artifactId>
-   <version>3.4.0-SNAPSHOT</version>
+   <version>3.3.0-SNAPSHOT</version>
    <relativePath>../../hadoop-project</relativePath>
 </parent>
   <artifactId>hadoop-client-runtime</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <description>Apache Hadoop Client</description>
@@ -60,12 +60,6 @@
       <artifactId>hadoop-client-api</artifactId>
       <scope>runtime</scope>
     </dependency>
-    <!-- This is the api's compile dependency, but we don't want it to be compile dependency here too. -->
-    <dependency>
-      <groupId>org.xerial.snappy</groupId>
-      <artifactId>snappy-java</artifactId>
-      <scope>runtime</scope>
-    </dependency>
     <!-- This comes from our parent pom. If we don't expressly change it here to get included,
          downstream will get warnings at compile time. -->
     <dependency>
@@ -81,9 +75,15 @@
     </dependency>
     <!-- Since hadoop-client is listed as optional, we have to list transitive
          dependencies that we still want to show up.
+         * HTrace
          * Slf4j API
          * commons-logging
       -->
+    <dependency>
+      <groupId>org.apache.htrace</groupId>
+      <artifactId>htrace-core4</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -128,6 +128,13 @@
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-shade-plugin</artifactId>
+            <dependencies>
+              <dependency>
+                <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-maven-plugins</artifactId>
+                <version>${project.version}</version>
+              </dependency>
+            </dependencies>
             <executions>
               <execution>
                 <phase>package</phase>
@@ -139,6 +146,8 @@
                     <excludes>
                       <!-- We need a filter that matches just those things that aer included in the api jar -->
                       <exclude>org.apache.hadoop:hadoop-client-api</exclude>
+                      <!-- Leave HTrace as an unshaded dependency on purpose, since a static class member is used to trace within a given JVM instance -->
+                      <exclude>org.apache.htrace:htrace-core4</exclude>
                       <!-- Leave slf4j unshaded so downstream users can configure logging. -->
                       <exclude>org.slf4j:slf4j-api</exclude>
                       <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
@@ -148,17 +157,13 @@
                       <!-- Leave javax APIs that are stable -->
                       <!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
                       <exclude>com.google.code.findbugs:jsr305</exclude>
-                      <exclude>io.netty:*</exclude>
                       <exclude>io.dropwizard.metrics:metrics-core</exclude>
+                      <exclude>org.eclipse.jetty.websocket:*</exclude>
                       <exclude>org.eclipse.jetty:jetty-servlet</exclude>
                       <exclude>org.eclipse.jetty:jetty-security</exclude>
                       <exclude>org.ow2.asm:*</exclude>
                       <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
                       <exclude>org.bouncycastle:*</exclude>
-                      <!-- Leave snappy that includes native methods which cannot be relocated. -->
-                      <exclude>org.xerial.snappy:*</exclude>
-                      <!-- leave out kotlin classes -->
-                      <exclude>org.jetbrains.kotlin:*</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>
@@ -205,13 +210,6 @@
                         <exclude>about.html</exclude>
                       </excludes>
                     </filter>
-                    <filter>
-                      <!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
-                      <artifact>org.eclipse.jetty.websocket:*</artifact>
-                      <excludes>
-                        <exclude>about.html</exclude>
-                      </excludes>
-                    </filter>
                     <filter>
                       <!-- skip docs on formats used in kerby -->
                       <artifact>org.apache.kerby:kerb-util</artifact>
@@ -231,26 +229,6 @@
                         <exclude>update*</exclude>
                       </excludes>
                     </filter>
-                    <filter>
-                      <artifact>com.google.protobuf:protobuf-java</artifact>
-                      <excludes>
-                        <exclude>google/protobuf/*.proto</exclude>
-                        <exclude>google/protobuf/**/*.proto</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>com.fasterxml.jackson.*:*</artifact>
-                      <excludes>
-                        <exclude>META-INF/versions/11/module-info.class</exclude>
-                      </excludes>
-                    </filter>
-                    <filter>
-                      <artifact>com.google.code.gson:gson</artifact>
-                      <excludes>
-                        <exclude>META-INF/versions/9/module-info.class</exclude>
-                      </excludes>
-                    </filter>
-
                   </filters>
                   <relocations>
                     <relocation>
@@ -259,7 +237,9 @@
                       <excludes>
                         <exclude>org/apache/hadoop/*</exclude>
                         <exclude>org/apache/hadoop/**/*</exclude>
-                        <!-- Our non-shaded logging libraries -->
+                        <!-- Our non-shaded htrace and logging libraries -->
+                        <exclude>org/apache/htrace/*</exclude>
+                        <exclude>org/apache/htrace/**/*</exclude>
                         <exclude>org/slf4j/*</exclude>
                         <exclude>org/slf4j/**/*</exclude>
                         <exclude>org/apache/commons/logging/*</exclude>
@@ -274,11 +254,6 @@
                         <exclude>org/w3c/dom/**/*</exclude>
                         <exclude>org/xml/sax/*</exclude>
                         <exclude>org/xml/sax/**/*</exclude>
-                        <exclude>org/bouncycastle/*</exclude>
-                        <exclude>org/bouncycastle/**/*</exclude>
-                        <!-- Exclude snappy-java -->
-                        <exclude>org/xerial/snappy/*</exclude>
-                        <exclude>org/xerial/snappy/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -297,8 +272,6 @@
                         <exclude>com/sun/security/**/*</exclude>
                         <exclude>com/sun/jndi/**/*</exclude>
                         <exclude>com/sun/management/**/*</exclude>
-                        <exclude>com/ibm/security/*</exclude>
-                        <exclude>com/ibm/security/**/*</exclude>
                       </excludes>
                     </relocation>
                     <relocation>
@@ -356,13 +329,6 @@
                         <exclude>**/pom.xml</exclude>
                       </excludes>
                     </relocation>
-                    <relocation>
-                      <pattern>javax/xml/bind/</pattern>
-                      <shadedPattern>${shaded.dependency.prefix}.javax.xml.bind.</shadedPattern>
-                      <excludes>
-                        <exclude>**/pom.xml</exclude>
-                      </excludes>
-                    </relocation>
                     <relocation>
                       <pattern>net/</pattern>
                       <shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
@@ -371,9 +337,6 @@
                         <!-- Exclude config keys for Hadoop that look like package names -->
                         <exclude>net/topology/*</exclude>
                         <exclude>net/topology/**/*</exclude>
-                        <!-- Exclude lz4-java -->
-                        <exclude>net/jpountz/*</exclude>
-                        <exclude>net/jpountz/**/*</exclude>
                       </excludes>
                     </relocation>
                     <!-- okio declares a top level package instead of nested -->
@@ -393,7 +356,8 @@
     -->
                   </relocations>
                   <transformers>
-                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                    <!-- Needed until MSHADE-182 -->
+                    <transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
                     <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
                       <resources>

hadoop-client-modules/hadoop-client/pom.xml

@@ -18,11 +18,11 @@
 <parent>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-project-dist</artifactId>
-   <version>3.4.0-SNAPSHOT</version>
+   <version>3.3.0-SNAPSHOT</version>
    <relativePath>../../hadoop-project-dist</relativePath>
 </parent>
   <artifactId>hadoop-client</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
 
   <description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
   <name>Apache Hadoop Client Aggregator</name>
@@ -66,13 +66,9 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.github.pjfanning</groupId>
+          <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jettison</groupId>
-          <artifactId>jettison</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-server</artifactId>
@@ -118,18 +114,6 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-server</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.jetbrains.kotlin</groupId>
-          <artifactId>kotlin-stdlib-common</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.squareup.okhttp3</groupId>
-          <artifactId>okhttp</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-core</artifactId>
@@ -183,13 +167,9 @@
           <artifactId>jersey-core</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.github.pjfanning</groupId>
+          <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jettison</groupId>
-          <artifactId>jettison</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
@@ -238,13 +218,9 @@
           <artifactId>jersey-server</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.github.pjfanning</groupId>
+          <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jettison</groupId>
-          <artifactId>jettison</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-servlet</artifactId>
@@ -299,13 +275,9 @@
           <artifactId>guice-servlet</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>com.github.pjfanning</groupId>
+          <groupId>com.sun.jersey</groupId>
           <artifactId>jersey-json</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.codehaus.jettison</groupId>
-          <artifactId>jettison</artifactId>
-        </exclusion>
         <exclusion>
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>

hadoop-client-modules/pom.xml

@@ -18,7 +18,7 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-client-modules</artifactId>

hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

@@ -18,11 +18,11 @@
   <parent>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
+    <version>3.3.0-SNAPSHOT</version>
     <relativePath>../../hadoop-project</relativePath>
   </parent>
   <artifactId>hadoop-cloud-storage</artifactId>
-  <version>3.4.0-SNAPSHOT</version>
+  <version>3.3.0-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <description>Apache Hadoop Cloud Storage</description>
@@ -101,10 +101,6 @@
           <groupId>org.apache.zookeeper</groupId>
           <artifactId>zookeeper</artifactId>
         </exclusion>
-        <exclusion>
-          <groupId>org.projectlombok</groupId>
-          <artifactId>lombok</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -129,12 +125,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-cos</artifactId>
-      <scope>compile</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-huaweicloud</artifactId>
+      <artifactId>hadoop-openstack</artifactId>
       <scope>compile</scope>
     </dependency>
   </dependencies>

hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml

@@ -1,23 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<FindBugsFilter>
-  <Match>
-    <Class name="org.apache.hadoop.fs.cosn.CosNInputStream$ReadBuffer"/>
-    <Method name="getBuffer"/>
-    <Bug pattern="EI_EXPOSE_REP"/>
-  </Match>
-</FindBugsFilter>

hadoop-cloud-storage-project/hadoop-cos/pom.xml

@@ -1,155 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-project</artifactId>
-    <version>3.4.0-SNAPSHOT</version>
-    <relativePath>../../hadoop-project</relativePath>
-  </parent>
-  <artifactId>hadoop-cos</artifactId>
-  <name>Apache Hadoop Tencent COS Support</name>
-  <description>
-    This module contains code to support integration with Tencent COS.
-    It also declares the dependencies needed to work with COS.
-  </description>
-  <packaging>jar</packaging>
-
-  <properties>
-    <file.encoding>UTF-8</file.encoding>
-    <downloadSources>true</downloadSources>
-  </properties>
-
-  <profiles>
-    <profile>
-      <id>tests-off</id>
-      <activation>
-        <file>
-          <missing>src/test/resources/auth-keys.xml</missing>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>true</maven.test.skip>
-      </properties>
-    </profile>
-    <profile>
-      <id>tests-on</id>
-      <activation>
-        <file>
-          <exists>src/test/resources/auth-keys.xml</exists>
-        </file>
-      </activation>
-      <properties>
-        <maven.test.skip>false</maven.test.skip>
-      </properties>
-    </profile>
-  </profiles>
-
-  <build>
-    <plugins>
-      <plugin>
-        <groupId>com.github.spotbugs</groupId>
-        <artifactId>spotbugs-maven-plugin</artifactId>
-        <configuration>
-          <xmlOutput>true</xmlOutput>
-          <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
-          </excludeFilterFile>
-          <effort>Max</effort>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-surefire-plugin</artifactId>
-        <configuration>
-          <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
-        </configuration>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-dependency-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>deplist</id>
-            <phase>compile</phase>
-            <goals>
-              <goal>list</goal>
-            </goals>
-            <configuration>
-              <outputFile>${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt</outputFile>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-    </plugins>
-  </build>
-
-  <dependencies>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>com.qcloud</groupId>
-      <artifactId>cos_api-bundle</artifactId>
-      <version>5.6.69</version>
-      <scope>compile</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>provided</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-yarn-server-tests</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-mapreduce-client-hs</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-distcp</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-distcp</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-
-  </dependencies>
-</project>

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java

@@ -1,240 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * BufferPool class is used to manage the buffers during program execution.
- * It is provided in a thread-safe singleton mode,and
- * keeps the program's memory and disk consumption at a stable value.
- */
-public final class BufferPool {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BufferPool.class);
-
-  private static BufferPool ourInstance = new BufferPool();
-
-  /**
-   * Use this method to get the instance of BufferPool.
-   *
-   * @return the instance of BufferPool
-   */
-  public static BufferPool getInstance() {
-    return ourInstance;
-  }
-
-  private BlockingQueue<ByteBuffer> bufferPool = null;
-  private long singleBufferSize = 0;
-  private File diskBufferDir = null;
-
-  private AtomicBoolean isInitialize = new AtomicBoolean(false);
-
-  private BufferPool() {
-  }
-
-  private File createDir(String dirPath) throws IOException {
-    File dir = new File(dirPath);
-    if (!dir.exists()) {
-      LOG.debug("Buffer dir: [{}] does not exists. create it first.",
-          dirPath);
-      if (dir.mkdirs()) {
-        if (!dir.setWritable(true) || !dir.setReadable(true)
-            || !dir.setExecutable(true)) {
-          LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
-              + "readable, executable] failed.", dir.getAbsolutePath());
-        }
-        LOG.debug("Buffer dir: [{}] is created successfully.",
-            dir.getAbsolutePath());
-      } else {
-        // Once again, check if it has been created successfully.
-        // Prevent problems created by multiple processes at the same time.
-        if (!dir.exists()) {
-          throw new IOException("buffer dir:" + dir.getAbsolutePath()
-              + " is created unsuccessfully");
-        }
-      }
-    } else {
-      LOG.debug("buffer dir: {} already exists.", dirPath);
-    }
-
-    return dir;
-  }
-
-  /**
-   * Create buffers correctly by reading the buffer file directory,
-   * buffer pool size,and file block size in the configuration.
-   *
-   * @param conf Provides configurations for the Hadoop runtime
-   * @throws IOException Configuration errors,
-   *                     insufficient or no access for memory or
-   *                     disk space may cause this exception
-   */
-  public synchronized void initialize(Configuration conf)
-      throws IOException {
-    if (this.isInitialize.get()) {
-      return;
-    }
-    this.singleBufferSize = conf.getLong(CosNConfigKeys.COSN_BLOCK_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_BLOCK_SIZE);
-
-    // The block size of CosN can only support up to 2GB.
-    if (this.singleBufferSize < Constants.MIN_PART_SIZE
-        || this.singleBufferSize > Constants.MAX_PART_SIZE) {
-      String exceptionMsg = String.format(
-          "The block size of CosN is limited to %d to %d",
-          Constants.MIN_PART_SIZE, Constants.MAX_PART_SIZE);
-      throw new IOException(exceptionMsg);
-    }
-
-    long memoryBufferLimit = conf.getLong(
-        CosNConfigKeys.COSN_UPLOAD_BUFFER_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_UPLOAD_BUFFER_SIZE);
-
-    this.diskBufferDir = this.createDir(conf.get(
-        CosNConfigKeys.COSN_BUFFER_DIR_KEY,
-        CosNConfigKeys.DEFAULT_BUFFER_DIR));
-
-    int bufferPoolSize = (int) (memoryBufferLimit / this.singleBufferSize);
-    if (0 == bufferPoolSize) {
-      throw new IOException(
-          String.format("The total size of the buffer [%d] is " +
-                  "smaller than a single block [%d]."
-                  + "please consider increase the buffer size " +
-                  "or decrease the block size",
-              memoryBufferLimit, this.singleBufferSize));
-    }
-    this.bufferPool = new LinkedBlockingQueue<>(bufferPoolSize);
-    for (int i = 0; i < bufferPoolSize; i++) {
-      this.bufferPool.add(ByteBuffer.allocateDirect(
-          (int) this.singleBufferSize));
-    }
-
-    this.isInitialize.set(true);
-  }
-
-  /**
-   * Check if the buffer pool has been initialized.
-   *
-   * @throws IOException if the buffer pool is not initialized
-   */
-  private void checkInitialize() throws IOException {
-    if (!this.isInitialize.get()) {
-      throw new IOException(
-          "The buffer pool has not been initialized yet");
-    }
-  }
-
-  /**
-   * Obtain a buffer from this buffer pool through the method.
-   *
-   * @param bufferSize expected buffer size to get
-   * @return a buffer wrapper that satisfies the bufferSize.
-   * @throws IOException if the buffer pool not initialized,
-   *                     or the bufferSize parameter is not within
-   *                     the range[1MB to the single buffer size]
-   */
-  public ByteBufferWrapper getBuffer(int bufferSize) throws IOException {
-    this.checkInitialize();
-    if (bufferSize > 0 && bufferSize <= this.singleBufferSize) {
-      ByteBufferWrapper byteBufferWrapper = this.getByteBuffer();
-      if (null == byteBufferWrapper) {
-        // Use a disk buffer when the memory buffer is not enough
-        byteBufferWrapper = this.getMappedBuffer();
-      }
-      return byteBufferWrapper;
-    } else {
-      String exceptionMsg = String.format(
-          "Parameter buffer size out of range: 1048576 to %d",
-          this.singleBufferSize
-      );
-      throw new IOException(exceptionMsg);
-    }
-  }
-
-  /**
-   * Get a ByteBufferWrapper from the buffer pool.
-   *
-   * @return a new byte buffer wrapper
-   * @throws IOException if the buffer pool is not initialized
-   */
-  private ByteBufferWrapper getByteBuffer() throws IOException {
-    this.checkInitialize();
-    ByteBuffer buffer = this.bufferPool.poll();
-    return buffer == null ? null : new ByteBufferWrapper(buffer);
-  }
-
-  /**
-   * Get a mapped buffer from the buffer pool.
-   *
-   * @return a new mapped buffer
-   * @throws IOException If the buffer pool is not initialized.
-   *                     or some I/O error occurs
-   */
-  private ByteBufferWrapper getMappedBuffer() throws IOException {
-    this.checkInitialize();
-    File tmpFile = File.createTempFile(Constants.BLOCK_TMP_FILE_PREFIX,
-        Constants.BLOCK_TMP_FILE_SUFFIX, this.diskBufferDir);
-    tmpFile.deleteOnExit();
-    RandomAccessFile raf = new RandomAccessFile(tmpFile, "rw");
-    raf.setLength(this.singleBufferSize);
-    MappedByteBuffer buf = raf.getChannel().map(
-        FileChannel.MapMode.READ_WRITE, 0, this.singleBufferSize);
-    return new ByteBufferWrapper(buf, raf, tmpFile);
-  }
-
-  /**
-   * return the byte buffer wrapper to the buffer pool.
-   *
-   * @param byteBufferWrapper the byte buffer wrapper getting from the pool
-   * @throws InterruptedException if interrupted while waiting
-   * @throws IOException          some io error occurs
-   */
-  public void returnBuffer(ByteBufferWrapper byteBufferWrapper)
-      throws InterruptedException, IOException {
-    if (null == this.bufferPool || null == byteBufferWrapper) {
-      return;
-    }
-
-    if (byteBufferWrapper.isDiskBuffer()) {
-      byteBufferWrapper.close();
-    } else {
-      ByteBuffer byteBuffer = byteBufferWrapper.getByteBuffer();
-      if (null != byteBuffer) {
-        byteBuffer.clear();
-        LOG.debug("Return the buffer to the buffer pool.");
-        if (!this.bufferPool.offer(byteBuffer)) {
-          LOG.error("Return the buffer to buffer pool failed.");
-        }
-      }
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/ByteBufferInputStream.java

@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.nio.InvalidMarkException;
-
-/**
- * The input stream class is used for buffered files.
- * The purpose of providing this class is to optimize buffer read performance.
- */
-public class ByteBufferInputStream extends InputStream {
-  private ByteBuffer byteBuffer;
-  private boolean isClosed;
-
-  public ByteBufferInputStream(ByteBuffer byteBuffer) throws IOException {
-    if (null == byteBuffer) {
-      throw new IOException("byte buffer is null");
-    }
-    this.byteBuffer = byteBuffer;
-    this.isClosed = false;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (null == this.byteBuffer) {
-      throw new IOException("this byte buffer for InputStream is null");
-    }
-    if (!this.byteBuffer.hasRemaining()) {
-      return -1;
-    }
-    return this.byteBuffer.get() & 0xFF;
-  }
-
-  @Override
-  public synchronized void mark(int readLimit) {
-    if (!this.markSupported()) {
-      return;
-    }
-    this.byteBuffer.mark();
-    // Parameter readLimit is ignored
-  }
-
-  @Override
-  public boolean markSupported() {
-    return true;
-  }
-
-  @Override
-  public synchronized void reset() throws IOException {
-    if (this.isClosed) {
-      throw new IOException("Closed in InputStream");
-    }
-    try {
-      this.byteBuffer.reset();
-    } catch (InvalidMarkException e) {
-      throw new IOException("Invalid mark");
-    }
-  }
-
-  @Override
-  public int available() {
-    return this.byteBuffer.remaining();
-  }
-
-  @Override
-  public void close() {
-    this.byteBuffer.rewind();
-    this.byteBuffer = null;
-    this.isClosed = true;
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/ByteBufferOutputStream.java

@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-
-/**
- * The input stream class is used for buffered files.
- * The purpose of providing this class is to optimize buffer write performance.
- */
-public class ByteBufferOutputStream extends OutputStream {
-  private ByteBuffer byteBuffer;
-  private boolean isFlush;
-  private boolean isClosed;
-
-  public ByteBufferOutputStream(ByteBuffer byteBuffer) throws IOException {
-    if (null == byteBuffer) {
-      throw new IOException("byte buffer is null");
-    }
-    this.byteBuffer = byteBuffer;
-    this.byteBuffer.clear();
-    this.isFlush = false;
-    this.isClosed = false;
-  }
-
-  @Override
-  public void write(int b) {
-    byte[] singleBytes = new byte[1];
-    singleBytes[0] = (byte) b;
-    this.byteBuffer.put(singleBytes, 0, 1);
-    this.isFlush = false;
-  }
-
-  @Override
-  public void flush() {
-    if (this.isFlush) {
-      return;
-    }
-    this.isFlush = true;
-  }
-
-  @Override
-  public void close() throws IOException {
-    if (this.isClosed) {
-      return;
-    }
-    if (null == this.byteBuffer) {
-      throw new IOException("Can not close a null object");
-    }
-
-    this.flush();
-    this.byteBuffer.flip();
-    this.byteBuffer = null;
-    this.isFlush = false;
-    this.isClosed = true;
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/ByteBufferWrapper.java

@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.util.CleanerUtil;
-
-/**
- * The wrapper for memory buffers and disk buffers.
- */
-public class ByteBufferWrapper {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ByteBufferWrapper.class);
-  private ByteBuffer byteBuffer;
-  private File file;
-  private RandomAccessFile randomAccessFile;
-
-  ByteBufferWrapper(ByteBuffer byteBuffer) {
-    this(byteBuffer, null, null);
-  }
-
-  ByteBufferWrapper(ByteBuffer byteBuffer, RandomAccessFile randomAccessFile,
-      File file) {
-    this.byteBuffer = byteBuffer;
-    this.file = file;
-    this.randomAccessFile = randomAccessFile;
-  }
-
-  public ByteBuffer getByteBuffer() {
-    return this.byteBuffer;
-  }
-
-  boolean isDiskBuffer() {
-    return this.file != null && this.randomAccessFile != null;
-  }
-
-  private void munmap(MappedByteBuffer buffer) {
-    if (CleanerUtil.UNMAP_SUPPORTED) {
-      try {
-        CleanerUtil.getCleaner().freeBuffer(buffer);
-      } catch (IOException e) {
-        LOG.warn("Failed to unmap the buffer", e);
-      }
-    } else {
-      LOG.trace(CleanerUtil.UNMAP_NOT_SUPPORTED_REASON);
-    }
-  }
-
-  void close() throws IOException {
-    if (null != this.byteBuffer) {
-      this.byteBuffer.clear();
-    }
-
-    IOException exception = null;
-    // catch all exceptions, and try to free up resources that can be freed.
-    try {
-      if (null != randomAccessFile) {
-        this.randomAccessFile.close();
-      }
-    } catch (IOException e) {
-      LOG.error("Close the random access file occurs an exception.", e);
-      exception = e;
-    }
-
-    if (this.byteBuffer instanceof MappedByteBuffer) {
-      munmap((MappedByteBuffer) this.byteBuffer);
-    }
-
-    if (null != this.file && this.file.exists()) {
-      if (!this.file.delete()) {
-        LOG.warn("Delete the tmp file: [{}] failed.",
-            this.file.getAbsolutePath());
-      }
-    }
-
-    if (null != exception) {
-      throw exception;
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/Constants.java

@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-/**
- * constant definition.
- */
-public final class Constants {
-  private Constants() {
-  }
-
-  public static final String BLOCK_TMP_FILE_PREFIX = "cos_";
-  public static final String BLOCK_TMP_FILE_SUFFIX = "_local_block";
-
-  // The maximum number of files listed in a single COS list request.
-  public static final int COS_MAX_LISTING_LENGTH = 999;
-
-  // The maximum number of parts supported by a multipart uploading.
-  public static final int MAX_PART_NUM = 10000;
-
-  // The maximum size of a part
-  public static final long MAX_PART_SIZE = (long) 2 * Unit.GB;
-  // The minimum size of a part
-  public static final long MIN_PART_SIZE = (long) Unit.MB;
-
-  public static final String COSN_SECRET_ID_ENV = "COSN_SECRET_ID";
-  public static final String COSN_SECRET_KEY_ENV = "COSN_SECRET_KEY";
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosN.java

@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DelegateToFileSystem;
-
-/**
- * CosN implementation for the Hadoop's AbstractFileSystem.
- * This implementation delegates to the CosNFileSystem {@link CosNFileSystem}.
- */
-public class CosN extends DelegateToFileSystem {
-  public CosN(URI theUri, Configuration conf)
-      throws IOException, URISyntaxException {
-    super(theUri, new CosNFileSystem(), conf, CosNFileSystem.SCHEME, false);
-  }
-
-  @Override
-  public int getUriDefaultPort() {
-    return -1;
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNConfigKeys.java

@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-
-/**
- * This class contains constants for configuration keys used in COS.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class CosNConfigKeys extends CommonConfigurationKeys {
-  public static final String USER_AGENT = "fs.cosn.user.agent";
-  public static final String DEFAULT_USER_AGENT = "cos-hadoop-plugin-v5.3";
-
-  public static final String COSN_CREDENTIALS_PROVIDER =
-      "fs.cosn.credentials.provider";
-  public static final String COSN_SECRET_ID_KEY = "fs.cosn.userinfo.secretId";
-  public static final String COSN_SECRET_KEY_KEY = "fs.cosn.userinfo.secretKey";
-  public static final String COSN_REGION_KEY = "fs.cosn.bucket.region";
-  public static final String COSN_ENDPOINT_SUFFIX_KEY =
-      "fs.cosn.bucket.endpoint_suffix";
-
-  public static final String COSN_USE_HTTPS_KEY = "fs.cosn.useHttps";
-  public static final boolean DEFAULT_USE_HTTPS = false;
-
-  public static final String COSN_BUFFER_DIR_KEY = "fs.cosn.tmp.dir";
-  public static final String DEFAULT_BUFFER_DIR = "/tmp/hadoop_cos";
-
-  public static final String COSN_UPLOAD_BUFFER_SIZE_KEY =
-      "fs.cosn.buffer.size";
-  public static final long DEFAULT_UPLOAD_BUFFER_SIZE = 32 * Unit.MB;
-
-  public static final String COSN_BLOCK_SIZE_KEY = "fs.cosn.block.size";
-  public static final long DEFAULT_BLOCK_SIZE = 8 * Unit.MB;
-
-  public static final String COSN_MAX_RETRIES_KEY = "fs.cosn.maxRetries";
-  public static final int DEFAULT_MAX_RETRIES = 3;
-  public static final String COSN_RETRY_INTERVAL_KEY =
-      "fs.cosn.retry.interval.seconds";
-  public static final long DEFAULT_RETRY_INTERVAL = 3;
-
-  public static final String UPLOAD_THREAD_POOL_SIZE_KEY =
-      "fs.cosn.upload_thread_pool";
-  public static final int DEFAULT_UPLOAD_THREAD_POOL_SIZE = 1;
-
-  public static final String COPY_THREAD_POOL_SIZE_KEY =
-      "fs.cosn.copy_thread_pool";
-  public static final int DEFAULT_COPY_THREAD_POOL_SIZE = 1;
-
-  /**
-   * This is the maximum time that excess idle threads will wait for new tasks
-   * before terminating. The time unit for it is second.
-   */
-  public static final String THREAD_KEEP_ALIVE_TIME_KEY =
-      "fs.cosn.threads.keep_alive_time";
-  // The default keep_alive_time is 60 seconds.
-  public static final long DEFAULT_THREAD_KEEP_ALIVE_TIME = 60L;
-
-  public static final String READ_AHEAD_BLOCK_SIZE_KEY =
-      "fs.cosn.read.ahead.block.size";
-  public static final long DEFAULT_READ_AHEAD_BLOCK_SIZE = 512 * Unit.KB;
-  public static final String READ_AHEAD_QUEUE_SIZE =
-      "fs.cosn.read.ahead.queue.size";
-  public static final int DEFAULT_READ_AHEAD_QUEUE_SIZE = 5;
-
-  public static final String MAX_CONNECTION_NUM = "fs.cosn.max.connection.num";
-  public static final int DEFAULT_MAX_CONNECTION_NUM = 2048;
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNCopyFileContext.java

@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * The context of the copy task, including concurrency control,
- * asynchronous acquisition of copy results and etc.
- */
-public class CosNCopyFileContext {
-
-  private final ReentrantLock lock = new ReentrantLock();
-  private Condition readyCondition = lock.newCondition();
-
-  private AtomicBoolean copySuccess = new AtomicBoolean(true);
-  private AtomicInteger copiesFinish = new AtomicInteger(0);
-
-  public void lock() {
-    this.lock.lock();
-  }
-
-  public void unlock() {
-    this.lock.unlock();
-  }
-
-  public void awaitAllFinish(int waitCopiesFinish) throws InterruptedException {
-    while (this.copiesFinish.get() != waitCopiesFinish) {
-      this.readyCondition.await();
-    }
-  }
-
-  public void signalAll() {
-    this.readyCondition.signalAll();
-  }
-
-  public boolean isCopySuccess() {
-    return this.copySuccess.get();
-  }
-
-  public void setCopySuccess(boolean copySuccess) {
-    this.copySuccess.set(copySuccess);
-  }
-
-  public void incCopiesFinish() {
-    this.copiesFinish.addAndGet(1);
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNCopyFileTask.java

@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Used by {@link CosNFileSystem} as an task that submitted
- * to the thread pool to accelerate the copy progress.
- * Each task is responsible for copying the source key to the destination.
- */
-public class CosNCopyFileTask implements Runnable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CosNCopyFileTask.class);
-
-  private NativeFileSystemStore store;
-  private String srcKey;
-  private String dstKey;
-  private CosNCopyFileContext cosCopyFileContext;
-
-  public CosNCopyFileTask(NativeFileSystemStore store, String srcKey,
-      String dstKey, CosNCopyFileContext cosCopyFileContext) {
-    this.store = store;
-    this.srcKey = srcKey;
-    this.dstKey = dstKey;
-    this.cosCopyFileContext = cosCopyFileContext;
-  }
-
-  @Override
-  public void run() {
-    boolean fail = false;
-    LOG.info(Thread.currentThread().getName() + "copying...");
-    try {
-      this.store.copy(srcKey, dstKey);
-    } catch (IOException e) {
-      LOG.warn("Exception thrown when copy from {} to {}, exception:{}",
-          this.srcKey, this.dstKey, e);
-      fail = true;
-    } finally {
-      this.cosCopyFileContext.lock();
-      if (fail) {
-        cosCopyFileContext.setCopySuccess(false);
-      }
-      cosCopyFileContext.incCopiesFinish();
-      cosCopyFileContext.signalAll();
-      this.cosCopyFileContext.unlock();
-    }
-  }
-
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java

@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-
-/**
- * Used by {@link CosNInputStream} as an asynchronous task
- * submitted to the thread pool.
- * Each task is responsible for reading a part of a large file.
- * It is used to pre-read the data from COS to accelerate file reading process.
- */
-public class CosNFileReadTask implements Runnable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CosNFileReadTask.class);
-
-  private final String key;
-  private final NativeFileSystemStore store;
-  private final CosNInputStream.ReadBuffer readBuffer;
-
-  private RetryPolicy retryPolicy;
-
-  public CosNFileReadTask(
-      Configuration conf,
-      String key, NativeFileSystemStore store,
-      CosNInputStream.ReadBuffer readBuffer) {
-    this.key = key;
-    this.store = store;
-    this.readBuffer = readBuffer;
-
-    RetryPolicy defaultPolicy =
-        RetryPolicies.retryUpToMaximumCountWithFixedSleep(
-            conf.getInt(
-                CosNConfigKeys.COSN_MAX_RETRIES_KEY,
-                CosNConfigKeys.DEFAULT_MAX_RETRIES),
-            conf.getLong(
-                CosNConfigKeys.COSN_RETRY_INTERVAL_KEY,
-                CosNConfigKeys.DEFAULT_RETRY_INTERVAL),
-            TimeUnit.SECONDS);
-    Map<Class<? extends Exception>, RetryPolicy> retryPolicyMap =
-        new HashMap<>();
-    retryPolicyMap.put(IOException.class, defaultPolicy);
-    retryPolicyMap.put(
-        IndexOutOfBoundsException.class, RetryPolicies.TRY_ONCE_THEN_FAIL);
-    retryPolicyMap.put(
-        NullPointerException.class, RetryPolicies.TRY_ONCE_THEN_FAIL);
-
-    this.retryPolicy = RetryPolicies.retryByException(
-        defaultPolicy, retryPolicyMap);
-  }
-
-  @Override
-  public void run() {
-    int retries = 0;
-    RetryPolicy.RetryAction retryAction;
-    try {
-      this.readBuffer.lock();
-      do {
-        try {
-          InputStream inputStream = this.store.retrieveBlock(this.key,
-              this.readBuffer.getStart(), this.readBuffer.getEnd());
-          IOUtils.readFully(inputStream, this.readBuffer.getBuffer(), 0,
-              readBuffer.getBuffer().length);
-          inputStream.close();
-          this.readBuffer.setStatus(CosNInputStream.ReadBuffer.SUCCESS);
-          break;
-        } catch (IOException e) {
-          this.readBuffer.setStatus(CosNInputStream.ReadBuffer.ERROR);
-          LOG.warn(
-              "Exception occurs when retrieve the block range start: "
-                  + String.valueOf(this.readBuffer.getStart()) + " end: "
-                  + this.readBuffer.getEnd());
-          try {
-            retryAction = this.retryPolicy.shouldRetry(
-                e, retries++, 0, true);
-            if (retryAction.action
-                == RetryPolicy.RetryAction.RetryDecision.RETRY) {
-              Thread.sleep(retryAction.delayMillis);
-            }
-          } catch (Exception e1) {
-            String errMsg = String.format("Exception occurs when retry[%s] "
-                    + "to retrieve the block range start: %s, end:%s",
-                this.retryPolicy.toString(),
-                String.valueOf(this.readBuffer.getStart()),
-                String.valueOf(this.readBuffer.getEnd()));
-            LOG.error(errMsg, e1);
-            break;
-          }
-        }
-      } while (retryAction.action ==
-          RetryPolicy.RetryAction.RetryDecision.RETRY);
-      this.readBuffer.signalAll();
-    } finally {
-      this.readBuffer.unLock();
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileSystem.java

@@ -1,814 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BufferedFSInputStream;
-import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathIOException;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
-import org.apache.hadoop.util.Progressable;
-
-/**
- * The core CosN Filesystem implementation.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public class CosNFileSystem extends FileSystem {
-  static final Logger LOG = LoggerFactory.getLogger(CosNFileSystem.class);
-
-  public static final String SCHEME = "cosn";
-  public static final String PATH_DELIMITER = Path.SEPARATOR;
-
-  private URI uri;
-  private String bucket;
-  private NativeFileSystemStore store;
-  private Path workingDir;
-  private String owner = "Unknown";
-  private String group = "Unknown";
-
-  private ExecutorService boundedIOThreadPool;
-  private ExecutorService boundedCopyThreadPool;
-
-  public CosNFileSystem() {
-  }
-
-  public CosNFileSystem(NativeFileSystemStore store) {
-    this.store = store;
-  }
-
-  /**
-   * Return the protocol scheme for the FileSystem.
-   *
-   * @return <code>cosn</code>
-   */
-  @Override
-  public String getScheme() {
-    return CosNFileSystem.SCHEME;
-  }
-
-  @Override
-  public void initialize(URI name, Configuration conf) throws IOException {
-    super.initialize(name, conf);
-    this.bucket = name.getHost();
-    if (this.store == null) {
-      this.store = createDefaultStore(conf);
-    }
-    this.store.initialize(name, conf);
-    setConf(conf);
-    this.uri = URI.create(name.getScheme() + "://" + name.getAuthority());
-    this.workingDir = new Path("/user",
-        System.getProperty("user.name")).makeQualified(
-        this.uri,
-        this.getWorkingDirectory());
-    this.owner = getOwnerId();
-    this.group = getGroupId();
-    LOG.debug("owner:" + owner + ", group:" + group);
-
-    BufferPool.getInstance().initialize(this.getConf());
-
-    // initialize the thread pool
-    int uploadThreadPoolSize = this.getConf().getInt(
-        CosNConfigKeys.UPLOAD_THREAD_POOL_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_UPLOAD_THREAD_POOL_SIZE
-    );
-    int readAheadPoolSize = this.getConf().getInt(
-        CosNConfigKeys.READ_AHEAD_QUEUE_SIZE,
-        CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE
-    );
-    int ioThreadPoolSize = uploadThreadPoolSize + readAheadPoolSize / 3;
-    long threadKeepAlive = this.getConf().getLong(
-        CosNConfigKeys.THREAD_KEEP_ALIVE_TIME_KEY,
-        CosNConfigKeys.DEFAULT_THREAD_KEEP_ALIVE_TIME
-    );
-    this.boundedIOThreadPool = BlockingThreadPoolExecutorService.newInstance(
-        ioThreadPoolSize / 2, ioThreadPoolSize,
-        threadKeepAlive, TimeUnit.SECONDS,
-        "cos-transfer-thread-pool");
-    int copyThreadPoolSize = this.getConf().getInt(
-        CosNConfigKeys.COPY_THREAD_POOL_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_COPY_THREAD_POOL_SIZE
-    );
-    this.boundedCopyThreadPool = BlockingThreadPoolExecutorService.newInstance(
-        CosNConfigKeys.DEFAULT_COPY_THREAD_POOL_SIZE, copyThreadPoolSize,
-        60L, TimeUnit.SECONDS,
-        "cos-copy-thread-pool");
-  }
-
-  private static NativeFileSystemStore createDefaultStore(Configuration conf) {
-    NativeFileSystemStore store = new CosNativeFileSystemStore();
-    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
-        conf.getInt(CosNConfigKeys.COSN_MAX_RETRIES_KEY,
-            CosNConfigKeys.DEFAULT_MAX_RETRIES),
-        conf.getLong(CosNConfigKeys.COSN_RETRY_INTERVAL_KEY,
-            CosNConfigKeys.DEFAULT_RETRY_INTERVAL),
-        TimeUnit.SECONDS);
-    Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
-        new HashMap<>();
-
-    exceptionToPolicyMap.put(IOException.class, basePolicy);
-    RetryPolicy methodPolicy = RetryPolicies.retryByException(
-        RetryPolicies.TRY_ONCE_THEN_FAIL,
-        exceptionToPolicyMap);
-    Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<>();
-    methodNameToPolicyMap.put("storeFile", methodPolicy);
-    methodNameToPolicyMap.put("rename", methodPolicy);
-
-    return (NativeFileSystemStore) RetryProxy.create(
-        NativeFileSystemStore.class, store, methodNameToPolicyMap);
-  }
-
-  private String getOwnerId() {
-    return System.getProperty("user.name");
-  }
-
-  private String getGroupId() {
-    return System.getProperty("user.name");
-  }
-
-  private String getOwnerInfo(boolean getOwnerId) {
-    String ownerInfoId = "";
-    try {
-      String userName = System.getProperty("user.name");
-      String command = "id -u " + userName;
-      if (!getOwnerId) {
-        command = "id -g " + userName;
-      }
-      Process child = Runtime.getRuntime().exec(command);
-      child.waitFor();
-
-      // Get the input stream and read from it
-      InputStream in = child.getInputStream();
-      StringBuilder strBuffer = new StringBuilder();
-      int c;
-      while ((c = in.read()) != -1) {
-        strBuffer.append((char) c);
-      }
-      in.close();
-      ownerInfoId = strBuffer.toString();
-    } catch (IOException | InterruptedException e) {
-      LOG.error("Getting owner info occurs a exception", e);
-    }
-    return ownerInfoId;
-  }
-
-  private static String pathToKey(Path path) {
-    if (path.toUri().getScheme() != null && path.toUri().getPath().isEmpty()) {
-      // allow uris without trailing slash after bucket to refer to root,
-      // like cosn://mybucket
-      return "";
-    }
-    if (!path.isAbsolute()) {
-      throw new IllegalArgumentException("Path must be absolute: " + path);
-    }
-    String ret = path.toUri().getPath();
-    if (ret.endsWith("/") && (ret.indexOf("/") != ret.length() - 1)) {
-      ret = ret.substring(0, ret.length() - 1);
-    }
-    return ret;
-  }
-
-  private static Path keyToPath(String key) {
-    if (!key.startsWith(PATH_DELIMITER)) {
-      return new Path("/" + key);
-    } else {
-      return new Path(key);
-    }
-  }
-
-  private Path makeAbsolute(Path path) {
-    if (path.isAbsolute()) {
-      return path;
-    }
-    return new Path(workingDir, path);
-  }
-
-  /**
-   * This optional operation is not yet supported.
-   */
-  @Override
-  public FSDataOutputStream append(Path f, int bufferSize,
-      Progressable progress) throws IOException {
-    throw new IOException("Not supported");
-  }
-
-  @Override
-  public FSDataOutputStream create(Path f, FsPermission permission,
-      boolean overwrite, int bufferSize, short replication, long blockSize,
-      Progressable progress) throws IOException {
-    FileStatus fileStatus;
-
-    try {
-      fileStatus = getFileStatus(f);
-      if (fileStatus.isDirectory()) {
-        throw new FileAlreadyExistsException(f + " is a directory");
-      }
-      if (!overwrite) {
-        // path references a file and overwrite is disabled
-        throw new FileAlreadyExistsException(f + " already exists");
-      }
-
-    } catch (FileNotFoundException e) {
-      LOG.debug("Creating a new file: [{}] in COS.", f);
-    }
-
-    Path absolutePath = makeAbsolute(f);
-    String key = pathToKey(absolutePath);
-    return new FSDataOutputStream(
-        new CosNOutputStream(getConf(), store, key, blockSize,
-            this.boundedIOThreadPool), statistics);
-  }
-
-  private boolean rejectRootDirectoryDelete(boolean isEmptyDir,
-      boolean recursive) throws PathIOException {
-    if (isEmptyDir) {
-      return true;
-    }
-    if (recursive) {
-      return false;
-    } else {
-      throw new PathIOException(this.bucket, "Can not delete root path");
-    }
-  }
-
-  @Override
-  public FSDataOutputStream createNonRecursive(Path f, FsPermission permission,
-      EnumSet<CreateFlag> flags, int bufferSize, short replication,
-      long blockSize, Progressable progress) throws IOException {
-    Path parent = f.getParent();
-    if (null != parent) {
-      if (!getFileStatus(parent).isDirectory()) {
-        throw new FileAlreadyExistsException("Not a directory: " + parent);
-      }
-    }
-
-    return create(f, permission, flags.contains(CreateFlag.OVERWRITE),
-        bufferSize, replication, blockSize, progress);
-  }
-
-  @Override
-  public boolean delete(Path f, boolean recursive) throws IOException {
-    LOG.debug("Ready to delete path: [{}]. recursive: [{}].", f, recursive);
-    FileStatus status;
-    try {
-      status = getFileStatus(f);
-    } catch (FileNotFoundException e) {
-      LOG.debug("Ready to delete the file: [{}], but it does not exist.", f);
-      return false;
-    }
-    Path absolutePath = makeAbsolute(f);
-    String key = pathToKey(absolutePath);
-    if (key.compareToIgnoreCase("/") == 0) {
-      FileStatus[] fileStatuses = listStatus(f);
-      return this.rejectRootDirectoryDelete(
-          fileStatuses.length == 0, recursive);
-    }
-
-    if (status.isDirectory()) {
-      if (!key.endsWith(PATH_DELIMITER)) {
-        key += PATH_DELIMITER;
-      }
-      if (!recursive && listStatus(f).length > 0) {
-        String errMsg = String.format("Can not delete the directory: [%s], as"
-            + " it is not empty and option recursive is false.", f);
-        throw new IOException(errMsg);
-      }
-
-      createParent(f);
-
-      String priorLastKey = null;
-      do {
-        PartialListing listing = store.list(
-            key,
-            Constants.COS_MAX_LISTING_LENGTH,
-            priorLastKey,
-            true);
-        for (FileMetadata file : listing.getFiles()) {
-          store.delete(file.getKey());
-        }
-        for (FileMetadata commonPrefix : listing.getCommonPrefixes()) {
-          store.delete(commonPrefix.getKey());
-        }
-        priorLastKey = listing.getPriorLastKey();
-      } while (priorLastKey != null);
-      try {
-        store.delete(key);
-      } catch (Exception e) {
-        LOG.error("Deleting the COS key: [{}] occurs an exception.", key, e);
-      }
-
-    } else {
-      LOG.debug("Delete the file: {}", f);
-      createParent(f);
-      store.delete(key);
-    }
-    return true;
-  }
-
-  @Override
-  public FileStatus getFileStatus(Path f) throws IOException {
-    Path absolutePath = makeAbsolute(f);
-    String key = pathToKey(absolutePath);
-
-    if (key.length() == 0) {
-      // root always exists
-      return newDirectory(absolutePath);
-    }
-
-    LOG.debug("Call the getFileStatus to obtain the metadata for "
-        + "the file: [{}].", f);
-
-    FileMetadata meta = store.retrieveMetadata(key);
-    if (meta != null) {
-      if (meta.isFile()) {
-        LOG.debug("Path: [{}] is a file. COS key: [{}]", f, key);
-        return newFile(meta, absolutePath);
-      } else {
-        LOG.debug("Path: [{}] is a dir. COS key: [{}]", f, key);
-        return newDirectory(meta, absolutePath);
-      }
-    }
-
-    if (!key.endsWith(PATH_DELIMITER)) {
-      key += PATH_DELIMITER;
-    }
-
-    // Considering that the object store's directory is a common prefix in
-    // the object key, it needs to check the existence of the path by listing
-    // the COS key.
-    LOG.debug("List COS key: [{}] to check the existence of the path.", key);
-    PartialListing listing = store.list(key, 1);
-    if (listing.getFiles().length > 0
-        || listing.getCommonPrefixes().length > 0) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Path: [{}] is a directory. COS key: [{}]", f, key);
-      }
-      return newDirectory(absolutePath);
-    }
-
-    throw new FileNotFoundException(
-        "No such file or directory '" + absolutePath + "'");
-  }
-
-  @Override
-  public URI getUri() {
-    return uri;
-  }
-
-  /**
-   * <p>
-   * If <code>f</code> is a file, this method will make a single call to COS.
-   * If <code>f</code> is a directory,
-   * this method will make a maximum of ( <i>n</i> / 199) + 2 calls to cos,
-   * where <i>n</i> is the total number of files
-   * and directories contained directly in <code>f</code>.
-   * </p>
-   */
-  @Override
-  public FileStatus[] listStatus(Path f) throws IOException {
-    Path absolutePath = makeAbsolute(f);
-    String key = pathToKey(absolutePath);
-
-    if (key.length() > 0) {
-      FileStatus fileStatus = this.getFileStatus(f);
-      if (fileStatus.isFile()) {
-        return new FileStatus[]{fileStatus};
-      }
-    }
-
-    if (!key.endsWith(PATH_DELIMITER)) {
-      key += PATH_DELIMITER;
-    }
-
-    URI pathUri = absolutePath.toUri();
-    Set<FileStatus> status = new TreeSet<>();
-    String priorLastKey = null;
-    do {
-      PartialListing listing = store.list(
-          key, Constants.COS_MAX_LISTING_LENGTH, priorLastKey, false);
-      for (FileMetadata fileMetadata : listing.getFiles()) {
-        Path subPath = keyToPath(fileMetadata.getKey());
-        if (fileMetadata.getKey().equals(key)) {
-          // this is just the directory we have been asked to list.
-          LOG.debug("The file list contains the COS key [{}] to be listed.",
-              key);
-        } else {
-          status.add(newFile(fileMetadata, subPath));
-        }
-      }
-
-      for (FileMetadata commonPrefix : listing.getCommonPrefixes()) {
-        Path subPath = keyToPath(commonPrefix.getKey());
-        String relativePath = pathUri.relativize(subPath.toUri()).getPath();
-        status.add(
-            newDirectory(commonPrefix, new Path(absolutePath, relativePath)));
-      }
-      priorLastKey = listing.getPriorLastKey();
-    } while (priorLastKey != null);
-
-    return status.toArray(new FileStatus[status.size()]);
-  }
-
-  private FileStatus newFile(FileMetadata meta, Path path) {
-    return new FileStatus(meta.getLength(), false, 1, getDefaultBlockSize(),
-        meta.getLastModified(), 0, null, this.owner, this.group,
-        path.makeQualified(this.getUri(), this.getWorkingDirectory()));
-  }
-
-  private FileStatus newDirectory(Path path) {
-    return new FileStatus(0, true, 1, 0, 0, 0, null, this.owner, this.group,
-        path.makeQualified(this.getUri(), this.getWorkingDirectory()));
-  }
-
-  private FileStatus newDirectory(FileMetadata meta, Path path) {
-    if (meta == null) {
-      return newDirectory(path);
-    }
-    return new FileStatus(0, true, 1, 0, meta.getLastModified(),
-        0, null, this.owner, this.group,
-        path.makeQualified(this.getUri(), this.getWorkingDirectory()));
-  }
-
-  /**
-   * Validate the path from the bottom up.
-   *
-   * @param path The path to be validated
-   * @throws FileAlreadyExistsException The specified path is an existing file
-   * @throws IOException                Getting the file status of the
-   *                                    specified path occurs
-   *                                    an IOException.
-   */
-  private void validatePath(Path path) throws IOException {
-    Path parent = path.getParent();
-    do {
-      try {
-        FileStatus fileStatus = getFileStatus(parent);
-        if (fileStatus.isDirectory()) {
-          break;
-        } else {
-          throw new FileAlreadyExistsException(String.format(
-              "Can't make directory for path '%s', it is a file.", parent));
-        }
-      } catch (FileNotFoundException e) {
-        LOG.debug("The Path: [{}] does not exist.", path);
-      }
-      parent = parent.getParent();
-    } while (parent != null);
-  }
-
-  @Override
-  public boolean mkdirs(Path f, FsPermission permission) throws IOException {
-    try {
-      FileStatus fileStatus = getFileStatus(f);
-      if (fileStatus.isDirectory()) {
-        return true;
-      } else {
-        throw new FileAlreadyExistsException("Path is a file: " + f);
-      }
-    } catch (FileNotFoundException e) {
-      validatePath(f);
-    }
-
-    return mkDirRecursively(f, permission);
-  }
-
-  /**
-   * Recursively create a directory.
-   *
-   * @param f          Absolute path to the directory.
-   * @param permission Directory permissions. Permission does not work for
-   *                   the CosN filesystem currently.
-   * @return Return true if the creation was successful,  throw a IOException.
-   * @throws IOException The specified path already exists or an error
-   *                     creating the path.
-   */
-  public boolean mkDirRecursively(Path f, FsPermission permission)
-      throws IOException {
-    Path absolutePath = makeAbsolute(f);
-    List<Path> paths = new ArrayList<>();
-    do {
-      paths.add(absolutePath);
-      absolutePath = absolutePath.getParent();
-    } while (absolutePath != null);
-
-    for (Path path : paths) {
-      if (path.equals(new Path(CosNFileSystem.PATH_DELIMITER))) {
-        break;
-      }
-      try {
-        FileStatus fileStatus = getFileStatus(path);
-        if (fileStatus.isFile()) {
-          throw new FileAlreadyExistsException(
-              String.format("Can't make directory for path: %s, "
-                  + "since it is a file.", f));
-        }
-        if (fileStatus.isDirectory()) {
-          break;
-        }
-      } catch (FileNotFoundException e) {
-        LOG.debug("Making dir: [{}] in COS", f);
-
-        String folderPath = pathToKey(makeAbsolute(f));
-        if (!folderPath.endsWith(PATH_DELIMITER)) {
-          folderPath += PATH_DELIMITER;
-        }
-        store.storeEmptyFile(folderPath);
-      }
-    }
-    return true;
-  }
-
-  private boolean mkdir(Path f) throws IOException {
-    try {
-      FileStatus fileStatus = getFileStatus(f);
-      if (fileStatus.isFile()) {
-        throw new FileAlreadyExistsException(
-            String.format(
-                "Can't make directory for path '%s' since it is a file.", f));
-      }
-    } catch (FileNotFoundException e) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Make directory: [{}] in COS.", f);
-      }
-
-      String folderPath = pathToKey(makeAbsolute(f));
-      if (!folderPath.endsWith(PATH_DELIMITER)) {
-        folderPath += PATH_DELIMITER;
-      }
-      store.storeEmptyFile(folderPath);
-    }
-    return true;
-  }
-
-  @Override
-  public FSDataInputStream open(Path f, int bufferSize) throws IOException {
-    FileStatus fs = getFileStatus(f); // will throw if the file doesn't
-    // exist
-    if (fs.isDirectory()) {
-      throw new FileNotFoundException("'" + f + "' is a directory");
-    }
-    LOG.info("Open the file: [{}] for reading.", f);
-    Path absolutePath = makeAbsolute(f);
-    String key = pathToKey(absolutePath);
-    long fileSize = store.getFileLength(key);
-    return new FSDataInputStream(new BufferedFSInputStream(
-        new CosNInputStream(this.getConf(), store, statistics, key, fileSize,
-            this.boundedIOThreadPool), bufferSize));
-  }
-
-  @Override
-  public boolean rename(Path src, Path dst) throws IOException {
-    LOG.debug("Rename source path: [{}] to dest path: [{}].", src, dst);
-
-    // Renaming the root directory is not allowed
-    if (src.isRoot()) {
-      LOG.debug("Cannot rename the root directory of a filesystem.");
-      return false;
-    }
-
-    // check the source path whether exists or not
-    FileStatus srcFileStatus = this.getFileStatus(src);
-
-    // Source path and destination path are not allowed to be the same
-    if (src.equals(dst)) {
-      LOG.debug("Source path and dest path refer to "
-          + "the same file or directory: [{}].", dst);
-      throw new IOException("Source path and dest path refer "
-          + "the same file or directory");
-    }
-
-    // It is not allowed to rename a parent directory to its subdirectory
-    Path dstParentPath;
-    for (dstParentPath = dst.getParent();
-         null != dstParentPath && !src.equals(dstParentPath);
-         dstParentPath = dstParentPath.getParent()) {
-      // Recursively find the common parent path of the source and
-      // destination paths.
-      LOG.debug("Recursively find the common parent directory of the source "
-              + "and destination paths. The currently found parent path: {}",
-          dstParentPath);
-    }
-
-    if (null != dstParentPath) {
-      LOG.debug("It is not allowed to rename a parent directory:[{}] "
-          + "to its subdirectory:[{}].", src, dst);
-      throw new IOException(String.format(
-          "It is not allowed to rename a parent directory: %s "
-              + "to its subdirectory: %s", src, dst));
-    }
-
-    FileStatus dstFileStatus;
-    try {
-      dstFileStatus = this.getFileStatus(dst);
-
-      // The destination path exists and is a file,
-      // and the rename operation is not allowed.
-      if (dstFileStatus.isFile()) {
-        throw new FileAlreadyExistsException(String.format(
-            "File: %s already exists", dstFileStatus.getPath()));
-      } else {
-        // The destination path is an existing directory,
-        // and it is checked whether there is a file or directory
-        // with the same name as the source path under the destination path
-        dst = new Path(dst, src.getName());
-        FileStatus[] statuses;
-        try {
-          statuses = this.listStatus(dst);
-        } catch (FileNotFoundException e) {
-          statuses = null;
-        }
-        if (null != statuses && statuses.length > 0) {
-          LOG.debug("Cannot rename source file: [{}] to dest file: [{}], "
-              + "because the file already exists.", src, dst);
-          throw new FileAlreadyExistsException(
-              String.format(
-                  "File: %s already exists", dst
-              )
-          );
-        }
-      }
-    } catch (FileNotFoundException e) {
-      // destination path not exists
-      Path tempDstParentPath = dst.getParent();
-      FileStatus dstParentStatus = this.getFileStatus(tempDstParentPath);
-      if (!dstParentStatus.isDirectory()) {
-        throw new IOException(String.format(
-            "Cannot rename %s to %s, %s is a file", src, dst, dst.getParent()
-        ));
-      }
-      // The default root directory is definitely there.
-    }
-
-    boolean result;
-    if (srcFileStatus.isDirectory()) {
-      result = this.copyDirectory(src, dst);
-    } else {
-      result = this.copyFile(src, dst);
-    }
-
-    if (!result) {
-      //Since rename is a non-atomic operation, after copy fails,
-      // it is not allowed to delete the data of the original path.
-      return false;
-    } else {
-      return this.delete(src, true);
-    }
-  }
-
-  private boolean copyFile(Path srcPath, Path dstPath) throws IOException {
-    String srcKey = pathToKey(srcPath);
-    String dstKey = pathToKey(dstPath);
-    this.store.copy(srcKey, dstKey);
-    return true;
-  }
-
-  private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
-    String srcKey = pathToKey(srcPath);
-    if (!srcKey.endsWith(PATH_DELIMITER)) {
-      srcKey += PATH_DELIMITER;
-    }
-    String dstKey = pathToKey(dstPath);
-    if (!dstKey.endsWith(PATH_DELIMITER)) {
-      dstKey += PATH_DELIMITER;
-    }
-
-    if (dstKey.startsWith(srcKey)) {
-      throw new IOException(
-          "can not copy a directory to a subdirectory of self");
-    }
-
-    this.store.storeEmptyFile(dstKey);
-    CosNCopyFileContext copyFileContext = new CosNCopyFileContext();
-
-    int copiesToFinishes = 0;
-    String priorLastKey = null;
-    do {
-      PartialListing objectList = this.store.list(
-          srcKey, Constants.COS_MAX_LISTING_LENGTH, priorLastKey, true);
-      for (FileMetadata file : objectList.getFiles()) {
-        this.boundedCopyThreadPool.execute(new CosNCopyFileTask(
-            this.store,
-            file.getKey(),
-            dstKey.concat(file.getKey().substring(srcKey.length())),
-            copyFileContext));
-        copiesToFinishes++;
-        if (!copyFileContext.isCopySuccess()) {
-          break;
-        }
-      }
-      priorLastKey = objectList.getPriorLastKey();
-    } while (null != priorLastKey);
-
-    copyFileContext.lock();
-    try {
-      copyFileContext.awaitAllFinish(copiesToFinishes);
-    } catch (InterruptedException e) {
-      LOG.warn("interrupted when wait copies to finish");
-    } finally {
-      copyFileContext.lock();
-    }
-
-    return copyFileContext.isCopySuccess();
-  }
-
-  private void createParent(Path path) throws IOException {
-    Path parent = path.getParent();
-    if (parent != null) {
-      String parentKey = pathToKey(parent);
-      LOG.debug("Create parent key: {}", parentKey);
-      if (!parentKey.equals(PATH_DELIMITER)) {
-        String key = pathToKey(makeAbsolute(parent));
-        if (key.length() > 0) {
-          try {
-            store.storeEmptyFile(key + PATH_DELIMITER);
-          } catch (IOException e) {
-            LOG.debug("Store a empty file in COS failed.", e);
-            throw e;
-          }
-        }
-      }
-    }
-  }
-
-  @Override
-  @SuppressWarnings("deprecation")
-  public long getDefaultBlockSize() {
-    return getConf().getLong(
-        CosNConfigKeys.COSN_BLOCK_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_BLOCK_SIZE);
-  }
-
-  /**
-   * Set the working directory to the given directory.
-   */
-  @Override
-  public void setWorkingDirectory(Path newDir) {
-    workingDir = newDir;
-  }
-
-  @Override
-  public Path getWorkingDirectory() {
-    return workingDir;
-  }
-
-  @Override
-  public String getCanonicalServiceName() {
-    // Does not support Token
-    return null;
-  }
-
-  @Override
-  public void close() throws IOException {
-    try {
-      this.store.close();
-      this.boundedIOThreadPool.shutdown();
-      this.boundedCopyThreadPool.shutdown();
-    } finally {
-      super.close();
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNInputStream.java

@@ -1,365 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Queue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.FSInputStream;
-import org.apache.hadoop.fs.FileSystem;
-
-/**
- * The input stream for the COS blob store.
- * Optimized sequential read flow based on a forward read-ahead queue
- */
-public class CosNInputStream extends FSInputStream {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CosNInputStream.class);
-
-  /**
-   * This class is used by {@link CosNInputStream}
-   * and {@link CosNFileReadTask} to buffer data that read from COS blob store.
-   */
-  public static class ReadBuffer {
-    public static final int INIT = 1;
-    public static final int SUCCESS = 0;
-    public static final int ERROR = -1;
-
-    private final ReentrantLock lock = new ReentrantLock();
-    private Condition readyCondition = lock.newCondition();
-
-    private byte[] buffer;
-    private int status;
-    private long start;
-    private long end;
-
-    public ReadBuffer(long start, long end) {
-      this.start = start;
-      this.end = end;
-      this.buffer = new byte[(int) (this.end - this.start) + 1];
-      this.status = INIT;
-    }
-
-    public void lock() {
-      this.lock.lock();
-    }
-
-    public void unLock() {
-      this.lock.unlock();
-    }
-
-    public void await(int waitStatus) throws InterruptedException {
-      while (this.status == waitStatus) {
-        readyCondition.await();
-      }
-    }
-
-    public void signalAll() {
-      readyCondition.signalAll();
-    }
-
-    public byte[] getBuffer() {
-      return this.buffer;
-    }
-
-    public int getStatus() {
-      return this.status;
-    }
-
-    public void setStatus(int status) {
-      this.status = status;
-    }
-
-    public long getStart() {
-      return start;
-    }
-
-    public long getEnd() {
-      return end;
-    }
-  }
-
-  private FileSystem.Statistics statistics;
-  private final Configuration conf;
-  private final NativeFileSystemStore store;
-  private final String key;
-  private long position = 0;
-  private long nextPos = 0;
-  private long fileSize;
-  private long partRemaining;
-  private final long preReadPartSize;
-  private final int maxReadPartNumber;
-  private byte[] buffer;
-  private boolean closed;
-
-  private final ExecutorService readAheadExecutorService;
-  private final Queue<ReadBuffer> readBufferQueue;
-
-  public CosNInputStream(Configuration conf, NativeFileSystemStore store,
-      FileSystem.Statistics statistics, String key, long fileSize,
-      ExecutorService readAheadExecutorService) {
-    super();
-    this.conf = conf;
-    this.store = store;
-    this.statistics = statistics;
-    this.key = key;
-    this.fileSize = fileSize;
-    this.preReadPartSize = conf.getLong(
-        CosNConfigKeys.READ_AHEAD_BLOCK_SIZE_KEY,
-        CosNConfigKeys.DEFAULT_READ_AHEAD_BLOCK_SIZE);
-    this.maxReadPartNumber = conf.getInt(
-        CosNConfigKeys.READ_AHEAD_QUEUE_SIZE,
-        CosNConfigKeys.DEFAULT_READ_AHEAD_QUEUE_SIZE);
-
-    this.readAheadExecutorService = readAheadExecutorService;
-    this.readBufferQueue = new ArrayDeque<>(this.maxReadPartNumber);
-    this.closed = false;
-  }
-
-  private synchronized void reopen(long pos) throws IOException {
-    long partSize;
-
-    if (pos < 0) {
-      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
-    } else if (pos > this.fileSize) {
-      throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
-    } else {
-      if (pos + this.preReadPartSize > this.fileSize) {
-        partSize = this.fileSize - pos;
-      } else {
-        partSize = this.preReadPartSize;
-      }
-    }
-
-    this.buffer = null;
-
-    boolean isRandomIO = true;
-    if (pos == this.nextPos) {
-      isRandomIO = false;
-    } else {
-      while (this.readBufferQueue.size() != 0) {
-        if (this.readBufferQueue.element().getStart() != pos) {
-          this.readBufferQueue.poll();
-        } else {
-          break;
-        }
-      }
-    }
-
-    this.nextPos = pos + partSize;
-
-    int currentBufferQueueSize = this.readBufferQueue.size();
-    long lastByteStart;
-    if (currentBufferQueueSize == 0) {
-      lastByteStart = pos - partSize;
-    } else {
-      ReadBuffer[] readBuffers =
-          this.readBufferQueue.toArray(
-              new ReadBuffer[currentBufferQueueSize]);
-      lastByteStart = readBuffers[currentBufferQueueSize - 1].getStart();
-    }
-
-    int maxLen = this.maxReadPartNumber - currentBufferQueueSize;
-    for (int i = 0; i < maxLen && i < (currentBufferQueueSize + 1) * 2; i++) {
-      if (lastByteStart + partSize * (i + 1) > this.fileSize) {
-        break;
-      }
-
-      long byteStart = lastByteStart + partSize * (i + 1);
-      long byteEnd = byteStart + partSize - 1;
-      if (byteEnd >= this.fileSize) {
-        byteEnd = this.fileSize - 1;
-      }
-
-      ReadBuffer readBuffer = new ReadBuffer(byteStart, byteEnd);
-      if (readBuffer.getBuffer().length == 0) {
-        readBuffer.setStatus(ReadBuffer.SUCCESS);
-      } else {
-        this.readAheadExecutorService.execute(
-            new CosNFileReadTask(
-                this.conf, this.key, this.store, readBuffer));
-      }
-
-      this.readBufferQueue.add(readBuffer);
-      if (isRandomIO) {
-        break;
-      }
-    }
-
-    ReadBuffer readBuffer = this.readBufferQueue.poll();
-    if (null != readBuffer) {
-      readBuffer.lock();
-      try {
-        readBuffer.await(ReadBuffer.INIT);
-        if (readBuffer.getStatus() == ReadBuffer.ERROR) {
-          this.buffer = null;
-        } else {
-          this.buffer = readBuffer.getBuffer();
-        }
-      } catch (InterruptedException e) {
-        LOG.warn("An interrupted exception occurred "
-            + "when waiting a read buffer.");
-      } finally {
-        readBuffer.unLock();
-      }
-    }
-
-    if (null == this.buffer) {
-      throw new IOException("Null IO stream");
-    }
-
-    this.position = pos;
-    this.partRemaining = partSize;
-  }
-
-  @Override
-  public void seek(long pos) throws IOException {
-    if (pos < 0) {
-      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
-    }
-    if (pos > this.fileSize) {
-      throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
-    }
-
-    if (this.position == pos) {
-      return;
-    }
-    if (pos > position && pos < this.position + partRemaining) {
-      long len = pos - this.position;
-      this.position = pos;
-      this.partRemaining -= len;
-    } else {
-      this.reopen(pos);
-    }
-  }
-
-  @Override
-  public long getPos() {
-    return this.position;
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) {
-    // Currently does not support to seek the offset of a new source
-    return false;
-  }
-
-  @Override
-  public int read() throws IOException {
-    if (this.closed) {
-      throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-    }
-
-    if (this.partRemaining <= 0 && this.position < this.fileSize) {
-      this.reopen(this.position);
-    }
-
-    int byteRead = -1;
-    if (this.partRemaining != 0) {
-      byteRead = this.buffer[
-          (int) (this.buffer.length - this.partRemaining)] & 0xff;
-    }
-    if (byteRead >= 0) {
-      this.position++;
-      this.partRemaining--;
-      if (null != this.statistics) {
-        this.statistics.incrementBytesRead(byteRead);
-      }
-    }
-
-    return byteRead;
-  }
-
-  @Override
-  public int read(byte[] b, int off, int len) throws IOException {
-    if (this.closed) {
-      throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-    }
-
-    if (len == 0) {
-      return 0;
-    }
-
-    if (off < 0 || len < 0 || len > b.length) {
-      throw new IndexOutOfBoundsException();
-    }
-
-    int bytesRead = 0;
-    while (position < fileSize && bytesRead < len) {
-      if (partRemaining <= 0) {
-        reopen(position);
-      }
-
-      int bytes = 0;
-      for (int i = this.buffer.length - (int) partRemaining;
-           i < this.buffer.length; i++) {
-        b[off + bytesRead] = this.buffer[i];
-        bytes++;
-        bytesRead++;
-        if (off + bytesRead >= len) {
-          break;
-        }
-      }
-
-      if (bytes > 0) {
-        this.position += bytes;
-        this.partRemaining -= bytes;
-      } else if (this.partRemaining != 0) {
-        throw new IOException(
-            "Failed to read from stream. Remaining: " + this.partRemaining);
-      }
-    }
-    if (null != this.statistics && bytesRead > 0) {
-      this.statistics.incrementBytesRead(bytesRead);
-    }
-
-    return bytesRead == 0 ? -1 : bytesRead;
-  }
-
-  @Override
-  public int available() throws IOException {
-    if (this.closed) {
-      throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
-    }
-
-    long remaining = this.fileSize - this.position;
-    if (remaining > Integer.MAX_VALUE) {
-      return Integer.MAX_VALUE;
-    }
-    return (int)remaining;
-  }
-
-  @Override
-  public void close() {
-    if (this.closed) {
-      return;
-    }
-    this.closed = true;
-    this.buffer = null;
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNOutputStream.java

@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService;
-import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors;
-import com.qcloud.cos.model.PartETag;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * The output stream for the COS blob store.
- * Implement streaming upload to COS based on the multipart upload function.
- * ( the maximum size of each part is 5GB)
- * Support up to 40TB single file by multipart upload (each part is 5GB).
- * Improve the upload performance of writing large files by using byte buffers
- * and a fixed thread pool.
- */
-public class CosNOutputStream extends OutputStream {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CosNOutputStream.class);
-
-  private final Configuration conf;
-  private final NativeFileSystemStore store;
-  private MessageDigest digest;
-  private long blockSize;
-  private String key;
-  private int currentBlockId = 0;
-  private Set<ByteBufferWrapper> blockCacheBuffers = new HashSet<>();
-  private ByteBufferWrapper currentBlockBuffer;
-  private OutputStream currentBlockOutputStream;
-  private String uploadId = null;
-  private ListeningExecutorService executorService;
-  private List<ListenableFuture<PartETag>> etagList = new LinkedList<>();
-  private int blockWritten = 0;
-  private boolean closed = false;
-
-  public CosNOutputStream(Configuration conf, NativeFileSystemStore store,
-      String key, long blockSize, ExecutorService executorService)
-      throws IOException {
-    this.conf = conf;
-    this.store = store;
-    this.key = key;
-    this.blockSize = blockSize;
-    if (this.blockSize < Constants.MIN_PART_SIZE) {
-      LOG.warn(
-          String.format(
-              "The minimum size of a single block is limited to %d.",
-              Constants.MIN_PART_SIZE));
-      this.blockSize = Constants.MIN_PART_SIZE;
-    }
-    if (this.blockSize > Constants.MAX_PART_SIZE) {
-      LOG.warn(
-          String.format(
-              "The maximum size of a single block is limited to %d.",
-              Constants.MAX_PART_SIZE));
-      this.blockSize = Constants.MAX_PART_SIZE;
-    }
-
-    // Use a blocking thread pool with fair scheduling
-    this.executorService = MoreExecutors.listeningDecorator(executorService);
-
-    try {
-      this.currentBlockBuffer =
-          BufferPool.getInstance().getBuffer((int) this.blockSize);
-    } catch (IOException e) {
-      throw new IOException("Getting a buffer size: "
-          + String.valueOf(this.blockSize)
-          + " from buffer pool occurs an exception: ", e);
-    }
-
-    try {
-      this.digest = MessageDigest.getInstance("MD5");
-      this.currentBlockOutputStream = new DigestOutputStream(
-          new ByteBufferOutputStream(this.currentBlockBuffer.getByteBuffer()),
-          this.digest);
-    } catch (NoSuchAlgorithmException e) {
-      this.digest = null;
-      this.currentBlockOutputStream =
-          new ByteBufferOutputStream(this.currentBlockBuffer.getByteBuffer());
-    }
-  }
-
-  @Override
-  public void flush() throws IOException {
-    this.currentBlockOutputStream.flush();
-  }
-
-  @Override
-  public synchronized void close() throws IOException {
-    if (this.closed) {
-      return;
-    }
-    this.currentBlockOutputStream.flush();
-    this.currentBlockOutputStream.close();
-    LOG.info("The output stream has been close, and "
-        + "begin to upload the last block: [{}].", this.currentBlockId);
-    this.blockCacheBuffers.add(this.currentBlockBuffer);
-    if (this.blockCacheBuffers.size() == 1) {
-      byte[] md5Hash = this.digest == null ? null : this.digest.digest();
-      store.storeFile(this.key,
-          new ByteBufferInputStream(this.currentBlockBuffer.getByteBuffer()),
-          md5Hash, this.currentBlockBuffer.getByteBuffer().remaining());
-    } else {
-      PartETag partETag = null;
-      if (this.blockWritten > 0) {
-        LOG.info("Upload the last part..., blockId: [{}], written bytes: [{}]",
-            this.currentBlockId, this.blockWritten);
-        partETag = store.uploadPart(
-            new ByteBufferInputStream(currentBlockBuffer.getByteBuffer()),
-            key, uploadId, currentBlockId + 1,
-            currentBlockBuffer.getByteBuffer().remaining());
-      }
-      final List<PartETag> futurePartETagList = this.waitForFinishPartUploads();
-      if (null == futurePartETagList) {
-        throw new IOException("Failed to multipart upload to cos, abort it.");
-      }
-      List<PartETag> tmpPartEtagList = new LinkedList<>(futurePartETagList);
-      if (null != partETag) {
-        tmpPartEtagList.add(partETag);
-      }
-      store.completeMultipartUpload(this.key, this.uploadId, tmpPartEtagList);
-    }
-    try {
-      BufferPool.getInstance().returnBuffer(this.currentBlockBuffer);
-    } catch (InterruptedException e) {
-      LOG.error("An exception occurred "
-          + "while returning the buffer to the buffer pool.", e);
-    }
-    LOG.info("The outputStream for key: [{}] has been uploaded.", key);
-    this.blockWritten = 0;
-    this.closed = true;
-  }
-
-  private List<PartETag> waitForFinishPartUploads() throws IOException {
-    try {
-      LOG.info("Wait for all parts to finish their uploading.");
-      return Futures.allAsList(this.etagList).get();
-    } catch (InterruptedException e) {
-      LOG.error("Interrupt the part upload.", e);
-      return null;
-    } catch (ExecutionException e) {
-      LOG.error("Cancelling futures.");
-      for (ListenableFuture<PartETag> future : this.etagList) {
-        future.cancel(true);
-      }
-      (store).abortMultipartUpload(this.key, this.uploadId);
-      LOG.error("Multipart upload with id: [{}] to COS key: [{}]",
-          this.uploadId, this.key, e);
-      throw new IOException("Multipart upload with id: "
-          + this.uploadId + " to " + this.key, e);
-    }
-  }
-
-  private void uploadPart() throws IOException {
-    this.currentBlockOutputStream.flush();
-    this.currentBlockOutputStream.close();
-    this.blockCacheBuffers.add(this.currentBlockBuffer);
-
-    if (this.currentBlockId == 0) {
-      uploadId = (store).getUploadId(key);
-    }
-
-    ListenableFuture<PartETag> partETagListenableFuture =
-        this.executorService.submit(
-            new Callable<PartETag>() {
-              private final ByteBufferWrapper buf = currentBlockBuffer;
-              private final String localKey = key;
-              private final String localUploadId = uploadId;
-              private final int blockId = currentBlockId;
-
-              @Override
-              public PartETag call() throws Exception {
-                if (LOG.isDebugEnabled()) {
-                  LOG.debug("{} is uploading a part.",
-                      Thread.currentThread().getName());
-                }
-                PartETag partETag = (store).uploadPart(
-                    new ByteBufferInputStream(this.buf.getByteBuffer()),
-                    this.localKey, this.localUploadId,
-                    this.blockId + 1, this.buf.getByteBuffer().remaining());
-                BufferPool.getInstance().returnBuffer(this.buf);
-                return partETag;
-              }
-            });
-    this.etagList.add(partETagListenableFuture);
-    try {
-      this.currentBlockBuffer =
-          BufferPool.getInstance().getBuffer((int) this.blockSize);
-    } catch (IOException e) {
-      String errMsg = String.format("Getting a buffer [size:%d] from "
-          + "the buffer pool failed.", this.blockSize);
-      throw new IOException(errMsg, e);
-    }
-    this.currentBlockId++;
-    if (null != this.digest) {
-      this.digest.reset();
-      this.currentBlockOutputStream = new DigestOutputStream(
-          new ByteBufferOutputStream(this.currentBlockBuffer.getByteBuffer()),
-          this.digest);
-    } else {
-      this.currentBlockOutputStream =
-          new ByteBufferOutputStream(this.currentBlockBuffer.getByteBuffer());
-    }
-  }
-
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    if (this.closed) {
-      throw new IOException("block stream has been closed.");
-    }
-
-    while (len > 0) {
-      long writeBytes;
-      if (this.blockWritten + len > this.blockSize) {
-        writeBytes = this.blockSize - this.blockWritten;
-      } else {
-        writeBytes = len;
-      }
-
-      this.currentBlockOutputStream.write(b, off, (int) writeBytes);
-      this.blockWritten += writeBytes;
-      if (this.blockWritten >= this.blockSize) {
-        this.uploadPart();
-        this.blockWritten = 0;
-      }
-      len -= writeBytes;
-      off += writeBytes;
-    }
-  }
-
-  @Override
-  public void write(byte[] b) throws IOException {
-    this.write(b, 0, b.length);
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    if (this.closed) {
-      throw new IOException("block stream has been closed.");
-    }
-
-    byte[] singleBytes = new byte[1];
-    singleBytes[0] = (byte) b;
-    this.currentBlockOutputStream.write(singleBytes, 0, 1);
-    this.blockWritten += 1;
-    if (this.blockWritten >= this.blockSize) {
-      this.uploadPart();
-      this.blockWritten = 0;
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java

@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.net.URI;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.qcloud.cos.auth.COSCredentialsProvider;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
-import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider;
-import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider;
-
-/**
- * Utility methods for CosN code.
- */
-public final class CosNUtils {
-  private static final Logger LOG = LoggerFactory.getLogger(CosNUtils.class);
-
-  static final String INSTANTIATION_EXCEPTION
-      = "instantiation exception";
-  static final String NOT_COS_CREDENTIAL_PROVIDER
-      = "is not cos credential provider";
-  static final String ABSTRACT_CREDENTIAL_PROVIDER
-      = "is abstract and therefore cannot be created";
-
-  private CosNUtils() {
-  }
-
-  public static COSCredentialsProviderList createCosCredentialsProviderSet(
-      URI uri,
-      Configuration conf) throws IOException {
-    COSCredentialsProviderList credentialProviderList =
-        new COSCredentialsProviderList();
-
-    Class<?>[] cosClasses = CosNUtils.loadCosProviderClasses(
-        conf,
-        CosNConfigKeys.COSN_CREDENTIALS_PROVIDER);
-    if (0 == cosClasses.length) {
-      credentialProviderList.add(
-          new SimpleCredentialsProvider(uri, conf));
-      credentialProviderList.add(
-          new EnvironmentVariableCredentialsProvider(uri, conf));
-    } else {
-      for (Class<?> credClass : cosClasses) {
-        credentialProviderList.add(createCOSCredentialProvider(uri, conf,
-            credClass));
-      }
-    }
-
-    return credentialProviderList;
-  }
-
-  public static Class<?>[] loadCosProviderClasses(
-      Configuration conf,
-      String key,
-      Class<?>... defaultValue) throws IOException {
-    try {
-      return conf.getClasses(key, defaultValue);
-    } catch (RuntimeException e) {
-      Throwable c = e.getCause() != null ? e.getCause() : e;
-      throw new IOException("From option " + key + ' ' + c, c);
-    }
-  }
-
-  public static COSCredentialsProvider createCOSCredentialProvider(
-      URI uri,
-      Configuration conf,
-      Class<?> credClass) throws IOException {
-    COSCredentialsProvider credentialsProvider;
-    if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) {
-      throw new IllegalArgumentException("class " + credClass + " " +
-          NOT_COS_CREDENTIAL_PROVIDER);
-    }
-    if (Modifier.isAbstract(credClass.getModifiers())) {
-      throw new IllegalArgumentException("class " + credClass + " " +
-          ABSTRACT_CREDENTIAL_PROVIDER);
-    }
-    LOG.debug("Credential Provider class: " + credClass.getName());
-
-    try {
-      // new credClass()
-      Constructor constructor = getConstructor(credClass);
-      if (constructor != null) {
-        credentialsProvider =
-            (COSCredentialsProvider) constructor.newInstance();
-        return credentialsProvider;
-      }
-      // new credClass(conf)
-      constructor = getConstructor(credClass, Configuration.class);
-      if (null != constructor) {
-        credentialsProvider =
-            (COSCredentialsProvider) constructor.newInstance(conf);
-        return credentialsProvider;
-      }
-
-      // new credClass(uri, conf)
-      constructor = getConstructor(credClass, URI.class,
-          Configuration.class);
-      if (null != constructor) {
-        credentialsProvider =
-            (COSCredentialsProvider) constructor.newInstance(uri,
-                conf);
-        return credentialsProvider;
-      }
-
-      Method factory = getFactoryMethod(credClass,
-          COSCredentialsProvider.class, "getInstance");
-      if (null != factory) {
-        credentialsProvider = (COSCredentialsProvider) factory.invoke(null);
-        return credentialsProvider;
-      }
-
-      throw new IllegalArgumentException(
-          "Not supported constructor or factory method found"
-      );
-
-    } catch (IllegalAccessException e) {
-      throw new IOException(
-          credClass.getName() + " " + INSTANTIATION_EXCEPTION + ": " + e, e);
-    } catch (InstantiationException e) {
-      throw new IOException(
-          credClass.getName() + " " + INSTANTIATION_EXCEPTION + ": " + e, e);
-    } catch (InvocationTargetException e) {
-      Throwable targetException = e.getTargetException();
-      if (targetException == null) {
-        targetException = e;
-      }
-      throw new IOException(
-          credClass.getName() + " " + INSTANTIATION_EXCEPTION + ": "
-              + targetException, targetException);
-    }
-  }
-
-  private static Constructor<?> getConstructor(Class<?> cl, Class<?>... args) {
-    try {
-      Constructor constructor = cl.getDeclaredConstructor(args);
-      return Modifier.isPublic(constructor.getModifiers()) ? constructor : null;
-    } catch (NoSuchMethodException e) {
-      return null;
-    }
-  }
-
-  private static Method getFactoryMethod(
-      Class<?> cl, Class<?> returnType, String methodName) {
-    try {
-      Method m = cl.getDeclaredMethod(methodName);
-      if (Modifier.isPublic(m.getModifiers())
-          && Modifier.isStatic(m.getModifiers())
-          && returnType.isAssignableFrom(m.getReturnType())) {
-        return m;
-      } else {
-        return null;
-      }
-    } catch (NoSuchMethodException e) {
-      return null;
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java

@@ -1,772 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
-
-import com.qcloud.cos.COSClient;
-import com.qcloud.cos.ClientConfig;
-import com.qcloud.cos.auth.BasicCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
-import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
-import com.qcloud.cos.exception.CosClientException;
-import com.qcloud.cos.exception.CosServiceException;
-import com.qcloud.cos.http.HttpProtocol;
-import com.qcloud.cos.model.AbortMultipartUploadRequest;
-import com.qcloud.cos.model.COSObject;
-import com.qcloud.cos.model.COSObjectSummary;
-import com.qcloud.cos.model.CompleteMultipartUploadRequest;
-import com.qcloud.cos.model.CompleteMultipartUploadResult;
-import com.qcloud.cos.model.CopyObjectRequest;
-import com.qcloud.cos.model.DeleteObjectRequest;
-import com.qcloud.cos.model.GetObjectMetadataRequest;
-import com.qcloud.cos.model.GetObjectRequest;
-import com.qcloud.cos.model.InitiateMultipartUploadRequest;
-import com.qcloud.cos.model.InitiateMultipartUploadResult;
-import com.qcloud.cos.model.ListObjectsRequest;
-import com.qcloud.cos.model.ObjectListing;
-import com.qcloud.cos.model.ObjectMetadata;
-import com.qcloud.cos.model.PartETag;
-import com.qcloud.cos.model.PutObjectRequest;
-import com.qcloud.cos.model.PutObjectResult;
-import com.qcloud.cos.model.UploadPartRequest;
-import com.qcloud.cos.model.UploadPartResult;
-import com.qcloud.cos.region.Region;
-import com.qcloud.cos.utils.Base64;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.http.HttpStatus;
-
-/**
- * The class actually performs access operation to the COS blob store.
- * It provides the bridging logic for the Hadoop's abstract filesystem and COS.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class CosNativeFileSystemStore implements NativeFileSystemStore {
-  private COSClient cosClient;
-  private String bucketName;
-  private int maxRetryTimes;
-
-  public static final Logger LOG =
-      LoggerFactory.getLogger(CosNativeFileSystemStore.class);
-
-  /**
-   * Initialize the client to access COS blob storage.
-   *
-   * @param conf Hadoop configuration with COS configuration options.
-   * @throws IOException Initialize the COS client failed,
-   *                     caused by incorrect options.
-   */
-  private void initCOSClient(URI uri, Configuration conf) throws IOException {
-    COSCredentialsProviderList credentialProviderList =
-        CosNUtils.createCosCredentialsProviderSet(uri, conf);
-    String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
-    String endpointSuffix = conf.get(
-        CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
-    if (null == region && null == endpointSuffix) {
-      String exceptionMsg = String.format("config %s and %s at least one",
-          CosNConfigKeys.COSN_REGION_KEY,
-          CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
-      throw new IOException(exceptionMsg);
-    }
-
-    COSCredentials cosCred;
-    cosCred = new BasicCOSCredentials(
-        credentialProviderList.getCredentials().getCOSAccessKeyId(),
-        credentialProviderList.getCredentials().getCOSSecretKey());
-
-    boolean useHttps = conf.getBoolean(CosNConfigKeys.COSN_USE_HTTPS_KEY,
-        CosNConfigKeys.DEFAULT_USE_HTTPS);
-
-    ClientConfig config;
-    if (null == region) {
-      config = new ClientConfig(new Region(""));
-      config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix));
-    } else {
-      config = new ClientConfig(new Region(region));
-    }
-    if (useHttps) {
-      config.setHttpProtocol(HttpProtocol.https);
-    }
-
-    config.setUserAgent(conf.get(CosNConfigKeys.USER_AGENT,
-        CosNConfigKeys.DEFAULT_USER_AGENT) + " For " + " Hadoop "
-        + VersionInfo.getVersion());
-
-    this.maxRetryTimes = conf.getInt(CosNConfigKeys.COSN_MAX_RETRIES_KEY,
-        CosNConfigKeys.DEFAULT_MAX_RETRIES);
-
-    config.setMaxConnectionsCount(
-        conf.getInt(CosNConfigKeys.MAX_CONNECTION_NUM,
-            CosNConfigKeys.DEFAULT_MAX_CONNECTION_NUM));
-
-    this.cosClient = new COSClient(cosCred, config);
-  }
-
-  /**
-   * Initialize the CosNativeFileSystemStore object, including
-   * its COS client and default COS bucket.
-   *
-   * @param uri  The URI of the COS bucket accessed by default.
-   * @param conf Hadoop configuration with COS configuration options.
-   * @throws IOException Initialize the COS client failed.
-   */
-  @Override
-  public void initialize(URI uri, Configuration conf) throws IOException {
-    try {
-      initCOSClient(uri, conf);
-      this.bucketName = uri.getHost();
-    } catch (Exception e) {
-      handleException(e, "");
-    }
-  }
-
-  /**
-   * Store a file into COS from the specified input stream, which would be
-   * retried until the success or maximum number.
-   *
-   * @param key         COS object key.
-   * @param inputStream Input stream to be uploaded into COS.
-   * @param md5Hash     MD5 value of the content to be uploaded.
-   * @param length      Length of uploaded content.
-   * @throws IOException Upload the file failed.
-   */
-  private void storeFileWithRetry(String key, InputStream inputStream,
-      byte[] md5Hash, long length) throws IOException {
-    try {
-      ObjectMetadata objectMetadata = new ObjectMetadata();
-      objectMetadata.setContentMD5(Base64.encodeAsString(md5Hash));
-      objectMetadata.setContentLength(length);
-      PutObjectRequest putObjectRequest =
-          new PutObjectRequest(bucketName, key, inputStream, objectMetadata);
-
-      PutObjectResult putObjectResult =
-          (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
-      LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].",
-          key, putObjectResult.getETag());
-    } catch (Exception e) {
-      String errMsg = String.format("Store file failed. COS key: [%s], "
-          + "exception: [%s]", key, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-    }
-  }
-
-  /**
-   * Store a local file into COS.
-   *
-   * @param key     COS object key.
-   * @param file    The local file to be uploaded.
-   * @param md5Hash The MD5 value of the file to be uploaded.
-   * @throws IOException Upload the file failed.
-   */
-  @Override
-  public void storeFile(String key, File file, byte[] md5Hash)
-      throws IOException {
-    LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " +
-        "[{}]", file.getCanonicalPath(), file.length(), key);
-    storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)),
-        md5Hash, file.length());
-  }
-
-  /**
-   * Store a file into COS from the specified input stream.
-   *
-   * @param key           COS object key.
-   * @param inputStream   The Input stream to be uploaded.
-   * @param md5Hash       The MD5 value of the content to be uploaded.
-   * @param contentLength Length of uploaded content.
-   * @throws IOException Upload the file failed.
-   */
-  @Override
-  public void storeFile(
-      String key,
-      InputStream inputStream,
-      byte[] md5Hash,
-      long contentLength) throws IOException {
-    LOG.info("Store file from input stream. COS key: [{}], "
-        + "length: [{}].", key, contentLength);
-    storeFileWithRetry(key, inputStream, md5Hash, contentLength);
-  }
-
-  // For cos, storeEmptyFile means creating a directory
-  @Override
-  public void storeEmptyFile(String key) throws IOException {
-    if (!key.endsWith(CosNFileSystem.PATH_DELIMITER)) {
-      key = key + CosNFileSystem.PATH_DELIMITER;
-    }
-
-    ObjectMetadata objectMetadata = new ObjectMetadata();
-    objectMetadata.setContentLength(0);
-    InputStream input = new ByteArrayInputStream(new byte[0]);
-    PutObjectRequest putObjectRequest =
-        new PutObjectRequest(bucketName, key, input, objectMetadata);
-    try {
-      PutObjectResult putObjectResult =
-          (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
-      LOG.debug("Store empty file successfully. COS key: [{}], ETag: [{}].",
-          key, putObjectResult.getETag());
-    } catch (Exception e) {
-      String errMsg = String.format("Store empty file failed. "
-          + "COS key: [%s], exception: [%s]", key, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-    }
-  }
-
-  public PartETag uploadPart(File file, String key, String uploadId,
-      int partNum) throws IOException {
-    InputStream inputStream = new FileInputStream(file);
-    try {
-      return uploadPart(inputStream, key, uploadId, partNum, file.length());
-    } finally {
-      inputStream.close();
-    }
-  }
-
-  @Override
-  public PartETag uploadPart(InputStream inputStream, String key,
-      String uploadId, int partNum, long partSize) throws IOException {
-    UploadPartRequest uploadPartRequest = new UploadPartRequest();
-    uploadPartRequest.setBucketName(this.bucketName);
-    uploadPartRequest.setUploadId(uploadId);
-    uploadPartRequest.setInputStream(inputStream);
-    uploadPartRequest.setPartNumber(partNum);
-    uploadPartRequest.setPartSize(partSize);
-    uploadPartRequest.setKey(key);
-
-    try {
-      UploadPartResult uploadPartResult =
-          (UploadPartResult) callCOSClientWithRetry(uploadPartRequest);
-      return uploadPartResult.getPartETag();
-    } catch (Exception e) {
-      String errMsg = String.format("Current thread: [%d], COS key: [%s], "
-              + "upload id: [%s], part num: [%d], exception: [%s]",
-          Thread.currentThread().getId(), key, uploadId, partNum, e.toString());
-      handleException(new Exception(errMsg), key);
-    }
-
-    return null;
-  }
-
-  public void abortMultipartUpload(String key, String uploadId) {
-    LOG.info("Abort the multipart upload. COS key: [{}], upload id: [{}].",
-        key, uploadId);
-    AbortMultipartUploadRequest abortMultipartUploadRequest =
-        new AbortMultipartUploadRequest(bucketName, key, uploadId);
-    cosClient.abortMultipartUpload(abortMultipartUploadRequest);
-  }
-
-  /**
-   * Initialize a multipart upload and return the upload id.
-   *
-   * @param key The COS object key initialized to multipart upload.
-   * @return The multipart upload id.
-   */
-  public String getUploadId(String key) {
-    if (null == key || key.length() == 0) {
-      return "";
-    }
-
-    LOG.info("Initiate a multipart upload. bucket: [{}], COS key: [{}].",
-        bucketName, key);
-    InitiateMultipartUploadRequest initiateMultipartUploadRequest =
-        new InitiateMultipartUploadRequest(bucketName, key);
-    InitiateMultipartUploadResult initiateMultipartUploadResult =
-        cosClient.initiateMultipartUpload(initiateMultipartUploadRequest);
-    return initiateMultipartUploadResult.getUploadId();
-  }
-
-  /**
-   * Finish a multipart upload process, which will merge all parts uploaded.
-   *
-   * @param key          The COS object key to be finished.
-   * @param uploadId     The upload id of the multipart upload to be finished.
-   * @param partETagList The etag list of the part that has been uploaded.
-   * @return The result object of completing the multipart upload process.
-   */
-  public CompleteMultipartUploadResult completeMultipartUpload(
-      String key, String uploadId, List<PartETag> partETagList) {
-    Collections.sort(partETagList, new Comparator<PartETag>() {
-      @Override
-      public int compare(PartETag o1, PartETag o2) {
-        return o1.getPartNumber() - o2.getPartNumber();
-      }
-    });
-    LOG.info("Complete the multipart upload. bucket: [{}], COS key: [{}], "
-        + "upload id: [{}].", bucketName, key, uploadId);
-    CompleteMultipartUploadRequest completeMultipartUploadRequest =
-        new CompleteMultipartUploadRequest(
-            bucketName, key, uploadId, partETagList);
-    return cosClient.completeMultipartUpload(completeMultipartUploadRequest);
-  }
-
-  private FileMetadata queryObjectMetadata(String key) throws IOException {
-    GetObjectMetadataRequest getObjectMetadataRequest =
-        new GetObjectMetadataRequest(bucketName, key);
-    try {
-      ObjectMetadata objectMetadata =
-          (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
-      long mtime = 0;
-      if (objectMetadata.getLastModified() != null) {
-        mtime = objectMetadata.getLastModified().getTime();
-      }
-      long fileSize = objectMetadata.getContentLength();
-      FileMetadata fileMetadata = new FileMetadata(key, fileSize, mtime,
-          !key.endsWith(CosNFileSystem.PATH_DELIMITER));
-      LOG.debug("Retrieve file metadata. COS key: [{}], ETag: [{}], "
-              + "length: [{}].", key, objectMetadata.getETag(),
-          objectMetadata.getContentLength());
-      return fileMetadata;
-    } catch (CosServiceException e) {
-      if (e.getStatusCode() != HttpStatus.SC_NOT_FOUND) {
-        String errorMsg = String.format("Retrieve file metadata file failed. "
-            + "COS key: [%s], CosServiceException: [%s].", key, e.toString());
-        LOG.error(errorMsg);
-        handleException(new Exception(errorMsg), key);
-      }
-    }
-    return null;
-  }
-
-  @Override
-  public FileMetadata retrieveMetadata(String key) throws IOException {
-    if (key.endsWith(CosNFileSystem.PATH_DELIMITER)) {
-      key = key.substring(0, key.length() - 1);
-    }
-
-    if (!key.isEmpty()) {
-      FileMetadata fileMetadata = queryObjectMetadata(key);
-      if (fileMetadata != null) {
-        return fileMetadata;
-      }
-    }
-
-    // If the key is a directory.
-    key = key + CosNFileSystem.PATH_DELIMITER;
-    return queryObjectMetadata(key);
-  }
-
-  /**
-   * Download a COS object and return the input stream associated with it.
-   *
-   * @param key The object key that is being retrieved from the COS bucket
-   * @return This method returns null if the key is not found
-   * @throws IOException if failed to download.
-   */
-  @Override
-  public InputStream retrieve(String key) throws IOException {
-    LOG.debug("Retrieve object key: [{}].", key);
-    GetObjectRequest getObjectRequest =
-        new GetObjectRequest(this.bucketName, key);
-    try {
-      COSObject cosObject =
-          (COSObject) callCOSClientWithRetry(getObjectRequest);
-      return cosObject.getObjectContent();
-    } catch (Exception e) {
-      String errMsg = String.format("Retrieving key: [%s] occurs "
-          + "an exception: [%s].", key, e.toString());
-      LOG.error("Retrieving COS key: [{}] occurs an exception: [{}].", key, e);
-      handleException(new Exception(errMsg), key);
-    }
-    // never will get here
-    return null;
-  }
-
-  /**
-   * Retrieved a part of a COS object, which is specified the start position.
-   *
-   * @param key            The object key that is being retrieved from
-   *                       the COS bucket.
-   * @param byteRangeStart The start position of the part to be retrieved in
-   *                       the object.
-   * @return The input stream associated with the retrieved object.
-   * @throws IOException if failed to retrieve.
-   */
-  @Override
-  public InputStream retrieve(String key, long byteRangeStart)
-      throws IOException {
-    try {
-      LOG.debug("Retrieve COS key:[{}]. range start:[{}].",
-          key, byteRangeStart);
-      long fileSize = getFileLength(key);
-      long byteRangeEnd = fileSize - 1;
-      GetObjectRequest getObjectRequest =
-          new GetObjectRequest(this.bucketName, key);
-      if (byteRangeEnd >= byteRangeStart) {
-        getObjectRequest.setRange(byteRangeStart, fileSize - 1);
-      }
-      COSObject cosObject =
-          (COSObject) callCOSClientWithRetry(getObjectRequest);
-      return cosObject.getObjectContent();
-    } catch (Exception e) {
-      String errMsg =
-          String.format("Retrieving COS key: [%s] occurs an exception. " +
-                  "byte range start: [%s], exception: [%s].",
-              key, byteRangeStart, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-    }
-
-    // never will get here
-    return null;
-  }
-
-  /**
-   * Download a part of a COS object, which is specified the start and
-   * end position.
-   *
-   * @param key            The object key that is being downloaded
-   * @param byteRangeStart The start position of the part to be retrieved in
-   *                       the object.
-   * @param byteRangeEnd   The end position of the part to be retrieved in
-   *                       the object.
-   * @return The input stream associated with the retrieved objects.
-   * @throws IOException If failed to retrieve.
-   */
-  @Override
-  public InputStream retrieveBlock(String key, long byteRangeStart,
-      long byteRangeEnd) throws IOException {
-    try {
-      GetObjectRequest request = new GetObjectRequest(this.bucketName, key);
-      request.setRange(byteRangeStart, byteRangeEnd);
-      COSObject cosObject = (COSObject) this.callCOSClientWithRetry(request);
-      return cosObject.getObjectContent();
-    } catch (CosServiceException e) {
-      String errMsg =
-          String.format("Retrieving key [%s] with byteRangeStart [%d] occurs " +
-                  "an CosServiceException: [%s].",
-              key, byteRangeStart, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-      return null;
-    } catch (CosClientException e) {
-      String errMsg =
-          String.format("Retrieving key [%s] with byteRangeStart [%d] "
-                  + "occurs an exception: [%s].",
-              key, byteRangeStart, e.toString());
-      LOG.error("Retrieving COS key: [{}] with byteRangeStart: [{}] " +
-          "occurs an exception: [{}].", key, byteRangeStart, e);
-      handleException(new Exception(errMsg), key);
-    }
-
-    return null;
-  }
-
-  @Override
-  public PartialListing list(String prefix, int maxListingLength)
-      throws IOException {
-    return list(prefix, maxListingLength, null, false);
-  }
-
-  @Override
-  public PartialListing list(String prefix, int maxListingLength,
-      String priorLastKey, boolean recurse) throws IOException {
-    return list(prefix, recurse ? null : CosNFileSystem.PATH_DELIMITER,
-        maxListingLength, priorLastKey);
-  }
-
-  /**
-   * List the metadata for all objects that
-   * the object key has the specified prefix.
-   *
-   * @param prefix           The prefix to be listed.
-   * @param delimiter        The delimiter is a sign, the same paths between
-   *                         are listed.
-   * @param maxListingLength The maximum number of listed entries.
-   * @param priorLastKey     The last key in any previous search.
-   * @return A metadata list on the match.
-   * @throws IOException If list objects failed.
-   */
-  private PartialListing list(String prefix, String delimiter,
-      int maxListingLength, String priorLastKey) throws IOException {
-    LOG.debug("List objects. prefix: [{}], delimiter: [{}], " +
-            "maxListLength: [{}], priorLastKey: [{}].",
-        prefix, delimiter, maxListingLength, priorLastKey);
-
-    if (!prefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
-      prefix += CosNFileSystem.PATH_DELIMITER;
-    }
-    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
-    listObjectsRequest.setBucketName(bucketName);
-    listObjectsRequest.setPrefix(prefix);
-    listObjectsRequest.setDelimiter(delimiter);
-    listObjectsRequest.setMarker(priorLastKey);
-    listObjectsRequest.setMaxKeys(maxListingLength);
-    ObjectListing objectListing = null;
-    try {
-      objectListing =
-          (ObjectListing) callCOSClientWithRetry(listObjectsRequest);
-    } catch (Exception e) {
-      String errMsg = String.format("prefix: [%s], delimiter: [%s], "
-              + "maxListingLength: [%d], priorLastKey: [%s]. "
-              + "List objects occur an exception: [%s].", prefix,
-          (delimiter == null) ? "" : delimiter, maxListingLength, priorLastKey,
-          e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), prefix);
-    }
-    ArrayList<FileMetadata> fileMetadataArray = new ArrayList<>();
-    ArrayList<FileMetadata> commonPrefixArray = new ArrayList<>();
-
-    if (null == objectListing) {
-      String errMsg = String.format("List the prefix: [%s] failed. " +
-              "delimiter: [%s], max listing length:" +
-              " [%s], prior last key: [%s]",
-          prefix, delimiter, maxListingLength, priorLastKey);
-      handleException(new Exception(errMsg), prefix);
-    }
-
-    List<COSObjectSummary> summaries = objectListing.getObjectSummaries();
-    for (COSObjectSummary cosObjectSummary : summaries) {
-      String filePath = cosObjectSummary.getKey();
-      if (!filePath.startsWith(CosNFileSystem.PATH_DELIMITER)) {
-        filePath = CosNFileSystem.PATH_DELIMITER + filePath;
-      }
-      if (filePath.equals(prefix)) {
-        continue;
-      }
-      long mtime = 0;
-      if (cosObjectSummary.getLastModified() != null) {
-        mtime = cosObjectSummary.getLastModified().getTime();
-      }
-      long fileLen = cosObjectSummary.getSize();
-      fileMetadataArray.add(
-          new FileMetadata(filePath, fileLen, mtime, true));
-    }
-    List<String> commonPrefixes = objectListing.getCommonPrefixes();
-    for (String commonPrefix : commonPrefixes) {
-      if (!commonPrefix.startsWith(CosNFileSystem.PATH_DELIMITER)) {
-        commonPrefix = CosNFileSystem.PATH_DELIMITER + commonPrefix;
-      }
-      commonPrefixArray.add(
-          new FileMetadata(commonPrefix, 0, 0, false));
-    }
-
-    FileMetadata[] fileMetadata = new FileMetadata[fileMetadataArray.size()];
-    for (int i = 0; i < fileMetadataArray.size(); ++i) {
-      fileMetadata[i] = fileMetadataArray.get(i);
-    }
-    FileMetadata[] commonPrefixMetaData =
-        new FileMetadata[commonPrefixArray.size()];
-    for (int i = 0; i < commonPrefixArray.size(); ++i) {
-      commonPrefixMetaData[i] = commonPrefixArray.get(i);
-    }
-    // when truncated is false, it means that listing is finished.
-    if (!objectListing.isTruncated()) {
-      return new PartialListing(
-          null, fileMetadata, commonPrefixMetaData);
-    } else {
-      return new PartialListing(
-          objectListing.getNextMarker(), fileMetadata, commonPrefixMetaData);
-    }
-  }
-
-  @Override
-  public void delete(String key) throws IOException {
-    LOG.debug("Delete object key: [{}] from bucket: {}.", key, this.bucketName);
-    try {
-      DeleteObjectRequest deleteObjectRequest =
-          new DeleteObjectRequest(bucketName, key);
-      callCOSClientWithRetry(deleteObjectRequest);
-    } catch (Exception e) {
-      String errMsg =
-          String.format("Delete key: [%s] occurs an exception: [%s].",
-              key, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-    }
-  }
-
-  public void rename(String srcKey, String dstKey) throws IOException {
-    LOG.debug("Rename source key: [{}] to dest key: [{}].", srcKey, dstKey);
-    try {
-      CopyObjectRequest copyObjectRequest =
-          new CopyObjectRequest(bucketName, srcKey, bucketName, dstKey);
-      callCOSClientWithRetry(copyObjectRequest);
-      DeleteObjectRequest deleteObjectRequest =
-          new DeleteObjectRequest(bucketName, srcKey);
-      callCOSClientWithRetry(deleteObjectRequest);
-    } catch (Exception e) {
-      String errMsg = String.format("Rename object unsuccessfully. "
-              + "source cos key: [%s], dest COS " +
-              "key: [%s], exception: [%s]",
-          srcKey,
-          dstKey, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), srcKey);
-    }
-  }
-
-  @Override
-  public void copy(String srcKey, String dstKey) throws IOException {
-    LOG.debug("Copy source key: [{}] to dest key: [{}].", srcKey, dstKey);
-    try {
-      CopyObjectRequest copyObjectRequest =
-          new CopyObjectRequest(bucketName, srcKey, bucketName, dstKey);
-      callCOSClientWithRetry(copyObjectRequest);
-    } catch (Exception e) {
-      String errMsg = String.format("Copy object unsuccessfully. "
-              + "source COS key: %s, dest COS key: " +
-              "%s, exception: %s",
-          srcKey,
-          dstKey, e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), srcKey);
-    }
-  }
-
-  @Override
-  public void purge(String prefix) throws IOException {
-    throw new IOException("purge not supported");
-  }
-
-  @Override
-  public void dump() throws IOException {
-    throw new IOException("dump not supported");
-  }
-
-  // process Exception and print detail
-  private void handleException(Exception e, String key) throws IOException {
-    String cosPath = CosNFileSystem.SCHEME + "://" + bucketName + key;
-    String exceptInfo = String.format("%s : %s", cosPath, e.toString());
-    throw new IOException(exceptInfo);
-  }
-
-  @Override
-  public long getFileLength(String key) throws IOException {
-    LOG.debug("Get file length. COS key: {}", key);
-    GetObjectMetadataRequest getObjectMetadataRequest =
-        new GetObjectMetadataRequest(bucketName, key);
-    try {
-      ObjectMetadata objectMetadata =
-          (ObjectMetadata) callCOSClientWithRetry(getObjectMetadataRequest);
-      return objectMetadata.getContentLength();
-    } catch (Exception e) {
-      String errMsg = String.format("Getting file length occurs an exception." +
-              "COS key: %s, exception: %s", key,
-          e.toString());
-      LOG.error(errMsg);
-      handleException(new Exception(errMsg), key);
-      return 0; // never will get here
-    }
-  }
-
-  private <X> Object callCOSClientWithRetry(X request)
-      throws CosServiceException, IOException {
-    String sdkMethod = "";
-    int retryIndex = 1;
-    while (true) {
-      try {
-        if (request instanceof PutObjectRequest) {
-          sdkMethod = "putObject";
-          return this.cosClient.putObject((PutObjectRequest) request);
-        } else if (request instanceof UploadPartRequest) {
-          sdkMethod = "uploadPart";
-          if (((UploadPartRequest) request).getInputStream()
-              instanceof ByteBufferInputStream) {
-            ((UploadPartRequest) request).getInputStream()
-                .mark((int) ((UploadPartRequest) request).getPartSize());
-          }
-          return this.cosClient.uploadPart((UploadPartRequest) request);
-        } else if (request instanceof GetObjectMetadataRequest) {
-          sdkMethod = "queryObjectMeta";
-          return this.cosClient.getObjectMetadata(
-              (GetObjectMetadataRequest) request);
-        } else if (request instanceof DeleteObjectRequest) {
-          sdkMethod = "deleteObject";
-          this.cosClient.deleteObject((DeleteObjectRequest) request);
-          return new Object();
-        } else if (request instanceof CopyObjectRequest) {
-          sdkMethod = "copyFile";
-          return this.cosClient.copyObject((CopyObjectRequest) request);
-        } else if (request instanceof GetObjectRequest) {
-          sdkMethod = "getObject";
-          return this.cosClient.getObject((GetObjectRequest) request);
-        } else if (request instanceof ListObjectsRequest) {
-          sdkMethod = "listObjects";
-          return this.cosClient.listObjects((ListObjectsRequest) request);
-        } else {
-          throw new IOException("no such method");
-        }
-      } catch (CosServiceException cse) {
-        String errMsg = String.format("Call cos sdk failed, "
-                + "retryIndex: [%d / %d], "
-                + "call method: %s, exception: %s",
-            retryIndex, this.maxRetryTimes, sdkMethod, cse.toString());
-        int statusCode = cse.getStatusCode();
-        // Retry all server errors
-        if (statusCode / 100 == 5) {
-          if (retryIndex <= this.maxRetryTimes) {
-            LOG.info(errMsg);
-            long sleepLeast = retryIndex * 300L;
-            long sleepBound = retryIndex * 500L;
-            try {
-              if (request instanceof UploadPartRequest) {
-                if (((UploadPartRequest) request).getInputStream()
-                    instanceof ByteBufferInputStream) {
-                  ((UploadPartRequest) request).getInputStream().reset();
-                }
-              }
-              Thread.sleep(
-                  ThreadLocalRandom.current().nextLong(sleepLeast, sleepBound));
-              ++retryIndex;
-            } catch (InterruptedException e) {
-              throw new IOException(e.toString());
-            }
-          } else {
-            LOG.error(errMsg);
-            throw new IOException(errMsg);
-          }
-        } else {
-          throw cse;
-        }
-      } catch (Exception e) {
-        String errMsg = String.format("Call cos sdk failed, "
-            + "call method: %s, exception: %s", sdkMethod, e.toString());
-        LOG.error(errMsg);
-        throw new IOException(errMsg);
-      }
-    }
-  }
-
-  @Override
-  public void close() {
-    if (null != this.cosClient) {
-      this.cosClient.shutdown();
-    }
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/FileMetadata.java

@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * <p>
- * Holds basic metadata for a file stored in a {@link NativeFileSystemStore}.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class FileMetadata {
-  private final String key;
-  private final long length;
-  private final long lastModified;
-  private final boolean isFile;
-
-  FileMetadata(String key, long length, long lastModified) {
-    this(key, length, lastModified, true);
-  }
-
-  FileMetadata(String key, long length, long lastModified, boolean isFile) {
-    this.key = key;
-    this.length = length;
-    this.lastModified = lastModified;
-    this.isFile = isFile;
-  }
-
-  public String getKey() {
-    return key;
-  }
-
-  public long getLength() {
-    return length;
-  }
-
-  public long getLastModified() {
-    return lastModified;
-  }
-
-  @Override
-  public String toString() {
-    return "FileMetadata[" + key + ", " + length + ", " + lastModified + ", "
-        + "file?" + isFile + "]";
-  }
-
-  public boolean isFile() {
-    return isFile;
-  }
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/NativeFileSystemStore.java

@@ -1,99 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.List;
-
-import com.qcloud.cos.model.CompleteMultipartUploadResult;
-import com.qcloud.cos.model.PartETag;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * <p>
- * An abstraction for a key-based {@link File} store.
- * </p>
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-interface NativeFileSystemStore {
-
-  void initialize(URI uri, Configuration conf) throws IOException;
-
-  void storeFile(String key, File file, byte[] md5Hash) throws IOException;
-
-  void storeFile(String key, InputStream inputStream, byte[] md5Hash,
-      long contentLength) throws IOException;
-
-  void storeEmptyFile(String key) throws IOException;
-
-  CompleteMultipartUploadResult completeMultipartUpload(
-      String key, String uploadId, List<PartETag> partETagList);
-
-  void abortMultipartUpload(String key, String uploadId);
-
-  String getUploadId(String key);
-
-  PartETag uploadPart(File file, String key, String uploadId, int partNum)
-      throws IOException;
-
-  PartETag uploadPart(InputStream inputStream, String key, String uploadId,
-      int partNum, long partSize) throws IOException;
-
-  FileMetadata retrieveMetadata(String key) throws IOException;
-
-  InputStream retrieve(String key) throws IOException;
-
-  InputStream retrieve(String key, long byteRangeStart) throws IOException;
-
-  InputStream retrieveBlock(String key, long byteRangeStart, long byteRangeEnd)
-      throws IOException;
-
-  long getFileLength(String key) throws IOException;
-
-  PartialListing list(String prefix, int maxListingLength) throws IOException;
-
-  PartialListing list(String prefix, int maxListingLength,
-      String priorLastKey, boolean recursive) throws IOException;
-
-  void delete(String key) throws IOException;
-
-  void copy(String srcKey, String dstKey) throws IOException;
-
-  /**
-   * Delete all keys with the given prefix. Used for testing.
-   *
-   * @throws IOException if purge is not supported
-   */
-  void purge(String prefix) throws IOException;
-
-  /**
-   * Diagnostic method to dump state to the console.
-   *
-   * @throws IOException if dump is not supported
-   */
-  void dump() throws IOException;
-
-  void close();
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/PartialListing.java

@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * <p>
- * Holds information on a directory listing for a
- * {@link NativeFileSystemStore}.
- * This includes the {@link FileMetadata files} and directories
- * (their names) contained in a directory.
- * </p>
- * <p>
- * This listing may be returned in chunks, so a <code>priorLastKey</code>
- * is provided so that the next chunk may be requested.
- * </p>
- *
- * @see NativeFileSystemStore#list(String, int, String, boolean)
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class PartialListing {
-
-  private final String priorLastKey;
-  private final FileMetadata[] files;
-  private final FileMetadata[] commonPrefixes;
-
-  PartialListing(String priorLastKey, FileMetadata[] files,
-                 FileMetadata[] commonPrefixes) {
-    this.priorLastKey = priorLastKey;
-    this.files = files;
-    this.commonPrefixes = commonPrefixes;
-  }
-
-  public FileMetadata[] getFiles() {
-    return files;
-  }
-
-  public FileMetadata[] getCommonPrefixes() {
-    return commonPrefixes;
-  }
-
-  public String getPriorLastKey() {
-    return priorLastKey;
-  }
-
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/Unit.java

@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn;
-
-/**
- * Constant definition of storage unit.
- */
-public final class Unit {
-  private Unit() {
-  }
-
-  public static final int KB = 1024;
-  public static final int MB = 1024 * KB;
-  public static final int GB = 1024 * MB;
-  public static final long TB = (long) 1024 * GB;
-  public static final long PB = (long) 1024 * TB;
-}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java

@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn.auth;
-
-import com.qcloud.cos.auth.COSCredentialsProvider;
-import org.apache.hadoop.conf.Configuration;
-
-import javax.annotation.Nullable;
-import java.net.URI;
-
-/**
- * The base class for COS credential providers which take a URI or
- * configuration in their constructor.
- */
-public abstract class AbstractCOSCredentialsProvider
-    implements COSCredentialsProvider {
-  private final URI uri;
-  private final Configuration conf;
-
-  public AbstractCOSCredentialsProvider(@Nullable URI uri,
-                                        Configuration conf) {
-    this.uri = uri;
-    this.conf = conf;
-  }
-
-  public URI getUri() {
-    return uri;
-  }
-
-  public Configuration getConf() {
-    return conf;
-  }
-}