HADOOP-12364. Deleting pid file after stop is causing the daemons to keep restarting (Sigi Li via aw)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -510,6 +510,9 @@ Trunk (Unreleased)
     HADOOP-11942. Add links to SLGUserGuide to site index.
     (Masatake Iwasaki via xyao)
 
+    HADOOP-12364. Deleting pid file after stop is causing the daemons to
+    keep restarting (Sigi Li via aw)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)

hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh

@@ -1501,7 +1501,7 @@ function hadoop_start_secure_daemon
   local daemonname=$1
   local class=$2
 
-  # pid file to create for our deamon
+  # pid file to create for our daemon
   local daemonpidfile=$3
 
   # where to send stdout. jsvc has bad habits so this *may* be &1
@@ -1662,6 +1662,7 @@ function hadoop_stop_daemon
   shift 2
 
   local pid
+  local cur_pid
 
   if [[ -f "${pidfile}" ]]; then
     pid=$(cat "$pidfile")
@@ -1675,7 +1676,12 @@ function hadoop_stop_daemon
     if ps -p "${pid}" > /dev/null 2>&1; then
       hadoop_error "ERROR: Unable to kill ${pid}"
     else
-      rm -f "${pidfile}" >/dev/null 2>&1
+      cur_pid=$(cat "$pidfile")
+      if [[ "${pid}" = "${cur_pid}" ]]; then
+        rm -f "${pidfile}" >/dev/null 2>&1
+      else
+        hadoop_error "WARNING: pid has changed for ${cmd}, skip deleting pid file"
+      fi
     fi
   fi
 }
@@ -1697,9 +1703,31 @@ function hadoop_stop_secure_daemon
   shift 3
   local ret
 
+  local daemon_pid
+  local priv_pid
+  local cur_daemon_pid
+  local cur_priv_pid
+
+  daemon_pid=$(cat "$daemonpidfile")
+  priv_pid=$(cat "$privpidfile")
+
   hadoop_stop_daemon "${command}" "${daemonpidfile}"
   ret=$?
-  rm -f "${daemonpidfile}" "${privpidfile}" 2>/dev/null
+
+  cur_daemon_pid=$(cat "$daemonpidfile")
+  cur_priv_pid=$(cat "$privpidfile")
+
+  if [[ "${daemon_pid}" = "${cur_daemon_pid}" ]]; then
+    rm -f "${daemonpidfile}" >/dev/null 2>&1
+  else
+    hadoop_error "WARNING: daemon pid has changed for ${command}, skip deleting daemon pid file"
+  fi
+
+  if [[ "${priv_pid}" = "${cur_priv_pid}" ]]; then
+    rm -f "${privpidfile}" >/dev/null 2>&1
+  else
+    hadoop_error "WARNING: priv pid has changed for ${command}, skip deleting priv pid file"
+  fi
   return ${ret}
 }
 
@@ -1956,4 +1984,4 @@ function hadoop_parse_args
   done
 
   hadoop_debug "hadoop_parse: asking caller to skip ${HADOOP_PARSE_COUNTER}"
-}
+}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_daemon.bats

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_stop_daemon" {
+  old_pid=12345
+  new_pid=54321
+  HADOOP_STOP_TIMEOUT=3
+
+  echo ${old_pid} > pidfile
+  run hadoop_stop_daemon stop pidfile &
+  sleep 1
+  echo ${new_pid} > pidfile
+  sleep ${HADOOP_STOP_TIMEOUT}
+
+  [ -f pidfile ]
+  [ "$(cat pidfile)" = "${new_pid}" ]
+}

hadoop-common-project/hadoop-common/src/test/scripts/hadoop_stop_secure_daemon.bats

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_stop_secure_daemon" {
+  old_daemon_pid=12345
+  old_priv_pid=23456
+  new_daemon_pid=54321
+  new_priv_pid=65432
+  HADOOP_STOP_TIMEOUT=3
+
+  echo ${old_daemon_pid} > daemonpidfile
+  echo ${old_priv_pid} > privpidfile
+  run hadoop_stop_secure_daemon stop daemonpidfile privpidfile &
+  sleep 1
+  echo ${new_daemon_pid} > daemonpidfile
+  echo ${new_priv_pid} > privpidfile
+  sleep ${HADOOP_STOP_TIMEOUT}
+
+  [ -f daemonpidfile ]
+  [ "$(cat daemonpidfile)" = "${new_daemon_pid}" ]
+  [ -f privpidfile ]
+  [ "$(cat privpidfile)" = "${new_priv_pid}" ]
+}