Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2198. Remove the need to run NodeManager as privileged account for Windows Secure Container Executor. Contributed by Remus Rusanu

This commit is contained in:
Jian He 2014-10-22 15:57:46 -07:00
parent a36399e09c
commit 3b12fd6cfb
37 changed files with 5555 additions and 787 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -4,6 +4,9 @@
*.orig
*.rej
**/.keep
*.sdf
*.suo
*.vcxproj.user
.idea
.svn
.classpath

5
hadoop-common-project/hadoop-common/pom.xml
View File

@ -34,6 +34,8 @@
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
<wsce.config.dir>../etc/hadoop</wsce.config.dir>
<wsce.config.file>wsce-site.xml</wsce.config.file>
</properties>
@ -714,6 +716,9 @@
<argument>/nologo</argument>
<argument>/p:Configuration=Release</argument>
<argument>/p:OutDir=${project.build.directory}/bin/</argument>
<argument>/p:IntermediateOutputPath=${project.build.directory}/winutils/</argument>
<argument>/p:WsceConfigDir=${wsce.config.dir}</argument>
<argument>/p:WsceConfigFile=${wsce.config.file}</argument>
</arguments>
</configuration>
</execution>

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
View File

@ -1186,6 +1186,11 @@ public class FileUtil {
return fileNames;
}
public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
Map<String, String> callerEnv) throws IOException {
return createJarWithClassPath(inputClassPath, pwd, pwd, callerEnv);
}
/**
* Create a jar file at the given path, containing a manifest with a classpath
* that references all specified entries.
@ -1210,6 +1215,7 @@ public class FileUtil {
*
* @param inputClassPath String input classpath to bundle into the jar manifest
* @param pwd Path to working directory to save jar
* @param targetDir path to where the jar execution will have its working dir
* @param callerEnv Map<String, String> caller's environment variables to use
* for expansion
* @return String[] with absolute path to new jar in position 0 and
@ -1217,6 +1223,7 @@ public class FileUtil {
* @throws IOException if there is an I/O error while writing the jar file
*/
public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
Path targetDir,
Map<String, String> callerEnv) throws IOException {
// Replace environment variables, case-insensitive on Windows
@SuppressWarnings("unchecked")
@ -1265,7 +1272,7 @@ public class FileUtil {
// Append just this entry
File fileCpEntry = null;
if(!new Path(classPathEntry).isAbsolute()) {
fileCpEntry = new File(workingDir, classPathEntry);
fileCpEntry = new File(targetDir.toString(), classPathEntry);
}
else {
fileCpEntry = new File(classPathEntry);

18
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java
View File

@ -268,7 +268,12 @@ public class RawLocalFileSystem extends FileSystem {
throw new IOException("Mkdirs failed to create " + parent.toString());
}
return new FSDataOutputStream(new BufferedOutputStream(
new LocalFSFileOutputStream(f, false), bufferSize), statistics);
createOutputStream(f, false), bufferSize), statistics);
}
protected OutputStream createOutputStream(Path f, boolean append)
throws IOException {
return new LocalFSFileOutputStream(f, append);
}
@Override
@ -407,6 +412,10 @@ public class RawLocalFileSystem extends FileSystem {
return Arrays.copyOf(results, j);
}
protected boolean mkOneDir(File p2f) throws IOException {
return p2f.mkdir();
}
/**
* Creates the specified directory hierarchy. Does not
* treat existence as an error.
@ -418,8 +427,9 @@ public class RawLocalFileSystem extends FileSystem {
}
Path parent = f.getParent();
File p2f = pathToFile(f);
File parent2f = null;
if(parent != null) {
File parent2f = pathToFile(parent);
parent2f = pathToFile(parent);
if(parent2f != null && parent2f.exists() && !parent2f.isDirectory()) {
throw new ParentNotDirectoryException("Parent path is not a directory: "
+ parent);
@ -429,8 +439,8 @@ public class RawLocalFileSystem extends FileSystem {
throw new FileNotFoundException("Destination exists" +
" and is not a directory: " + p2f.getCanonicalPath());
}
return (parent == null || mkdirs(parent)) &&
(p2f.mkdir() || p2f.isDirectory());
return (parent == null || parent2f.exists() || mkdirs(parent)) &&
(mkOneDir(p2f) || p2f.isDirectory());
}
@Override

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
View File

@ -22,6 +22,8 @@ import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.reflect.Field;
import java.nio.ByteBuffer;
@ -35,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.HardLink;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
@ -505,6 +508,8 @@ public class NativeIO {
public static final long FILE_CURRENT = 1;
public static final long FILE_END = 2;
public static final long FILE_ATTRIBUTE_NORMAL = 0x00000080L;
/** Wrapper around CreateFile() on Windows */
public static native FileDescriptor createFile(String path,
long desiredAccess, long shareMode, long creationDisposition)

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
View File

@ -649,6 +649,18 @@ abstract public class Shell {
}
}
public interface CommandExecutor {
void execute() throws IOException;
int getExitCode() throws IOException;
String getOutput() throws IOException;
void close();
}
/**
* A simple shell command executor.
*
@ -657,7 +669,8 @@ abstract public class Shell {
* directory and the environment remains unchanged. The output of the command
* is stored as-is and is expected to be small.
*/
public static class ShellCommandExecutor extends Shell {
public static class ShellCommandExecutor extends Shell
implements CommandExecutor {
private String[] command;
private StringBuffer output;
@ -749,6 +762,10 @@ abstract public class Shell {
}
return builder.toString();
}
@Override
public void close() {
}
}
/**

4
hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
View File

@ -107,6 +107,9 @@
<AdditionalOptions Condition="'$(SnappyEnabled)' == 'true'">/D HADOOP_SNAPPY_LIBRARY=L\"snappy.dll\"</AdditionalOptions>
</ClCompile>
<ClCompile Include="src\org\apache\hadoop\util\NativeCrc32.c" />
<ClCompile Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.c">
<AdditionalIncludeDirectories>src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" />
@ -120,6 +123,7 @@
<ClInclude Include="src\org\apache\hadoop\util\crc32c_tables.h" />
<ClInclude Include="src\org\apache\hadoop\util\crc32_zlib_polynomial_tables.h" />
<ClInclude Include="src\org_apache_hadoop.h" />
<ClInclude Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">

11
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
View File

@ -71,8 +71,13 @@ static jmethodID nioe_ctor;
// Please see HADOOP-7156 for details.
jobject pw_lock_object;
/*
* Throw a java.IO.IOException, generating the message from errno.
* NB. this is also used form windows_secure_container_executor.c
*/
extern void throw_ioe(JNIEnv* env, int errnum);
// Internal functions
static void throw_ioe(JNIEnv* env, int errnum);
#ifdef UNIX
static ssize_t get_pw_buflen();
#endif
@ -216,7 +221,7 @@ static int map_fadvise_flag(jint flag) {
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_nativeio_NativeIO_initNative(
JNIEnv *env, jclass clazz) {
JNIEnv *env, jclass clazz) {
stat_init(env, clazz);
PASS_EXCEPTIONS_GOTO(env, error);
nioe_init(env);
@ -802,7 +807,7 @@ cleanup:
/*
* Throw a java.IO.IOException, generating the message from errno.
*/
static void throw_ioe(JNIEnv* env, int errnum)
void throw_ioe(JNIEnv* env, int errnum)
{
#ifdef UNIX
char message[80];

569
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/yarn/server/nodemanager/windows_secure_container_executor.c Normal file
View File

@ -0,0 +1,569 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
#include <jni.h>
#include "org_apache_hadoop.h"
#include "windows_secure_container_executor.h"
#include "winutils.h"
#include "file_descriptor.h"
// class of org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor.Native.WinutilsProcessStub
static jclass wps_class = NULL;
/*
* private static native void initWsceNative();
*
* We rely on this function rather than lazy initialization because
* the lazy approach may have a race if multiple callers try to
* init at the same time.
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_initWsceNative(
JNIEnv *env, jclass clazz) {
#ifdef WINDOWS
winutils_process_stub_init(env);
PASS_EXCEPTIONS_GOTO(env, error);
#endif
return;
error:
// these are all idempodent and safe to call even if the
// class wasn't initted yet
#ifdef WINDOWS
winutils_process_stub_deinit(env);
#endif
}
static jmethodID wps_constructor = NULL;
static jfieldID wps_hProcess = NULL;
static jfieldID wps_hThread = NULL;
static jfieldID wps_disposed = NULL;
extern void throw_ioe(JNIEnv* env, int errnum);
void winutils_process_stub_init(JNIEnv *env) {
if (wps_class != NULL) return; // already initted
wps_class = (*env)->FindClass(env, WINUTILS_PROCESS_STUB_CLASS);
PASS_EXCEPTIONS(env);
wps_class = (*env)->NewGlobalRef(env, wps_class);
PASS_EXCEPTIONS(env);
wps_hProcess = (*env)->GetFieldID(env, wps_class, "hProcess", "J");
PASS_EXCEPTIONS(env);
wps_hThread = (*env)->GetFieldID(env, wps_class, "hThread", "J");
PASS_EXCEPTIONS(env);
wps_disposed = (*env)->GetFieldID(env, wps_class, "disposed", "Z");
PASS_EXCEPTIONS(env);
wps_constructor = (*env)->GetMethodID(env, wps_class, "<init>", "(JJJJJ)V");
PASS_EXCEPTIONS(env);
LogDebugMessage(L"winutils_process_stub_init\n");
}
void winutils_process_stub_deinit(JNIEnv *env) {
if (wps_class != NULL) {
(*env)->DeleteGlobalRef(env, wps_class);
wps_class = NULL;
}
wps_hProcess = NULL;
wps_hThread = NULL;
wps_disposed = NULL;
wps_constructor = NULL;
LogDebugMessage(L"winutils_process_stub_deinit\n");
}
jobject winutils_process_stub_create(JNIEnv *env,
jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr) {
jobject obj = (*env)->NewObject(env, wps_class, wps_constructor,
hProcess, hThread, hStdIn, hStdOut, hStdErr);
PASS_EXCEPTIONS_RET(env, NULL);
LogDebugMessage(L"winutils_process_stub_create: %p\n", obj);
return obj;
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native
* Method: createTaskAsUser
* Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String)Lorg/apache/hadoop/io/nativeio/NativeIO$WinutilsProcessStub
*/
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_createTaskAsUser0(JNIEnv* env,
jclass clazz, jstring jcwd, jstring jjobName, jstring juser, jstring jpidFile, jstring jcmdLine) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function createTaskAsUser is not supported on Unix");
return NULL;
#endif
#ifdef WINDOWS
LPCWSTR cwd = NULL, jobName = NULL,
user = NULL, pidFile = NULL, cmdLine = NULL;
DWORD dwError = ERROR_SUCCESS;
HANDLE hProcess = INVALID_HANDLE_VALUE,
hThread = INVALID_HANDLE_VALUE,
hStdIn = INVALID_HANDLE_VALUE,
hStdOut = INVALID_HANDLE_VALUE,
hStdErr = INVALID_HANDLE_VALUE;
jobject ret = NULL;
cwd = (LPCWSTR) (*env)->GetStringChars(env, jcwd, NULL);
if (!cwd) goto done; // exception was thrown
jobName = (LPCWSTR) (*env)->GetStringChars(env, jjobName, NULL);
if (!jobName) goto done; // exception was thrown
user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL);
if (!user) goto done; // exception was thrown
pidFile = (LPCWSTR) (*env)->GetStringChars(env, jpidFile, NULL);
if (!pidFile) goto done; // exception was thrown
cmdLine = (LPCWSTR) (*env)->GetStringChars(env, jcmdLine, NULL);
if (!cmdLine) goto done; // exception was thrown
LogDebugMessage(L"createTaskAsUser: jcwd:%s job:%s juser:%s pid:%s cmd:%s\n",
cwd, jobName, user, pidFile, cmdLine);
dwError = RpcCall_TaskCreateAsUser(cwd, jobName, user, pidFile, cmdLine,
&hProcess, &hThread, &hStdIn, &hStdOut, &hStdErr);
if (ERROR_SUCCESS == dwError) {
ret = winutils_process_stub_create(env, (jlong) hProcess, (jlong) hThread,
(jlong) hStdIn, (jlong) hStdOut, (jlong) hStdErr);
if (NULL == ret) {
TerminateProcess(hProcess, EXIT_FAILURE);
CloseHandle(hThread);
CloseHandle(hProcess);
CloseHandle(hStdIn);
CloseHandle(hStdOut);
CloseHandle(hStdErr);
}
}
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (cwd) (*env)->ReleaseStringChars(env, jcwd, cwd);
if (jobName) (*env)->ReleaseStringChars(env, jjobName, jobName);
if (user) (*env)->ReleaseStringChars(env, juser, user);
if (pidFile) (*env)->ReleaseStringChars(env, jpidFile, pidFile);
if (cmdLine) (*env)->ReleaseStringChars(env, jcmdLine, cmdLine);
return ret;
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedKillTaskImpl
* Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedKillTaskImpl(JNIEnv* env,
jclass clazz, jstring jtask_name) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedSetOwner0 is not supported on Unix");
return;
#endif
#ifdef WINDOWS
LPCWSTR task_name = NULL;
DWORD dwError;
task_name = (LPCWSTR) (*env)->GetStringChars(env, jtask_name, NULL);
if (!task_name) goto done; // exception was thrown
dwError = RpcCall_WinutilsKillTask(task_name);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (task_name) (*env)->ReleaseStringChars(env, jtask_name, task_name);
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedChownImpl
* Signature: (Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedChownImpl(JNIEnv* env,
jclass clazz, jstring jpath, jstring juser, jstring jgroup) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedSetOwner0 is not supported on Unix");
return;
#endif
#ifdef WINDOWS
LPCWSTR path = NULL, user = NULL, group = NULL;
DWORD dwError;
path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
if (!path) goto done; // exception was thrown
if (juser) {
user = (LPCWSTR) (*env)->GetStringChars(env, juser, NULL);
if (!user) goto done; // exception was thrown
}
if (jgroup) {
group = (LPCWSTR) (*env)->GetStringChars(env, jgroup, NULL);
if (!group) goto done; // exception was thrown
}
dwError = RpcCall_WinutilsChown(path, user, group);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (path) (*env)->ReleaseStringChars(env, jpath, path);
if (user) (*env)->ReleaseStringChars(env, juser, user);
if (group) (*env)->ReleaseStringChars(env, jgroup, group);
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedMkDirImpl
* Signature: (Ljava/lang/String;)V
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedMkDirImpl(JNIEnv* env,
jclass clazz, jstring jpath) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedMkDirImpl is not supported on Unix");
return;
#endif
#ifdef WINDOWS
LPCWSTR path = NULL, user = NULL, group = NULL;
DWORD dwError;
path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
if (!path) goto done; // exception was thrown
dwError = RpcCall_WinutilsMkDir(path);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (path) (*env)->ReleaseStringChars(env, jpath, path);
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedChmodImpl
* Signature: (Ljava/lang/String;I)V
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedChmodImpl(JNIEnv* env,
jclass clazz, jstring jpath, jint jmode) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedChmodImpl is not supported on Unix");
return;
#endif
#ifdef WINDOWS
LPCWSTR path = NULL;
DWORD dwError;
path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
if (!path) goto done; // exception was thrown
dwError = RpcCall_WinutilsChmod(path, (int) jmode);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (path) (*env)->ReleaseStringChars(env, jpath, path);
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedCopyImpl
* Signature: (I;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Z)V
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedCopyImpl(JNIEnv* env,
jclass clazz, jint joperation, jstring jsourcePath, jstring jdestinationPath, jboolean replaceExisting) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedCopyImpl is not supported on Unix");
return;
#endif
#ifdef WINDOWS
LPCWSTR sourcePath = NULL, destinationPath = NULL;
DWORD dwError;
sourcePath = (LPCWSTR) (*env)->GetStringChars(env, jsourcePath, NULL);
if (!sourcePath) goto done; // exception was thrown
destinationPath = (LPCWSTR) (*env)->GetStringChars(env, jdestinationPath, NULL);
if (!destinationPath) goto done; // exception was thrown
dwError = RpcCall_WinutilsMoveFile((INT) joperation, sourcePath, destinationPath, (BOOL) replaceExisting);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (sourcePath) (*env)->ReleaseStringChars(env, jsourcePath, sourcePath);
if (destinationPath) (*env)->ReleaseStringChars(env, jdestinationPath, destinationPath);
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedCreateImpl
* Signature: (Ljava/lang/String;J;J;J;J)J
*/
JNIEXPORT jlong JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedCreateImpl(JNIEnv* env,
jclass clazz, jstring jpath, jlong jdesired_access, jlong jshare_mode, jlong jcreation_disposition, jlong jflags) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedCreateImpl is not supported on Unix");
return 0;
#endif
#ifdef WINDOWS
LPCWSTR path = NULL;
DWORD dwError;
HANDLE hFile = INVALID_HANDLE_VALUE;
path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
if (!path) goto done; // exception was thrown
dwError = RpcCall_WinutilsCreateFile(path,
(DWORD) jdesired_access, (DWORD) jshare_mode, (DWORD) jcreation_disposition, (DWORD) jflags,
&hFile);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (path) (*env)->ReleaseStringChars(env, jpath, path);
return hFile;
#endif
}
/*
* Class: org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor$Native$Elevated
* Method: elevatedDeletePathImpl
* Signature: (Ljava/lang/String;Z)Z
*/
JNIEXPORT jboolean JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024Elevated_elevatedDeletePathImpl(JNIEnv* env,
jclass clazz, jstring jpath, jboolean jIsDir) {
#ifdef UNIX
THROW(env, "java/io/IOException",
"The function elevatedDeleteFileImpl is not supported on Unix");
return JNI_FALSE;
#endif
#ifdef WINDOWS
LPCWSTR path = NULL;
DWORD dwError;
BOOL deleted = FALSE;
path = (LPCWSTR) (*env)->GetStringChars(env, jpath, NULL);
if (!path) goto done; // exception was thrown
dwError = RpcCall_WinutilsDeletePath(path, (BOOL) jIsDir, &deleted);
if (dwError != ERROR_SUCCESS) {
throw_ioe (env, dwError);
}
done:
if (path) (*env)->ReleaseStringChars(env, jpath, path);
return (jboolean) deleted;
#endif
}
/*
* native void destroy();
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_destroy(
JNIEnv *env, jobject objSelf) {
HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
LogDebugMessage(L"TerminateProcess: %x\n", hProcess);
TerminateProcess(hProcess, EXIT_FAILURE);
}
/*
* native void waitFor();
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_waitFor(
JNIEnv *env, jobject objSelf) {
HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
LogDebugMessage(L"WaitForSingleObject: %x\n", hProcess);
WaitForSingleObject(hProcess, INFINITE);
}
/*
* native void resume();
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_resume(
JNIEnv *env, jobject objSelf) {
DWORD dwError;
HANDLE hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread);
if (-1 == ResumeThread(hThread)) {
dwError = GetLastError();
LogDebugMessage(L"ResumeThread: %x error:%d\n", hThread, dwError);
throw_ioe(env, dwError);
}
}
/*
* native int exitValue();
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT jint JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_exitValue(
JNIEnv *env, jobject objSelf) {
DWORD exitCode;
DWORD dwError;
HANDLE hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
if (!GetExitCodeProcess(hProcess, &exitCode)) {
dwError = GetLastError();
throw_ioe(env, dwError);
return dwError; // exception was thrown, return value doesn't really matter
}
LogDebugMessage(L"GetExitCodeProcess: %x :%d\n", hProcess, exitCode);
return exitCode;
}
/*
* native void dispose();
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT void JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_dispose(
JNIEnv *env, jobject objSelf) {
HANDLE hProcess = INVALID_HANDLE_VALUE,
hThread = INVALID_HANDLE_VALUE;
jboolean disposed = (*env)->GetBooleanField(env, objSelf, wps_disposed);
if (JNI_TRUE != disposed) {
hProcess = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hProcess);
hThread = (HANDLE)(*env)->GetLongField(env, objSelf, wps_hThread);
CloseHandle(hProcess);
CloseHandle(hThread);
(*env)->SetBooleanField(env, objSelf, wps_disposed, JNI_TRUE);
LogDebugMessage(L"disposed: %p\n", objSelf);
}
}
/*
* native static FileDescriptor getFileDescriptorFromHandle(long handle);
*
* The "00024" in the function name is an artifact of how JNI encodes
* special characters. U+0024 is '$'.
*/
JNIEXPORT jobject JNICALL
Java_org_apache_hadoop_yarn_server_nodemanager_WindowsSecureContainerExecutor_00024Native_00024WinutilsProcessStub_getFileDescriptorFromHandle(
JNIEnv *env, jclass klass, jlong handle) {
LogDebugMessage(L"getFileDescriptorFromHandle: %x\n", handle);
return fd_create(env, (long) handle);
}

27
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/yarn/server/nodemanager/windows_secure_container_executor.h Normal file
View File

@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#define WINUTILS_PROCESS_STUB_CLASS "org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor$Native$WinutilsProcessStub"
void winutils_process_stub_init(JNIEnv *env);
void winutils_process_stub_deinit(JNIEnv *env);
jobject winutils_process_stub_create(JNIEnv *env,
jlong hProcess, jlong hThread, jlong hStdIn, jlong hStdOut, jlong hStdErr);

130
hadoop-common-project/hadoop-common/src/main/winutils/chown.c
View File

@ -17,93 +17,6 @@
#include "winutils.h"
//----------------------------------------------------------------------------
// Function: ChangeFileOwnerBySid
//
// Description:
// Change a file or directory ownership by giving new owner and group SIDs
//
// Returns:
// ERROR_SUCCESS: on success
// Error code: otherwise
//
// Notes:
// This function is long path safe, i.e. the path will be converted to long
// path format if not already converted. So the caller does not need to do
// the converstion before calling the method.
//
static DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
__in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
{
LPWSTR longPathName = NULL;
INT oldMode = 0;
SECURITY_INFORMATION securityInformation = 0;
DWORD dwRtnCode = ERROR_SUCCESS;
// Convert the path the the long path
//
dwRtnCode = ConvertToLongPath(path, &longPathName);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// Get a pointer to the existing owner information and DACL
//
dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// We need SeTakeOwnershipPrivilege to set the owner if the caller does not
// have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
// SID is not contained in the caller's token, and have the SE_GROUP_OWNER
// permission enabled.
//
if (EnablePrivilege(L"SeTakeOwnershipPrivilege") != ERROR_SUCCESS)
{
fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
}
if (EnablePrivilege(L"SeRestorePrivilege") != ERROR_SUCCESS)
{
fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
}
assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
// Set the owners of the file.
//
if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
dwRtnCode = SetNamedSecurityInfoW(
longPathName,
SE_FILE_OBJECT,
securityInformation,
pNewOwnerSid,
pNewGroupSid,
NULL,
NULL);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// Set the permission on the file for the new owner.
//
dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
ChangeFileOwnerByNameEnd:
LocalFree(longPathName);
return dwRtnCode;
}
//----------------------------------------------------------------------------
// Function: Chown
//
@ -130,9 +43,6 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
LPWSTR groupName = NULL;
size_t groupNameLen = 0;
PSID pNewOwnerSid = NULL;
PSID pNewGroupSid = NULL;
DWORD dwRtnCode = 0;
int ret = EXIT_FAILURE;
@ -210,48 +120,16 @@ int Chown(__in int argc, __in_ecount(argc) wchar_t *argv[])
goto ChownEnd;
}
if (userName != NULL)
{
dwRtnCode = GetSidFromAcctNameW(userName, &pNewOwnerSid);
if (dwRtnCode != ERROR_SUCCESS)
{
ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
fwprintf(stderr, L"Invalid user name: %s\n", userName);
goto ChownEnd;
}
}
if (groupName != NULL)
{
dwRtnCode = GetSidFromAcctNameW(groupName, &pNewGroupSid);
if (dwRtnCode != ERROR_SUCCESS)
{
ReportErrorCode(L"GetSidFromAcctName", dwRtnCode);
fwprintf(stderr, L"Invalid group name: %s\n", groupName);
goto ChownEnd;
}
}
if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
{
fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
goto ChownEnd;
}
dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
if (dwRtnCode != ERROR_SUCCESS)
{
ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode);
goto ChownEnd;
}
dwRtnCode = ChownImpl(userName, groupName, pathName);
if (dwRtnCode) {
goto ChownEnd;
}
ret = EXIT_SUCCESS;
ChownEnd:
LocalFree(userName);
LocalFree(groupName);
LocalFree(pNewOwnerSid);
LocalFree(pNewGroupSid);
return ret;
}

498
hadoop-common-project/hadoop-common/src/main/winutils/client.c Normal file
View File

@ -0,0 +1,498 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
#include "winutils.h"
#include <Winsvc.h>
#include <errno.h>
#include "hadoopwinutilsvc_h.h"
#pragma comment(lib, "Rpcrt4.lib")
#pragma comment(lib, "advapi32.lib")
static ACCESS_MASK CLIENT_MASK = 1;
VOID ReportClientError(LPWSTR lpszLocation, DWORD dwError) {
LPWSTR debugMsg = NULL;
int len;
WCHAR hexError[32];
HRESULT hr;
if (IsDebuggerPresent()) {
len = FormatMessageW(
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
NULL, dwError,
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPWSTR)&debugMsg, 0, NULL);
LogDebugMessage(L"%s: %s: %x: %.*s\n", GetSystemTimeString(), lpszLocation, dwError, len, debugMsg);
}
if (NULL != debugMsg) LocalFree(debugMsg);
}
DWORD PrepareRpcBindingHandle(
__out RPC_BINDING_HANDLE* pHadoopWinutilsSvcBinding) {
DWORD dwError = EXIT_FAILURE;
RPC_STATUS status;
LPWSTR lpszStringBinding = NULL;
ULONG ulCode;
RPC_SECURITY_QOS_V3 qos;
SID_IDENTIFIER_AUTHORITY authNT = SECURITY_NT_AUTHORITY;
BOOL rpcBindingInit = FALSE;
PSID pLocalSystemSid = NULL;
DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE;
pLocalSystemSid = (PSID) LocalAlloc(LPTR, cbSystemSidSize);
if (!pLocalSystemSid) {
dwError = GetLastError();
ReportClientError(L"LocalAlloc", dwError);
goto done;
}
if (!CreateWellKnownSid(WinLocalSystemSid, NULL, pLocalSystemSid, &cbSystemSidSize)) {
dwError = GetLastError();
ReportClientError(L"CreateWellKnownSid", dwError);
goto done;
}
ZeroMemory(&qos, sizeof(qos));
qos.Version = RPC_C_SECURITY_QOS_VERSION_3;
qos.Capabilities = RPC_C_QOS_CAPABILITIES_LOCAL_MA_HINT | RPC_C_QOS_CAPABILITIES_MUTUAL_AUTH;
qos.IdentityTracking = RPC_C_QOS_IDENTITY_DYNAMIC;
qos.ImpersonationType = RPC_C_IMP_LEVEL_DEFAULT;
qos.Sid = pLocalSystemSid;
status = RpcStringBindingCompose(NULL,
SVCBINDING,
NULL,
SVCNAME,
NULL,
&lpszStringBinding);
if (RPC_S_OK != status) {
ReportClientError(L"RpcStringBindingCompose", status);
dwError = status;
goto done;
}
status = RpcBindingFromStringBinding(lpszStringBinding, pHadoopWinutilsSvcBinding);
if (RPC_S_OK != status) {
ReportClientError(L"RpcBindingFromStringBinding", status);
dwError = status;
goto done;
}
rpcBindingInit = TRUE;
status = RpcBindingSetAuthInfoEx(
*pHadoopWinutilsSvcBinding,
NULL,
RPC_C_AUTHN_LEVEL_PKT_PRIVACY, // AuthnLevel
RPC_C_AUTHN_WINNT, // AuthnSvc
NULL, // AuthnIdentity (self)
RPC_C_AUTHZ_NONE, // AuthzSvc
&qos);
if (RPC_S_OK != status) {
ReportClientError(L"RpcBindingSetAuthInfoEx", status);
dwError = status;
goto done;
}
dwError = ERROR_SUCCESS;
done:
if (dwError && rpcBindingInit) RpcBindingFree(pHadoopWinutilsSvcBinding);
if (pLocalSystemSid) LocalFree(pLocalSystemSid);
if (NULL != lpszStringBinding) {
status = RpcStringFree(&lpszStringBinding);
if (RPC_S_OK != status) {
ReportClientError(L"RpcStringFree", status);
}
}
return dwError;
}
DWORD RpcCall_WinutilsKillTask(
__in LPCWSTR taskName) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
KILLTASK_REQUEST request;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.taskName = taskName;
RpcTryExcept {
dwError = WinutilsKillTask(hHadoopWinutilsSvcBinding, &request);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
LogDebugMessage(L"RpcCall_WinutilsKillTask: %s :%d\n", taskName, dwError);
return dwError;
}
DWORD RpcCall_WinutilsMkDir(
__in LPCWSTR filePath) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
MKDIR_REQUEST request;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.filePath = filePath;
RpcTryExcept {
dwError = WinutilsMkDir(hHadoopWinutilsSvcBinding, &request);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
LogDebugMessage(L"RpcCall_WinutilsMkDir: %s :%d\n", filePath, dwError);
return dwError;
}
DWORD RpcCall_WinutilsChown(
__in LPCWSTR filePath,
__in_opt LPCWSTR ownerName,
__in_opt LPCWSTR groupName) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
CHOWN_REQUEST request;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.filePath = filePath;
request.ownerName = ownerName;
request.groupName = groupName;
RpcTryExcept {
dwError = WinutilsChown(hHadoopWinutilsSvcBinding, &request);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
LogDebugMessage(L"RpcCall_WinutilsChown: %s %s %s :%d\n",
ownerName, groupName, filePath, dwError);
return dwError;
}
DWORD RpcCall_WinutilsChmod(
__in LPCWSTR filePath,
__in int mode) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
CHMOD_REQUEST request;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.filePath = filePath;
request.mode = mode;
RpcTryExcept {
dwError = WinutilsChmod(hHadoopWinutilsSvcBinding, &request);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
LogDebugMessage(L"RpcCall_WinutilsChmod: %s %o :%d\n",
filePath, mode, dwError);
return dwError;
}
DWORD RpcCall_WinutilsMoveFile(
__in int operation,
__in LPCWSTR sourcePath,
__in LPCWSTR destinationPath,
__in BOOL replaceExisting) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
MOVEFILE_REQUEST request;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.operation = operation;
request.sourcePath = sourcePath;
request.destinationPath = destinationPath;
request.replaceExisting = replaceExisting;
RpcTryExcept {
dwError = WinutilsMoveFile(hHadoopWinutilsSvcBinding, &request);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
LogDebugMessage(L"RpcCall_WinutilsMoveFile: %s %s %d :%d\n",
sourcePath, destinationPath, replaceExisting, dwError);
return dwError;
}
DWORD RpcCall_WinutilsCreateFile(
__in LPCWSTR path,
__in DWORD desiredAccess,
__in DWORD shareMode,
__in DWORD creationDisposition,
__in DWORD flags,
__out HANDLE* hFile) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
DWORD dwSelfPid = GetCurrentProcessId();
CREATEFILE_REQUEST request;
CREATEFILE_RESPONSE *response = NULL;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.path = path;
request.desiredAccess = desiredAccess;
request.shareMode = shareMode;
request.creationDisposition = creationDisposition;
request.flags = flags;
RpcTryExcept {
dwError = WinutilsCreateFile(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
if (ERROR_SUCCESS == dwError) {
*hFile = response->hFile;
}
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
if(NULL != response) MIDL_user_free(response);
LogDebugMessage(L"RpcCall_WinutilsCreateFile: %s %d, %d, %d, %d :%d\n",
path, desiredAccess, shareMode, creationDisposition, flags, dwError);
return dwError;
}
DWORD RpcCall_WinutilsDeletePath(
__in LPCWSTR path,
__in BOOL isDir,
__out BOOL* pDeleted) {
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
DELETEPATH_REQUEST request;
DELETEPATH_RESPONSE *response = NULL;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
pDeleted = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.path = path;
request.type = isDir ? PATH_IS_DIR : PATH_IS_FILE;
RpcTryExcept {
dwError = WinutilsDeletePath(hHadoopWinutilsSvcBinding, &request, &response);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
if (ERROR_SUCCESS == dwError) {
*pDeleted = response->deleted;
}
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
if(NULL != response) MIDL_user_free(response);
LogDebugMessage(L"RpcCall_WinutilsDeletePath: %s %d: %d %d\n",
path, isDir, *pDeleted, dwError);
return dwError;
}
DWORD RpcCall_TaskCreateAsUser(
LPCWSTR cwd, LPCWSTR jobName,
LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr)
{
DWORD dwError = EXIT_FAILURE;
ULONG ulCode;
DWORD dwSelfPid = GetCurrentProcessId();
CREATE_PROCESS_REQUEST request;
CREATE_PROCESS_RESPONSE *response = NULL;
RPC_BINDING_HANDLE hHadoopWinutilsSvcBinding;
BOOL rpcBindingInit = FALSE;
dwError = PrepareRpcBindingHandle(&hHadoopWinutilsSvcBinding);
if (dwError) {
ReportClientError(L"PrepareRpcBindingHandle", dwError);
goto done;
}
rpcBindingInit = TRUE;
ZeroMemory(&request, sizeof(request));
request.cwd = cwd;
request.jobName = jobName;
request.user = user;
request.pidFile = pidFile;
request.cmdLine = cmdLine;
RpcTryExcept {
dwError = WinutilsCreateProcessAsUser(hHadoopWinutilsSvcBinding, dwSelfPid, &request, &response);
}
RpcExcept(1) {
ulCode = RpcExceptionCode();
ReportClientError(L"RpcExcept", ulCode);
dwError = (DWORD) ulCode;
}
RpcEndExcept;
if (ERROR_SUCCESS == dwError) {
*phProcess = response->hProcess;
*phThread = response->hThread;
*phStdIn = response->hStdIn;
*phStdOut = response->hStdOut;
*phStdErr = response->hStdErr;
}
done:
if (rpcBindingInit) RpcBindingFree(&hHadoopWinutilsSvcBinding);
if (NULL != response) {
MIDL_user_free(response);
}
return dwError;
}

174
hadoop-common-project/hadoop-common/src/main/winutils/config.cpp Normal file
View File

@ -0,0 +1,174 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
#include "winutils.h"
#include <string.h>
#include <stdlib.h>
#import "msxml6.dll"
#define ERROR_CHECK_HRESULT_DONE(hr, message) \
if (FAILED(hr)) { \
dwError = (DWORD) hr; \
LogDebugMessage(L"%s: %x", message, hr); \
goto done; \
}
DWORD BuildPathRelativeToModule(
__in LPCWSTR relativePath,
__in size_t len,
__out_ecount(len) LPWSTR buffer) {
DWORD dwError = ERROR_SUCCESS;
WCHAR moduleFile[MAX_PATH];
WCHAR modulePath[_MAX_DIR];
WCHAR moduleDrive[_MAX_DRIVE];
DWORD size;
HRESULT hr = S_OK;
errno_t errno;
size = GetModuleFileName(NULL, moduleFile, MAX_PATH);
dwError = GetLastError(); // Always check due to ERROR_INSUFFICIENT_BUFFER
if (dwError) {
LogDebugMessage(L"GetModuleFileName: %x\n", dwError);
goto done;
}
errno = _wsplitpath_s(moduleFile,
moduleDrive, _MAX_DRIVE,
modulePath, _MAX_DIR,
NULL, 0, // fname, not interesting
NULL, 0); // extenssion, not interesting
if (errno) {
LogDebugMessage(L"_wsplitpath_s: %x\n", errno);
dwError = ERROR_BAD_PATHNAME;
goto done;
}
hr = StringCbPrintf(buffer, len, L"%s%s%s", moduleDrive, modulePath, relativePath);
if (FAILED(hr)) {
// There is no reliable HRESULT to WIN32 mapping, use code.
// see http://blogs.msdn.com/b/oldnewthing/archive/2006/11/03/942851.aspx
//
dwError = HRESULT_CODE(hr);
goto done;
}
LogDebugMessage(L"BuildPathRelativeToModule: %s (%s)\n", buffer, relativePath);
done:
return dwError;
}
DWORD GetConfigValue(
__in LPCWSTR relativePath,
__in LPCWSTR keyName,
__out size_t* len, __out_ecount(len) LPCWSTR* value) {
DWORD dwError = ERROR_SUCCESS;
WCHAR xmlPath[MAX_PATH];
*len = 0;
*value = NULL;
dwError = BuildPathRelativeToModule(
relativePath,
sizeof(xmlPath)/sizeof(WCHAR),
xmlPath);
if (dwError) {
goto done;
}
dwError = GetConfigValueFromXmlFile(xmlPath, keyName, len, value);
done:
if (*len) {
LogDebugMessage(L"GetConfigValue:%d key:%s len:%d value:%.*s from:%s\n", dwError, keyName, *len, *len, *value, xmlPath);
}
return dwError;
}
DWORD GetConfigValueFromXmlFile(__in LPCWSTR xmlFile, __in LPCWSTR keyName,
__out size_t* outLen, __out_ecount(len) LPCWSTR* outValue) {
DWORD dwError = ERROR_SUCCESS;
HRESULT hr;
WCHAR keyXsl[8192];
size_t len = 0;
LPWSTR value = NULL;
BOOL comInitialized = FALSE;
*outLen = 0;
*outValue = NULL;
hr = CoInitialize(NULL);
ERROR_CHECK_HRESULT_DONE(hr, L"CoInitialize");
comInitialized = TRUE;
hr = StringCbPrintf(keyXsl, sizeof(keyXsl), L"//configuration/property[name='%s']/value/text()", keyName);
ERROR_CHECK_HRESULT_DONE(hr, L"StringCbPrintf");
try {
MSXML2::IXMLDOMDocument2Ptr pDoc;
hr = pDoc.CreateInstance(__uuidof(MSXML2::DOMDocument60), NULL, CLSCTX_INPROC_SERVER);
ERROR_CHECK_HRESULT_DONE(hr, L"CreateInstance");
pDoc->async = VARIANT_FALSE;
pDoc->validateOnParse = VARIANT_FALSE;
pDoc->resolveExternals = VARIANT_FALSE;
_variant_t file(xmlFile);
if (VARIANT_FALSE == pDoc->load(file)) {
dwError = pDoc->parseError->errorCode;
LogDebugMessage(L"load %s failed:%d %s\n", xmlFile, dwError,
static_cast<LPCWSTR>(pDoc->parseError->Getreason()));
goto done;
}
MSXML2::IXMLDOMElementPtr pRoot = pDoc->documentElement;
MSXML2::IXMLDOMNodePtr keyNode = pRoot->selectSingleNode(keyXsl);
if (keyNode) {
_bstr_t bstrValue = static_cast<_bstr_t>(keyNode->nodeValue);
len = bstrValue.length();
value = (LPWSTR) LocalAlloc(LPTR, (len+1) * sizeof(WCHAR));
LPCWSTR lpwszValue = static_cast<LPCWSTR>(bstrValue);
memcpy(value, lpwszValue, (len) * sizeof(WCHAR));
LogDebugMessage(L"key:%s :%.*s [%s]\n", keyName, len, value, lpwszValue);
*outLen = len;
*outValue = value;
}
else {
LogDebugMessage(L"node Xpath:%s not found in:%s\n", keyXsl, xmlFile);
}
}
catch(_com_error errorObject) {
dwError = errorObject.Error();
LogDebugMessage(L"catch _com_error:%x %s\n", dwError, errorObject.ErrorMessage());
goto done;
}
done:
if (comInitialized) {
CoUninitialize();
}
return dwError;
}

130
hadoop-common-project/hadoop-common/src/main/winutils/hadoopwinutilsvc.idl Normal file
View File

@ -0,0 +1,130 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import "oaidl.idl";
import "ocidl.idl";
[
uuid(0492311C-1718-4F53-A6EB-86AD7039988D),
version(1.0),
pointer_default(unique),
//implicit_handle(handle_t hHadoopWinutilsSvcBinding),
endpoint("ncalrpc:[hadoopwinutilsvc]"),
#ifndef __midl
explicit_handle
#endif
]
interface HadoopWinutilSvc
{
typedef struct {
[string] const wchar_t* cwd;
[string] const wchar_t* jobName;
[string] const wchar_t* user;
[string] const wchar_t* pidFile;
[string] const wchar_t* cmdLine;
} CREATE_PROCESS_REQUEST;
typedef struct {
LONG_PTR hProcess;
LONG_PTR hThread;
LONG_PTR hStdIn;
LONG_PTR hStdOut;
LONG_PTR hStdErr;
} CREATE_PROCESS_RESPONSE;
typedef struct {
[string] const wchar_t* filePath;
[string] const wchar_t* ownerName;
[string] const wchar_t* groupName;
} CHOWN_REQUEST;
typedef struct {
[string] const wchar_t* filePath;
int mode;
} CHMOD_REQUEST;
typedef struct {
[string] const wchar_t* filePath;
} MKDIR_REQUEST;
typedef enum { MOVE_FILE = 1, COPY_FILE = 2} MOVE_COPY_OPERATION;
typedef struct {
MOVE_COPY_OPERATION operation;
[string] const wchar_t* sourcePath;
[string] const wchar_t* destinationPath;
boolean replaceExisting;
} MOVEFILE_REQUEST;
typedef struct {
[string] const wchar_t* path;
int desiredAccess;
int shareMode;
int creationDisposition;
int flags;
} CREATEFILE_REQUEST;
typedef struct {
LONG_PTR hFile;
} CREATEFILE_RESPONSE;
typedef enum {PATH_IS_DIR = 1, PATH_IS_FILE = 2} DELETEPATH_TYPE;
typedef struct {
DELETEPATH_TYPE type;
[string] const wchar_t* path;
} DELETEPATH_REQUEST;
typedef struct {
boolean deleted;
} DELETEPATH_RESPONSE;
typedef struct {
[string] const wchar_t* taskName;
} KILLTASK_REQUEST;
error_status_t WinutilsKillTask(
[in] KILLTASK_REQUEST *request);
error_status_t WinutilsMkDir(
[in] MKDIR_REQUEST *request);
error_status_t WinutilsMoveFile(
[in] MOVEFILE_REQUEST *request);
error_status_t WinutilsChown(
[in] CHOWN_REQUEST *request);
error_status_t WinutilsChmod(
[in] CHMOD_REQUEST *request);
error_status_t WinutilsCreateFile(
[in] int nmPid,
[in] CREATEFILE_REQUEST *request,
[out] CREATEFILE_RESPONSE **response);
error_status_t WinutilsDeletePath(
[in] DELETEPATH_REQUEST *request,
[out] DELETEPATH_RESPONSE **response);
error_status_t WinutilsCreateProcessAsUser(
[in] int nmPid,
[in] CREATE_PROCESS_REQUEST *request,
[out] CREATE_PROCESS_RESPONSE **response);
}

114
hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h
View File

@ -30,6 +30,11 @@
#include <ntsecapi.h>
#include <userenv.h>
#ifdef __cplusplus
extern "C" {
#endif
enum EXIT_CODE
{
/* Common success exit code shared among all utilities */
@ -38,6 +43,12 @@ enum EXIT_CODE
FAILURE = EXIT_FAILURE,
/* Failure code indicates the user does not privilege to create symlinks */
SYMLINK_NO_PRIVILEGE = 2,
ERROR_TASK_NOT_ALIVE = 1,
// This exit code for killed processes is compatible with Unix, where a killed
// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
KILLED_PROCESS_EXIT_CODE = 137,
};
@ -101,6 +112,8 @@ void GroupsUsage(LPCWSTR program);
int Hardlink(__in int argc, __in_ecount(argc) wchar_t *argv[]);
void HardlinkUsage();
DWORD KillTask(PCWSTR jobObjName);
int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]);
void TaskUsage();
@ -167,7 +180,7 @@ void UnregisterWithLsa(__in HANDLE lsaHandle);
DWORD LookupKerberosAuthenticationPackageId(__in HANDLE lsaHandle, __out ULONG * packageId);
DWORD CreateLogonForUser(__in HANDLE lsaHandle,
DWORD CreateLogonTokenForUser(__in HANDLE lsaHandle,
__in const char * tokenSourceName,
__in const char * tokenOriginName,
__in ULONG authnPkgId,
@ -178,3 +191,102 @@ DWORD LoadUserProfileForLogon(__in HANDLE logonHandle, __out PROFILEINFO * pi);
DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi);
DWORD EnableImpersonatePrivileges();
DWORD RunService(__in int argc, __in_ecount(argc) wchar_t *argv[]);
void ServiceUsage();
DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
__in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid);
DWORD ChownImpl(
__in_opt LPCWSTR userName,
__in_opt LPCWSTR groupName,
__in LPCWSTR pathName);
LPCWSTR GetSystemTimeString();
VOID LogDebugMessage(LPCWSTR format, ...);
DWORD SplitStringIgnoreSpaceW(
__in size_t len,
__in_ecount(len) LPCWSTR source,
__in WCHAR deli,
__out size_t* count, __out_ecount(count) WCHAR*** out);
DWORD BuildPathRelativeToModule(
__in LPCWSTR relativePath,
__in size_t len,
__out_ecount(len) LPWSTR buffer);
DWORD GetConfigValue(
__in LPCWSTR relativePath,
__in LPCWSTR keyName,
__out size_t* len,
__out_ecount(len) LPCWSTR* value);
DWORD GetConfigValueFromXmlFile(
__in LPCWSTR xmlFile,
__in LPCWSTR keyName,
__out size_t* len,
__out_ecount(len) LPCWSTR* value);
DWORD BuildServiceSecurityDescriptor(
__in ACCESS_MASK accessMask,
__in size_t grantSidCount,
__in_ecount(grantSidCount) PSID* pGrantSids,
__in size_t denySidCount,
__in_ecount(denySidCount) PSID* pDenySids,
__in_opt PSID pOwner,
__out PSECURITY_DESCRIPTOR* pSD);
DWORD AddNodeManagerAndUserACEsToObject(
__in HANDLE hProcess,
__in LPWSTR user);
DWORD GetSecureJobObjectName(
__in LPCWSTR jobName,
__in size_t cchSecureJobName,
__out_ecount(cchSecureJobName) LPWSTR secureJobName);
extern const WCHAR* wsceConfigRelativePath;
extern LPCWSTR NM_WSCE_ALLOWED;
#define SVCNAME TEXT("hadoopwinutilsvc")
#define SVCBINDING TEXT("ncalrpc")
DWORD RpcCall_WinutilsKillTask(
__in LPCWSTR taskName);
DWORD RpcCall_TaskCreateAsUser(
LPCWSTR cwd, LPCWSTR jobName,
LPCWSTR user, LPCWSTR pidFile, LPCWSTR cmdLine,
HANDLE* phProcess, HANDLE* phThread, HANDLE* phStdIn, HANDLE* phStdOut, HANDLE* phStdErr);
DWORD RpcCall_WinutilsCreateFile(
__in LPCWSTR path,
__in DWORD desiredAccess,
__in DWORD shareMode,
__in DWORD creationDisposition,
__in DWORD flags,
__out HANDLE* hFile);
DWORD RpcCall_WinutilsMoveFile(
__in LPCWSTR sourcePath,
__in LPCWSTR destinationPath,
__in BOOL replaceExisting);
DWORD RpcCall_WinutilsDeletePath(
__in LPCWSTR path,
__in BOOL isDir,
__out BOOL* pDeleted);
#ifdef __cplusplus
}
#endif

748
hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c
View File

@ -19,9 +19,24 @@
#pragma comment(lib, "netapi32.lib")
#pragma comment(lib, "Secur32.lib")
#pragma comment(lib, "Userenv.lib")
#pragma comment(lib, "Ntdsapi.lib")
#include "winutils.h"
#include <ctype.h>
#include <Winsvc.h>
#include <authz.h>
#include <sddl.h>
#include <Ntdsapi.h>
#include <malloc.h>
#define WIDEN_STRING(x) WIDEN_STRING_(x)
#define WIDEN_STRING_(x) L ## x
#define STRINGIFY(x) STRINGIFY_(x)
#define STRINGIFY_(x) #x
#pragma message("WSCE config is " STRINGIFY(WSCE_CONFIG_DIR) "\\" STRINGIFY(WSCE_CONFIG_FILE))
const WCHAR* wsceConfigRelativePath = WIDEN_STRING(STRINGIFY(WSCE_CONFIG_DIR)) L"\\" WIDEN_STRING(STRINGIFY(WSCE_CONFIG_FILE));
/*
* The array of 12 months' three-letter abbreviations
@ -237,10 +252,10 @@ ConvertToLongPathExit:
// Function: IsDirFileInfo
//
// Description:
// Test if the given file information is a directory
// Test if the given file information is a directory
//
// Returns:
// TRUE if it is a directory
// TRUE if it is a directory
// FALSE otherwise
//
// Notes:
@ -257,10 +272,10 @@ BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation)
// Function: CheckFileAttributes
//
// Description:
// Check if the given file has all the given attribute(s)
// Check if the given file has all the given attribute(s)
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -281,10 +296,10 @@ static DWORD FileAttributesCheck(
// Function: IsDirectory
//
// Description:
// Check if the given file is a directory
// Check if the given file is a directory
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -298,10 +313,10 @@ DWORD DirectoryCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsReparsePoint
//
// Description:
// Check if the given file is a reparse point
// Check if the given file is a reparse point
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -315,10 +330,10 @@ static DWORD ReparsePointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: CheckReparseTag
//
// Description:
// Check if the given file is a reparse point of the given tag.
// Check if the given file is a reparse point of the given tag.
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -356,10 +371,10 @@ static DWORD ReparseTagCheck(__in LPCWSTR path, __in DWORD tag, __out PBOOL res)
// Function: IsSymbolicLink
//
// Description:
// Check if the given file is a symbolic link.
// Check if the given file is a symbolic link.
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -373,10 +388,10 @@ DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: IsJunctionPoint
//
// Description:
// Check if the given file is a junction point.
// Check if the given file is a junction point.
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// error code otherwise
//
// Notes:
@ -390,14 +405,14 @@ DWORD JunctionPointCheck(__in LPCWSTR pathName, __out PBOOL res)
// Function: GetSidFromAcctNameW
//
// Description:
// To retrieve the SID for a user account
// To retrieve the SID for a user account
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
// Caller needs to destroy the memory of Sid by calling LocalFree()
// Caller needs to destroy the memory of Sid by calling LocalFree()
//
DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
{
@ -479,10 +494,10 @@ DWORD GetSidFromAcctNameW(__in PCWSTR acctName, __out PSID *ppSid)
// Function: GetUnixAccessMask
//
// Description:
// Compute the 3 bit Unix mask for the owner, group, or, others
// Compute the 3 bit Unix mask for the owner, group, or, others
//
// Returns:
// The 3 bit Unix mask in INT
// The 3 bit Unix mask in INT
//
// Notes:
//
@ -506,10 +521,10 @@ static INT GetUnixAccessMask(ACCESS_MASK Mask)
// Function: GetAccess
//
// Description:
// Get Windows acces mask by AuthZ methods
// Get Windows acces mask by AuthZ methods
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
//
// Notes:
//
@ -554,10 +569,10 @@ static DWORD GetAccess(AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClient,
// Function: GetEffectiveRightsForSid
//
// Description:
// Get Windows acces mask by AuthZ methods
// Get Windows acces mask by AuthZ methods
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
//
// Notes:
// We run into problems for local user accounts when using the method
@ -714,11 +729,11 @@ CheckAccessEnd:
// Function: FindFileOwnerAndPermissionByHandle
//
// Description:
// Find the owner, primary group and permissions of a file object given the
// Find the owner, primary group and permissions of a file object given the
// the file object handle. The function will always follow symbolic links.
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@ -778,10 +793,10 @@ FindFileOwnerAndPermissionByHandleEnd:
// Function: FindFileOwnerAndPermission
//
// Description:
// Find the owner, primary group and permissions of a file object
// Find the owner, primary group and permissions of a file object
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
// Error code otherwise
//
// Notes:
@ -1207,14 +1222,14 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
if (winUserAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
NO_PROPAGATE_INHERIT_ACE,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessDenyMask, pOwnerSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
NO_PROPAGATE_INHERIT_ACE,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winUserAccessAllowMask, pOwnerSid))
{
ret = GetLastError();
@ -1222,21 +1237,21 @@ static DWORD GetWindowsDACLs(__in INT unixMask,
}
if (winGroupAccessDenyMask &&
!AddAccessDeniedAceEx(pNewDACL, ACL_REVISION,
NO_PROPAGATE_INHERIT_ACE,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessDenyMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
NO_PROPAGATE_INHERIT_ACE,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winGroupAccessAllowMask, pGroupSid))
{
ret = GetLastError();
goto GetWindowsDACLsEnd;
}
if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION,
NO_PROPAGATE_INHERIT_ACE,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
winOtherAccessAllowMask, pEveryoneSid))
{
ret = GetLastError();
@ -1440,14 +1455,14 @@ ChangeFileModeByMaskEnd:
// Function: GetAccntNameFromSid
//
// Description:
// To retrieve an account name given the SID
// To retrieve an account name given the SID
//
// Returns:
// ERROR_SUCCESS: on success
// ERROR_SUCCESS: on success
// Other error code: otherwise
//
// Notes:
// Caller needs to destroy the memory of account name by calling LocalFree()
// Caller needs to destroy the memory of account name by calling LocalFree()
//
DWORD GetAccntNameFromSid(__in PSID pSid, __out PWSTR *ppAcctName)
{
@ -1536,10 +1551,10 @@ GetAccntNameFromSidEnd:
// Function: GetLocalGroupsForUser
//
// Description:
// Get an array of groups for the given user.
// Get an array of groups for the given user.
//
// Returns:
// ERROR_SUCCESS on success
// ERROR_SUCCESS on success
// Other error code on failure
//
// Notes:
@ -1631,11 +1646,12 @@ GetLocalGroupsForUserEnd:
return ret;
}
//----------------------------------------------------------------------------
// Function: EnablePrivilege
//
// Description:
// Check if the process has the given privilege. If yes, enable the privilege
// Check if the process has the given privilege. If yes, enable the privilege
// to the process's access token.
//
// Returns:
@ -1672,8 +1688,8 @@ DWORD EnablePrivilege(__in LPCWSTR privilegeName)
// As stated on MSDN, we need to use GetLastError() to check if
// AdjustTokenPrivileges() adjusted all of the specified privileges.
//
if( !AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL) ) {
dwErrCode = GetLastError();
if (!AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL)) {
dwErrCode = GetLastError();
}
CloseHandle(hToken);
@ -1706,12 +1722,15 @@ void ReportErrorCode(LPCWSTR func, DWORD err)
(LPWSTR)&msg, 0, NULL);
if (len > 0)
{
LogDebugMessage(L"%s error (%d): %s\n", func, err, msg);
fwprintf(stderr, L"%s error (%d): %s\n", func, err, msg);
}
else
{
LogDebugMessage(L"%s error code: %d.\n", func, err);
fwprintf(stderr, L"%s error code: %d.\n", func, err);
}
if (msg != NULL) LocalFree(msg);
}
@ -1843,7 +1862,7 @@ DWORD LookupKerberosAuthenticationPackageId(__in HANDLE lsaHandle, __out ULONG *
}
//----------------------------------------------------------------------------
// Function: CreateLogonForUser
// Function: CreateLogonTokenForUser
//
// Description:
// Contacts the local LSA and performs a logon without credential for the
@ -1858,7 +1877,7 @@ DWORD LookupKerberosAuthenticationPackageId(__in HANDLE lsaHandle, __out ULONG *
// This call assumes that all required privileges have already been enabled (TCB etc).
// IMPORTANT **** tokenOriginName must be immutable!
//
DWORD CreateLogonForUser(__in HANDLE lsaHandle,
DWORD CreateLogonTokenForUser(__in HANDLE lsaHandle,
__in const char * tokenSourceName,
__in const char * tokenOriginName, // must be immutable, will not be copied!
__in ULONG authnPkgId,
@ -2026,6 +2045,8 @@ done:
return loadProfileStatus;
}
DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi)
{
DWORD touchProfileStatus = ERROR_ASSERTION_FAILURE; // Failure to set status should trigger error
@ -2046,3 +2067,646 @@ DWORD UnloadProfileForLogon(__in HANDLE logonHandle, __in PROFILEINFO * pi)
done:
return touchProfileStatus;
}
//----------------------------------------------------------------------------
// Function: ChangeFileOwnerBySid
//
// Description:
// Change a file or directory ownership by giving new owner and group SIDs
//
// Returns:
// ERROR_SUCCESS: on success
// Error code: otherwise
//
// Notes:
// This function is long path safe, i.e. the path will be converted to long
// path format if not already converted. So the caller does not need to do
// the converstion before calling the method.
//
DWORD ChangeFileOwnerBySid(__in LPCWSTR path,
__in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid)
{
LPWSTR longPathName = NULL;
INT oldMode = 0;
SECURITY_INFORMATION securityInformation = 0;
DWORD dwRtnCode = ERROR_SUCCESS;
// Convert the path the the long path
//
dwRtnCode = ConvertToLongPath(path, &longPathName);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// Get a pointer to the existing owner information and DACL
//
dwRtnCode = FindFileOwnerAndPermission(longPathName, FALSE, NULL, NULL, &oldMode);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// We need SeTakeOwnershipPrivilege to set the owner if the caller does not
// have WRITE_OWNER access to the object; we need SeRestorePrivilege if the
// SID is not contained in the caller's token, and have the SE_GROUP_OWNER
// permission enabled.
//
if (EnablePrivilege(L"SeTakeOwnershipPrivilege") != ERROR_SUCCESS)
{
fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n");
}
if (EnablePrivilege(L"SeRestorePrivilege") != ERROR_SUCCESS)
{
fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n");
}
assert(pNewOwnerSid != NULL || pNewGroupSid != NULL);
// Set the owners of the file.
//
if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION;
if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION;
dwRtnCode = SetNamedSecurityInfoW(
longPathName,
SE_FILE_OBJECT,
securityInformation,
pNewOwnerSid,
pNewGroupSid,
NULL,
NULL);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
// Set the permission on the file for the new owner.
//
dwRtnCode = ChangeFileModeByMask(longPathName, oldMode);
if (dwRtnCode != ERROR_SUCCESS)
{
goto ChangeFileOwnerByNameEnd;
}
ChangeFileOwnerByNameEnd:
LocalFree(longPathName);
return dwRtnCode;
}
//-----------------------------------------------------------------------------
// Function: GetSecureJobObjectName
//
// Description:
// Creates a job object name usable in a secure environment: adds the Golbal\
//
DWORD GetSecureJobObjectName(
__in LPCWSTR jobName,
__in size_t cchSecureJobName,
__out_ecount(cchSecureJobName) LPWSTR secureJobName) {
HRESULT hr = StringCchPrintf(secureJobName, cchSecureJobName,
L"Global\\%s", jobName);
if (FAILED(hr)) {
return HRESULT_CODE(hr);
}
return ERROR_SUCCESS;
}
//-----------------------------------------------------------------------------
// Function: EnableImpersonatePrivileges
//
// Description:
// Enables the required privileges for S4U impersonation
//
// Returns:
// ERROR_SUCCESS: On success
//
DWORD EnableImpersonatePrivileges() {
DWORD dwError = ERROR_SUCCESS;
LPCWSTR privilege = NULL;
int crt = 0;
LPCWSTR privileges[] = {
SE_IMPERSONATE_NAME,
SE_TCB_NAME,
SE_ASSIGNPRIMARYTOKEN_NAME,
SE_INCREASE_QUOTA_NAME,
SE_RESTORE_NAME,
SE_DEBUG_NAME,
SE_SECURITY_NAME,
};
for (crt = 0; crt < sizeof(privileges)/sizeof(LPCWSTR); ++crt) {
LPCWSTR privilege = privileges[crt];
dwError = EnablePrivilege(privilege);
if( dwError != ERROR_SUCCESS ) {
LogDebugMessage(L"Failed to enable privilege: %s\n", privilege);
ReportErrorCode(L"EnablePrivilege", dwError);
goto done;
}
}
done:
return dwError;
}
//-----------------------------------------------------------------------------
// Function: KillTask
//
// Description:
// Kills a task via a jobobject. Outputs the
// appropriate information to stdout on success, or stderr on failure.
//
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
DWORD KillTask(PCWSTR jobObjName)
{
DWORD dwError = ERROR_SUCCESS;
HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName);
if(jobObject == NULL)
{
dwError = GetLastError();
if(dwError == ERROR_FILE_NOT_FOUND)
{
// job object does not exist. assume its not alive
dwError = ERROR_SUCCESS;
}
goto done;
}
if(TerminateJobObject(jobObject, KILLED_PROCESS_EXIT_CODE) == 0)
{
dwError = GetLastError();
}
done:
CloseHandle(jobObject);
return dwError;
}
DWORD ChownImpl(
__in_opt LPCWSTR userName,
__in_opt LPCWSTR groupName,
__in LPCWSTR pathName) {
DWORD dwError;
PSID pNewOwnerSid = NULL;
PSID pNewGroupSid = NULL;
if (userName != NULL)
{
dwError = GetSidFromAcctNameW(userName, &pNewOwnerSid);
if (dwError != ERROR_SUCCESS)
{
ReportErrorCode(L"GetSidFromAcctName", dwError);
fwprintf(stderr, L"Invalid user name: %s\n", userName);
goto done;
}
}
if (groupName != NULL)
{
dwError = GetSidFromAcctNameW(groupName, &pNewGroupSid);
if (dwError != ERROR_SUCCESS)
{
ReportErrorCode(L"GetSidFromAcctName", dwError);
fwprintf(stderr, L"Invalid group name: %s\n", groupName);
goto done;
}
}
if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0)
{
fwprintf(stderr, L"Incorrect file name format: %s\n", pathName);
goto done;
}
dwError = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid);
if (dwError != ERROR_SUCCESS)
{
ReportErrorCode(L"ChangeFileOwnerBySid", dwError);
goto done;
}
done:
LocalFree(pNewOwnerSid);
LocalFree(pNewGroupSid);
return dwError;
}
LPCWSTR GetSystemTimeString() {
__declspec(thread) static WCHAR buffer[1024];
DWORD dwError;
FILETIME ftime;
SYSTEMTIME systime;
LARGE_INTEGER counter, frequency;
int subSec;
double qpc;
HRESULT hr;
buffer[0] = L'\0';
// GetSystemTimePreciseAsFileTime is only available in Win8+ and our libs do not link against it
GetSystemTimeAsFileTime(&ftime);
if (!FileTimeToSystemTime(&ftime, &systime)) {
dwError = GetLastError();
LogDebugMessage(L"FileTimeToSystemTime error:%d\n", dwError);
goto done;
}
// Get the ms from QPC. GetSystemTimeAdjustment is ignored...
QueryPerformanceCounter(&counter);
QueryPerformanceFrequency(&frequency);
qpc = (double) counter.QuadPart / (double) frequency.QuadPart;
subSec = ((qpc - (long)qpc) * 1000000);
hr = StringCbPrintf(buffer, sizeof(buffer), L"%02d:%02d:%02d.%06d",
(int)systime.wHour, (int)systime.wMinute, (int)systime.wSecond, (int)subSec);
if (FAILED(hr)) {
LogDebugMessage(L"StringCbPrintf error:%d\n", hr);
}
done:
return buffer;
}
//----------------------------------------------------------------------------
// Function: LogDebugMessage
//
// Description:
// Sends a message to the debugger console, if one is attached
//
// Notes:
// Native debugger: windbg, ntsd, cdb, visual studio
//
VOID LogDebugMessage(LPCWSTR format, ...) {
LPWSTR buffer[8192];
va_list args;
HRESULT hr;
if (!IsDebuggerPresent()) return;
va_start(args, format);
hr = StringCbVPrintf(buffer, sizeof(buffer), format, args);
if (SUCCEEDED(hr)) {
OutputDebugString(buffer);
}
va_end(args);
}
//----------------------------------------------------------------------------
// Function: SplitStringIgnoreSpaceW
//
// Description:
// splits a null-terminated string based on a delimiter
//
// Returns:
// ERROR_SUCCESS: on success
// error code: otherwise
//
// Notes:
// The tokes are also null-terminated
// Caller should use LocalFree to clear outTokens
//
DWORD SplitStringIgnoreSpaceW(
__in size_t len,
__in_ecount(len) LPCWSTR source,
__in WCHAR deli,
__out size_t* count,
__out_ecount(count) WCHAR*** outTokens) {
size_t tokenCount = 0;
size_t crtSource;
size_t crtToken = 0;
WCHAR* lpwszTokenStart = NULL;
WCHAR* lpwszTokenEnd = NULL;
WCHAR* lpwszBuffer = NULL;
size_t tokenLength = 0;
size_t cchBufferLength = 0;
WCHAR crt;
WCHAR** tokens = NULL;
enum {BLANK, TOKEN, DELIMITER} State = BLANK;
for(crtSource = 0; crtSource < len; ++crtSource) {
crt = source[crtSource];
switch(State) {
case BLANK: // intentional fallthrough
case DELIMITER:
if (crt == deli) {
State = DELIMITER;
}
else if (!iswspace(crt)) {
++tokenCount;
lpwszTokenEnd = lpwszTokenStart = source + crtSource;
State = TOKEN;
}
else {
State = BLANK;
}
break;
case TOKEN:
if (crt == deli) {
State = DELIMITER;
cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2;
}
else if (!iswspace(crt)) {
lpwszTokenEnd = source + crtSource;
}
break;
}
}
if (State == TOKEN) {
cchBufferLength += lpwszTokenEnd - lpwszTokenStart + 2;
}
LogDebugMessage(L"counted %d [buffer:%d] tokens in %s\n", tokenCount, cchBufferLength, source);
#define COPY_CURRENT_TOKEN \
tokenLength = lpwszTokenEnd - lpwszTokenStart + 1; \
tokens[crtToken] = lpwszBuffer; \
memcpy(tokens[crtToken], lpwszTokenStart, tokenLength*sizeof(WCHAR)); \
tokens[crtToken][tokenLength] = L'\0'; \
lpwszBuffer += (tokenLength+1); \
++crtToken;
if (tokenCount) {
// We use one contigous memory for both the pointer arrays and the data copy buffers
// We cannot use in-place references (zero-copy) because the function users
// need null-terminated strings for the tokens
tokens = (WCHAR**) LocalAlloc(LPTR,
sizeof(WCHAR*) * tokenCount + // for the pointers
sizeof(WCHAR) * cchBufferLength); // for the data
// Data will be copied after the array
lpwszBuffer = (WCHAR*)(((BYTE*)tokens) + (sizeof(WCHAR*) * tokenCount));
State = BLANK;
for(crtSource = 0; crtSource < len; ++crtSource) {
crt = source[crtSource];
switch(State) {
case DELIMITER: // intentional fallthrough
case BLANK:
if (crt == deli) {
State = DELIMITER;
}
else if (!iswspace(crt)) {
lpwszTokenEnd = lpwszTokenStart = source + crtSource;
State = TOKEN;
}
else {
State = BLANK;
}
break;
case TOKEN:
if (crt == deli) {
COPY_CURRENT_TOKEN;
State = DELIMITER;
}
else if (!iswspace(crt)) {
lpwszTokenEnd = source + crtSource;
}
break;
}
}
// Copy out last token, if any
if (TOKEN == State) {
COPY_CURRENT_TOKEN;
}
}
*count = tokenCount;
*outTokens = tokens;
return ERROR_SUCCESS;
}
//----------------------------------------------------------------------------
// Function: BuildServiceSecurityDescriptor
//
// Description:
// Builds a security descriptor for an arbitrary object
//
// Returns:
// ERROR_SUCCESS: on success
// error code: otherwise
//
// Notes:
// The SD is a of the self-contained flavor (offsets, not pointers)
// Caller should use LocalFree to clear allocated pSD
//
DWORD BuildServiceSecurityDescriptor(
__in ACCESS_MASK accessMask,
__in size_t grantSidCount,
__in_ecount(grantSidCount) PSID* pGrantSids,
__in size_t denySidCount,
__in_ecount(denySidCount) PSID* pDenySids,
__in_opt PSID pOwner,
__out PSECURITY_DESCRIPTOR* pSD) {
DWORD dwError = ERROR_SUCCESS;
int crt = 0;
int len = 0;
EXPLICIT_ACCESS* eas = NULL;
LPWSTR lpszSD = NULL;
ULONG cchSD = 0;
HANDLE hToken = INVALID_HANDLE_VALUE;
DWORD dwBufferSize = 0;
PTOKEN_USER pTokenUser = NULL;
PTOKEN_PRIMARY_GROUP pTokenGroup = NULL;
PSECURITY_DESCRIPTOR pTempSD = NULL;
ULONG cbSD = 0;
TRUSTEE owner, group;
ZeroMemory(&owner, sizeof(owner));
// We'll need our own SID to add as SD owner
if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &hToken)) {
dwError = GetLastError();
LogDebugMessage(L"OpenProcessToken: %d\n", dwError);
goto done;
}
if (NULL == pOwner) {
if (!GetTokenInformation(hToken, TokenUser, NULL, 0, &dwBufferSize)) {
dwError = GetLastError();
if (ERROR_INSUFFICIENT_BUFFER != dwError) {
LogDebugMessage(L"GetTokenInformation: %d\n", dwError);
goto done;
}
}
pTokenUser = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize);
if (NULL == pTokenUser) {
dwError = GetLastError();
LogDebugMessage(L"LocalAlloc:pTokenUser: %d\n", dwError);
goto done;
}
if (!GetTokenInformation(hToken, TokenUser, pTokenUser, dwBufferSize, &dwBufferSize)) {
dwError = GetLastError();
LogDebugMessage(L"GetTokenInformation: %d\n", dwError);
goto done;
}
if (!IsValidSid(pTokenUser->User.Sid)) {
dwError = ERROR_INVALID_PARAMETER;
LogDebugMessage(L"IsValidSid: %d\n", dwError);
goto done;
}
pOwner = pTokenUser->User.Sid;
}
dwBufferSize = 0;
if (!GetTokenInformation(hToken, TokenPrimaryGroup, NULL, 0, &dwBufferSize)) {
dwError = GetLastError();
if (ERROR_INSUFFICIENT_BUFFER != dwError) {
LogDebugMessage(L"GetTokenInformation: %d\n", dwError);
goto done;
}
}
pTokenGroup = (PTOKEN_USER) LocalAlloc(LPTR, dwBufferSize);
if (NULL == pTokenGroup) {
dwError = GetLastError();
LogDebugMessage(L"LocalAlloc:pTokenGroup: %d\n", dwError);
goto done;
}
if (!GetTokenInformation(hToken, TokenPrimaryGroup, pTokenGroup, dwBufferSize, &dwBufferSize)) {
dwError = GetLastError();
LogDebugMessage(L"GetTokenInformation: %d\n", dwError);
goto done;
}
if (!IsValidSid(pTokenGroup->PrimaryGroup)) {
dwError = ERROR_INVALID_PARAMETER;
LogDebugMessage(L"IsValidSid: %d\n", dwError);
goto done;
}
owner.TrusteeForm = TRUSTEE_IS_SID;
owner.TrusteeType = TRUSTEE_IS_UNKNOWN;
owner.ptstrName = (LPCWSTR) pOwner;
group.TrusteeForm = TRUSTEE_IS_SID;
group.TrusteeType = TRUSTEE_IS_UNKNOWN;
group.ptstrName = (LPCWSTR) pTokenGroup->PrimaryGroup;
eas = (EXPLICIT_ACCESS*) LocalAlloc(LPTR, sizeof(EXPLICIT_ACCESS) * (grantSidCount + denySidCount));
if (NULL == eas) {
dwError = ERROR_OUTOFMEMORY;
LogDebugMessage(L"LocalAlloc: %d\n", dwError);
goto done;
}
// Build the granted list
for (crt = 0; crt < grantSidCount; ++crt) {
eas[crt].grfAccessPermissions = accessMask;
eas[crt].grfAccessMode = GRANT_ACCESS;
eas[crt].grfInheritance = NO_INHERITANCE;
eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
eas[crt].Trustee.ptstrName = (LPCWSTR) pGrantSids[crt];
eas[crt].Trustee.pMultipleTrustee = NULL;
eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
}
// Build the deny list
for (; crt < grantSidCount + denySidCount; ++crt) {
eas[crt].grfAccessPermissions = accessMask;
eas[crt].grfAccessMode = DENY_ACCESS;
eas[crt].grfInheritance = NO_INHERITANCE;
eas[crt].Trustee.TrusteeForm = TRUSTEE_IS_SID;
eas[crt].Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN;
eas[crt].Trustee.ptstrName = (LPCWSTR) pDenySids[crt - grantSidCount];
eas[crt].Trustee.pMultipleTrustee = NULL;
eas[crt].Trustee.MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE;
}
dwError = BuildSecurityDescriptor(
&owner,
&group,
crt,
eas,
0, // cCountOfAuditEntries
NULL, // pListOfAuditEntries
NULL, // pOldSD
&cbSD,
&pTempSD);
if (ERROR_SUCCESS != dwError) {
LogDebugMessage(L"BuildSecurityDescriptor: %d\n", dwError);
goto done;
}
*pSD = pTempSD;
pTempSD = NULL;
if (IsDebuggerPresent()) {
ConvertSecurityDescriptorToStringSecurityDescriptor(*pSD,
SDDL_REVISION_1,
DACL_SECURITY_INFORMATION,
&lpszSD,
&cchSD);
LogDebugMessage(L"pSD: %.*s\n", cchSD, lpszSD);
}
done:
if (eas) LocalFree(eas);
if (pTokenUser) LocalFree(pTokenUser);
if (INVALID_HANDLE_VALUE != hToken) CloseHandle(hToken);
if (lpszSD) LocalFree(lpszSD);
if (pTempSD) LocalFree(pTempSD);
return dwError;
}
//----------------------------------------------------------------------------
// Function: MIDL_user_allocate
//
// Description:
// Hard-coded function name used by RPC midl code for allocations
//
// Notes:
// Must match the de-allocation mechanism used in MIDL_user_free
//
void __RPC_FAR * __RPC_USER MIDL_user_allocate(size_t len)
{
return LocalAlloc(LPTR, len);
}
//----------------------------------------------------------------------------
// Function: MIDL_user_free
//
// Description:
// Hard-coded function name used by RPC midl code for deallocations
//
// NoteS:
// Must match the allocation mechanism used in MIDL_user_allocate
//
void __RPC_USER MIDL_user_free(void __RPC_FAR * ptr)
{
LocalFree(ptr);
}

94
hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj
View File

@ -19,18 +19,10 @@
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
@ -42,22 +34,11 @@
<RootNamespace>winutils</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>StaticLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
@ -67,15 +48,9 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
@ -83,74 +58,35 @@
<PropertyGroup>
<IncludePath>include;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir />
<IntDir>..\..\..\target\winutils\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<OutDir>..\..\..\target\bin\</OutDir>
<IntDir>..\..\..\target\winutils\$(Platform)\$(Configuration)\</IntDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>WIN32;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<!-- <Optimization>MaxSpeed</Optimization> -->
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>WIN32;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -159,12 +95,34 @@
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(IntermediateOutputPath)</AdditionalIncludeDirectories>
</ClCompile>
<Midl>
<ApplicationConfigurationMode>true</ApplicationConfigurationMode>
<TargetEnvironment>X64</TargetEnvironment>
<OutputDirectory>$(IntermediateOutputPath)</OutputDirectory>
<GenerateStublessProxies>true</GenerateStublessProxies>
<ValidateAllParameters>true</ValidateAllParameters>
<WarnAsError>true</WarnAsError>
<WarningLevel>2</WarningLevel>
</Midl>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="client.c" />
<ClCompile Include="$(IntermediateOutputPath)\hadoopwinutilsvc_c.c" />
<ClCompile Include="libwinutils.c" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="config.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="include/winutils.h" />
</ItemGroup>
<ItemGroup>
<Midl Include="hadoopwinutilsvc.idl" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>

23
hadoop-common-project/hadoop-common/src/main/winutils/main.c
View File

@ -17,12 +17,27 @@
#include "winutils.h"
#include <winbase.h>
static void Usage(LPCWSTR program);
LONG WINAPI WinutilsSehUnhandled(_In_ struct _EXCEPTION_POINTERS *ecxr) {
LogDebugMessage(L"unhandled SEH: code:%x flags:%d\n",
ecxr->ExceptionRecord->ExceptionCode,
ecxr->ExceptionRecord->ExceptionFlags);
fwprintf(stderr, L"Unhandled exception code:%x at address:%p",
ecxr->ExceptionRecord->ExceptionCode,
ecxr->ExceptionRecord->ExceptionAddress);
ExitProcess(ERROR_UNHANDLED_EXCEPTION);
return EXCEPTION_EXECUTE_HANDLER; // not that it matters...
}
int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[])
{
LPCWSTR cmd = NULL;
SetUnhandledExceptionFilter(WinutilsSehUnhandled);
if (argc < 2)
{
Usage(argv[0]);
@ -67,6 +82,10 @@ int wmain(__in int argc, __in_ecount(argc) wchar_t* argv[])
{
return SystemInfo();
}
else if (wcscmp(L"service", cmd) == 0)
{
return RunService(argc - 1, argv + 1);
}
else if (wcscmp(L"help", cmd) == 0)
{
Usage(argv[0]);
@ -119,5 +138,9 @@ The available commands and their usages are:\n\n", program);
fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations.");
TaskUsage();
fwprintf(stdout, L"%-15s%s\n\n", L"service", L"Service operations.");
ServiceUsage();
fwprintf(stdout, L"\n\n");
}

1485
hadoop-common-project/hadoop-common/src/main/winutils/service.c Normal file
View File

File diff suppressed because it is too large Load Diff

707
hadoop-common-project/hadoop-common/src/main/winutils/task.c
View File

@ -19,15 +19,18 @@
#include <errno.h>
#include <psapi.h>
#include <malloc.h>
#include <authz.h>
#include <sddl.h>
#define PSAPI_VERSION 1
#pragma comment(lib, "psapi.lib")
#define ERROR_TASK_NOT_ALIVE 1
#define NM_WSCE_IMPERSONATE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.impersonate.allowed"
#define NM_WSCE_IMPERSONATE_DENIED L"yarn.nodemanager.windows-secure-container-executor.impersonate.denied"
// The S4U impersonation access check mask. Arbitrary value (we use 1 for the service access check)
#define SERVICE_IMPERSONATE_MASK 0x00000002
// This exit code for killed processes is compatible with Unix, where a killed
// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137.
#define KILLED_PROCESS_EXIT_CODE 137
// Name for tracking this logon process when registering with LSA
static const char *LOGON_PROCESS_NAME="Hadoop Container Executor";
@ -104,6 +107,459 @@ static BOOL ParseCommandLine(__in int argc,
return FALSE;
}
//----------------------------------------------------------------------------
// Function: BuildImpersonateSecurityDescriptor
//
// Description:
// Builds the security descriptor for the S4U impersonation permissions
// This describes what users can be impersonated and what not
//
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
//
DWORD BuildImpersonateSecurityDescriptor(__out PSECURITY_DESCRIPTOR* ppSD) {
DWORD dwError = ERROR_SUCCESS;
size_t countAllowed = 0;
PSID* allowedSids = NULL;
size_t countDenied = 0;
PSID* deniedSids = NULL;
LPCWSTR value = NULL;
WCHAR** tokens = NULL;
size_t len = 0;
size_t count = 0;
int crt = 0;
PSECURITY_DESCRIPTOR pSD = NULL;
dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_ALLOWED, &len, &value);
if (dwError) {
ReportErrorCode(L"GetConfigValue:1", dwError);
goto done;
}
if (0 == len) {
dwError = ERROR_BAD_CONFIGURATION;
ReportErrorCode(L"GetConfigValue:2", dwError);
goto done;
}
dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens);
if (dwError) {
ReportErrorCode(L"SplitStringIgnoreSpaceW:1", dwError);
goto done;
}
allowedSids = LocalAlloc(LPTR, sizeof(PSID) * count);
if (NULL == allowedSids) {
dwError = GetLastError();
ReportErrorCode(L"LocalAlloc:1", dwError);
goto done;
}
for(crt = 0; crt < count; ++crt) {
dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]);
if (dwError) {
ReportErrorCode(L"GetSidFromAcctNameW:1", dwError);
goto done;
}
}
countAllowed = count;
LocalFree(tokens);
tokens = NULL;
LocalFree(value);
value = NULL;
dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_DENIED, &len, &value);
if (dwError) {
ReportErrorCode(L"GetConfigValue:3", dwError);
goto done;
}
if (0 != len) {
dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens);
if (dwError) {
ReportErrorCode(L"SplitStringIgnoreSpaceW:2", dwError);
goto done;
}
deniedSids = LocalAlloc(LPTR, sizeof(PSID) * count);
if (NULL == allowedSids) {
dwError = GetLastError();
ReportErrorCode(L"LocalAlloc:2", dwError);
goto done;
}
for(crt = 0; crt < count; ++crt) {
dwError = GetSidFromAcctNameW(tokens[crt], &deniedSids[crt]);
if (dwError) {
ReportErrorCode(L"GetSidFromAcctNameW:2", dwError);
goto done;
}
}
countDenied = count;
}
dwError = BuildServiceSecurityDescriptor(
SERVICE_IMPERSONATE_MASK,
countAllowed, allowedSids,
countDenied, deniedSids,
NULL,
&pSD);
if (dwError) {
ReportErrorCode(L"BuildServiceSecurityDescriptor", dwError);
goto done;
}
*ppSD = pSD;
pSD = NULL;
done:
if (pSD) LocalFree(pSD);
if (tokens) LocalFree(tokens);
if (allowedSids) LocalFree(allowedSids);
if (deniedSids) LocalFree(deniedSids);
return dwError;
}
//----------------------------------------------------------------------------
// Function: AddNodeManagerAndUserACEsToObject
//
// Description:
// Adds ACEs to grant NM and user the provided access mask over a given handle
//
// Returns:
// ERROR_SUCCESS: on success
//
DWORD AddNodeManagerAndUserACEsToObject(
__in HANDLE hObject,
__in LPWSTR user,
__in ACCESS_MASK accessMask) {
DWORD dwError = ERROR_SUCCESS;
int countTokens = 0;
size_t len = 0;
LPCWSTR value = NULL;
WCHAR** tokens = NULL;
int crt = 0;
PACL pDacl = NULL;
PSECURITY_DESCRIPTOR psdProcess = NULL;
LPSTR lpszOldDacl = NULL, lpszNewDacl = NULL;
ULONG daclLen = 0;
PACL pNewDacl = NULL;
ACL_SIZE_INFORMATION si;
DWORD dwNewAclSize = 0;
PACE_HEADER pTempAce = NULL;
BYTE sidTemp[SECURITY_MAX_SID_SIZE];
DWORD cbSid = SECURITY_MAX_SID_SIZE;
PSID tokenSid = NULL;
// These hard-coded SIDs are allways added
WELL_KNOWN_SID_TYPE forcesSidTypes[] = {
WinLocalSystemSid,
WinBuiltinAdministratorsSid};
BOOL logSDs = IsDebuggerPresent(); // Check only once to avoid attach-while-running
dwError = GetSecurityInfo(hObject,
SE_KERNEL_OBJECT,
DACL_SECURITY_INFORMATION,
NULL,
NULL,
&pDacl,
NULL,
&psdProcess);
if (dwError) {
ReportErrorCode(L"GetSecurityInfo", dwError);
goto done;
}
// This is debug only output for troubleshooting
if (logSDs) {
if (!ConvertSecurityDescriptorToStringSecurityDescriptor(
psdProcess,
SDDL_REVISION_1,
DACL_SECURITY_INFORMATION,
&lpszOldDacl,
&daclLen)) {
dwError = GetLastError();
ReportErrorCode(L"ConvertSecurityDescriptorToStringSecurityDescriptor", dwError);
goto done;
}
}
ZeroMemory(&si, sizeof(si));
if (!GetAclInformation(pDacl, &si, sizeof(si), AclSizeInformation)) {
dwError = GetLastError();
ReportErrorCode(L"GetAclInformation", dwError);
goto done;
}
dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_ALLOWED, &len, &value);
if (ERROR_SUCCESS != dwError) {
ReportErrorCode(L"GetConfigValue", dwError);
goto done;
}
if (0 == len) {
dwError = ERROR_BAD_CONFIGURATION;
ReportErrorCode(L"GetConfigValue", dwError);
goto done;
}
dwError = SplitStringIgnoreSpaceW(len, value, L',', &countTokens, &tokens);
if (ERROR_SUCCESS != dwError) {
ReportErrorCode(L"SplitStringIgnoreSpaceW", dwError);
goto done;
}
// We're gonna add 1 ACE for each token found, +1 for user and +1 for each forcesSidTypes[]
// ACCESS_ALLOWED_ACE struct contains the first DWORD of the SID
//
dwNewAclSize = si.AclBytesInUse +
(countTokens + 1 + sizeof(forcesSidTypes)/sizeof(forcesSidTypes[0])) *
(sizeof(ACCESS_ALLOWED_ACE) + SECURITY_MAX_SID_SIZE - sizeof(DWORD));
pNewDacl = (PSID) LocalAlloc(LPTR, dwNewAclSize);
if (!pNewDacl) {
dwError = ERROR_OUTOFMEMORY;
ReportErrorCode(L"LocalAlloc", dwError);
goto done;
}
if (!InitializeAcl(pNewDacl, dwNewAclSize, ACL_REVISION)) {
dwError = ERROR_OUTOFMEMORY;
ReportErrorCode(L"InitializeAcl", dwError);
goto done;
}
// Copy over old ACEs
for (crt = 0; crt < si.AceCount; ++crt) {
if (!GetAce(pDacl, crt, &pTempAce)) {
dwError = ERROR_OUTOFMEMORY;
ReportErrorCode(L"InitializeAcl", dwError);
goto done;
}
if (!AddAce(pNewDacl, ACL_REVISION, MAXDWORD, pTempAce, pTempAce->AceSize)) {
dwError = ERROR_OUTOFMEMORY;
ReportErrorCode(L"InitializeAcl", dwError);
goto done;
}
}
// Add the configured allowed SIDs
for (crt = 0; crt < countTokens; ++crt) {
dwError = GetSidFromAcctNameW(tokens[crt], &tokenSid);
if (ERROR_SUCCESS != dwError) {
ReportErrorCode(L"GetSidFromAcctNameW", dwError);
goto done;
}
if (!AddAccessAllowedAceEx(
pNewDacl,
ACL_REVISION_DS,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
PROCESS_ALL_ACCESS,
tokenSid)) {
dwError = GetLastError();
ReportErrorCode(L"AddAccessAllowedAceEx:1", dwError);
goto done;
}
LocalFree(tokenSid);
tokenSid = NULL;
}
// add the forced SIDs ACE
for (crt = 0; crt < sizeof(forcesSidTypes)/sizeof(forcesSidTypes[0]); ++crt) {
cbSid = SECURITY_MAX_SID_SIZE;
if (!CreateWellKnownSid(forcesSidTypes[crt], NULL, &sidTemp, &cbSid)) {
dwError = GetLastError();
ReportErrorCode(L"CreateWellKnownSid", dwError);
goto done;
}
if (!AddAccessAllowedAceEx(
pNewDacl,
ACL_REVISION_DS,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
accessMask,
(PSID) sidTemp)) {
dwError = GetLastError();
ReportErrorCode(L"AddAccessAllowedAceEx:2", dwError);
goto done;
}
}
// add the user ACE
dwError = GetSidFromAcctNameW(user, &tokenSid);
if (ERROR_SUCCESS != dwError) {
ReportErrorCode(L"GetSidFromAcctNameW:user", dwError);
goto done;
}
if (!AddAccessAllowedAceEx(
pNewDacl,
ACL_REVISION_DS,
CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE,
PROCESS_ALL_ACCESS,
tokenSid)) {
dwError = GetLastError();
ReportErrorCode(L"AddAccessAllowedAceEx:3", dwError);
goto done;
}
LocalFree(tokenSid);
tokenSid = NULL;
dwError = SetSecurityInfo(hObject,
SE_KERNEL_OBJECT,
DACL_SECURITY_INFORMATION,
NULL,
NULL,
pNewDacl,
NULL);
if (dwError) {
ReportErrorCode(L"SetSecurityInfo", dwError);
goto done;
}
// This is debug only output for troubleshooting
if (logSDs) {
dwError = GetSecurityInfo(hObject,
SE_KERNEL_OBJECT,
DACL_SECURITY_INFORMATION,
NULL,
NULL,
&pDacl,
NULL,
&psdProcess);
if (dwError) {
ReportErrorCode(L"GetSecurityInfo:2", dwError);
goto done;
}
if (!ConvertSecurityDescriptorToStringSecurityDescriptor(
psdProcess,
SDDL_REVISION_1,
DACL_SECURITY_INFORMATION,
&lpszNewDacl,
&daclLen)) {
dwError = GetLastError();
ReportErrorCode(L"ConvertSecurityDescriptorToStringSecurityDescriptor:2", dwError);
goto done;
}
LogDebugMessage(L"Old DACL: %s\nNew DACL: %s\n", lpszOldDacl, lpszNewDacl);
}
done:
if (tokenSid) LocalFree(tokenSid);
if (pNewDacl) LocalFree(pNewDacl);
if (lpszOldDacl) LocalFree(lpszOldDacl);
if (lpszNewDacl) LocalFree(lpszNewDacl);
if (psdProcess) LocalFree(psdProcess);
return dwError;
}
//----------------------------------------------------------------------------
// Function: ValidateImpersonateAccessCheck
//
// Description:
// Performs the access check for S4U impersonation
//
// Returns:
// ERROR_SUCCESS: On success
// ERROR_ACCESS_DENIED, GetLastError: otherwise
//
DWORD ValidateImpersonateAccessCheck(__in HANDLE logonHandle) {
DWORD dwError = ERROR_SUCCESS;
PSECURITY_DESCRIPTOR pSD = NULL;
LUID luidUnused;
AUTHZ_ACCESS_REQUEST request;
AUTHZ_ACCESS_REPLY reply;
DWORD authError = ERROR_SUCCESS;
DWORD saclResult = 0;
ACCESS_MASK grantedMask = 0;
AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL;
AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzToken = NULL;
ZeroMemory(&luidUnused, sizeof(luidUnused));
ZeroMemory(&request, sizeof(request));
ZeroMemory(&reply, sizeof(reply));
dwError = BuildImpersonateSecurityDescriptor(&pSD);
if (dwError) {
ReportErrorCode(L"BuildImpersonateSecurityDescriptor", dwError);
goto done;
}
request.DesiredAccess = MAXIMUM_ALLOWED;
reply.Error = &authError;
reply.SaclEvaluationResults = &saclResult;
reply.ResultListLength = 1;
reply.GrantedAccessMask = &grantedMask;
if (!AuthzInitializeResourceManager(
AUTHZ_RM_FLAG_NO_AUDIT,
NULL, // pfnAccessCheck
NULL, // pfnComputeDynamicGroups
NULL, // pfnFreeDynamicGroups
NULL, // szResourceManagerName
&hManager)) {
dwError = GetLastError();
ReportErrorCode(L"AuthzInitializeResourceManager", dwError);
goto done;
}
if (!AuthzInitializeContextFromToken(
0,
logonHandle,
hManager,
NULL, // expiration time
luidUnused, // not used
NULL, // callback args
&hAuthzToken)) {
dwError = GetLastError();
ReportErrorCode(L"AuthzInitializeContextFromToken", dwError);
goto done;
}
if (!AuthzAccessCheck(
0,
hAuthzToken,
&request,
NULL, // AuditEvent
pSD,
NULL, // OptionalSecurityDescriptorArray
0, // OptionalSecurityDescriptorCount
&reply,
NULL // phAccessCheckResults
)) {
dwError = GetLastError();
ReportErrorCode(L"AuthzAccessCheck", dwError);
goto done;
}
LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n",
authError, saclResult, grantedMask);
if (authError != ERROR_SUCCESS) {
ReportErrorCode(L"AuthzAccessCheck:REPLY:1", authError);
dwError = authError;
}
else if (!(grantedMask & SERVICE_IMPERSONATE_MASK)) {
ReportErrorCode(L"AuthzAccessCheck:REPLY:2", ERROR_ACCESS_DENIED);
dwError = ERROR_ACCESS_DENIED;
}
done:
if (hAuthzToken) AuthzFreeContext(hAuthzToken);
if (hManager) AuthzFreeResourceManager(hManager);
if (pSD) LocalFree(pSD);
return dwError;
}
//----------------------------------------------------------------------------
// Function: CreateTaskImpl
//
@ -116,7 +572,8 @@ static BOOL ParseCommandLine(__in int argc,
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PWSTR cmdLine)
DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PCWSTR cmdLine,
__in LPCWSTR userName)
{
DWORD dwErrorCode = ERROR_SUCCESS;
DWORD exitCode = EXIT_FAILURE;
@ -126,19 +583,36 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
HANDLE jobObject = NULL;
JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 };
void * envBlock = NULL;
BOOL createProcessResult = FALSE;
WCHAR secureJobNameBuffer[MAX_PATH];
LPCWSTR secureJobName = jobObjName;
wchar_t* curr_dir = NULL;
FILE *stream = NULL;
if (NULL != logonHandle) {
dwErrorCode = ValidateImpersonateAccessCheck(logonHandle);
if (dwErrorCode) {
ReportErrorCode(L"ValidateImpersonateAccessCheck", dwErrorCode);
return dwErrorCode;
}
dwErrorCode = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer);
if (dwErrorCode) {
ReportErrorCode(L"GetSecureJobObjectName", dwErrorCode);
return dwErrorCode;
}
secureJobName = secureJobNameBuffer;
}
// Create un-inheritable job object handle and set job object to terminate
// when last handle is closed. So winutils.exe invocation has the only open
// job object handle. Exit of winutils.exe ensures termination of job object.
// Either a clean exit of winutils or crash or external termination.
jobObject = CreateJobObject(NULL, jobObjName);
jobObject = CreateJobObject(NULL, secureJobName);
dwErrorCode = GetLastError();
if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS)
{
ReportErrorCode(L"CreateJobObject", dwErrorCode);
return dwErrorCode;
}
jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
@ -148,6 +622,14 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
sizeof(jeli)) == 0)
{
dwErrorCode = GetLastError();
ReportErrorCode(L"SetInformationJobObject", dwErrorCode);
CloseHandle(jobObject);
return dwErrorCode;
}
dwErrorCode = AddNodeManagerAndUserACEsToObject(jobObject, userName, JOB_OBJECT_ALL_ACCESS);
if (dwErrorCode) {
ReportErrorCode(L"AddNodeManagerAndUserACEsToObject", dwErrorCode);
CloseHandle(jobObject);
return dwErrorCode;
}
@ -155,6 +637,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0)
{
dwErrorCode = GetLastError();
ReportErrorCode(L"AssignProcessToJobObject", dwErrorCode);
CloseHandle(jobObject);
return dwErrorCode;
}
@ -164,6 +647,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if(SetEnvironmentVariable(L"JVM_PID", jobObjName) == 0)
{
dwErrorCode = GetLastError();
ReportErrorCode(L"SetEnvironmentVariable", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
@ -180,6 +664,7 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
logonHandle,
TRUE )) {
dwErrorCode = GetLastError();
ReportErrorCode(L"CreateEnvironmentBlock", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
@ -197,42 +682,72 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
if (0 == currDirCnt) {
dwErrorCode = GetLastError();
ReportErrorCode(L"GetCurrentDirectory", dwErrorCode);
// We have to explictly Terminate, passing in the error code
// simply closing the job would kill our own process with success exit status
TerminateJobObject(jobObject, dwErrorCode);
return dwErrorCode;
}
dwErrorCode = ERROR_SUCCESS;
if (logonHandle == NULL) {
createProcessResult = CreateProcess(
NULL, // ApplicationName
cmdLine, // command line
NULL, // process security attributes
NULL, // thread security attributes
TRUE, // inherit handles
0, // creation flags
NULL, // environment
curr_dir, // current directory
&si, // startup info
&pi); // process info
}
else {
createProcessResult = CreateProcessAsUser(
logonHandle, // logon token handle
NULL, // Application handle
cmdLine, // command line
NULL, // process security attributes
NULL, // thread security attributes
FALSE, // inherit handles
CREATE_UNICODE_ENVIRONMENT, // creation flags
envBlock, // environment
curr_dir, // current directory
&si, // startup info
&pi); // process info
if (!CreateProcess(
NULL, // ApplicationName
cmdLine, // command line
NULL, // process security attributes
NULL, // thread security attributes
TRUE, // inherit handles
0, // creation flags
NULL, // environment
curr_dir, // current directory
&si, // startup info
&pi)) { // process info
dwErrorCode = GetLastError();
ReportErrorCode(L"CreateProcess", dwErrorCode);
}
goto create_process_done;
}
if (FALSE == createProcessResult) {
// From here on is the secure S4U implementation for CreateProcessAsUser
// We desire to grant process access to NM so that it can interogate process status
// and resource utilization. Passing in a security descriptor though results in the
// S4U privilege checks being done against that SD and CreateProcessAsUser fails.
// So instead we create the process suspended and then we add the desired ACEs.
//
if (!CreateProcessAsUser(
logonHandle, // logon token handle
NULL, // Application handle
cmdLine, // command line
NULL, // process security attributes
NULL, // thread security attributes
FALSE, // inherit handles
CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, // creation flags
envBlock, // environment
curr_dir, // current directory
&si, // startup info
&pi)) { // process info
dwErrorCode = GetLastError();
ReportErrorCode(L"CreateProcessAsUser", dwErrorCode);
goto create_process_done;
}
dwErrorCode = AddNodeManagerAndUserACEsToObject(pi.hProcess, userName, PROCESS_ALL_ACCESS);
if (dwErrorCode) {
ReportErrorCode(L"AddNodeManagerAndUserACEsToObject", dwErrorCode);
goto create_process_done;
}
if (-1 == ResumeThread(pi.hThread)) {
dwErrorCode = GetLastError();
ReportErrorCode(L"ResumeThread", dwErrorCode);
goto create_process_done;
}
create_process_done:
if (dwErrorCode) {
if( envBlock != NULL ) {
DestroyEnvironmentBlock( envBlock );
envBlock = NULL;
@ -293,10 +808,11 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW
DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
{
// call with null logon in order to create tasks utilizing the current logon
return CreateTaskImpl( NULL, jobObjName, cmdLine );
return CreateTaskImpl( NULL, jobObjName, cmdLine, NULL);
}
//----------------------------------------------------------------------------
// Function: CreateTask
// Function: CreateTaskAsUser
//
// Description:
// Creates a task via a jobobject. Outputs the
@ -305,7 +821,8 @@ DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine)
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
DWORD CreateTaskAsUser(__in PCWSTR jobObjName,__in PWSTR user, __in PWSTR pidFilePath, __in PWSTR cmdLine)
DWORD CreateTaskAsUser(__in PCWSTR jobObjName,
__in PCWSTR user, __in PCWSTR pidFilePath, __in PCWSTR cmdLine)
{
DWORD err = ERROR_SUCCESS;
DWORD exitCode = EXIT_FAILURE;
@ -314,53 +831,50 @@ DWORD CreateTaskAsUser(__in PCWSTR jobObjName,__in PWSTR user, __in PWSTR pidFil
PROFILEINFO pi;
BOOL profileIsLoaded = FALSE;
FILE* pidFile = NULL;
DWORD retLen = 0;
HANDLE logonHandle = NULL;
err = EnablePrivilege(SE_TCB_NAME);
err = EnableImpersonatePrivileges();
if( err != ERROR_SUCCESS ) {
fwprintf(stdout, L"INFO: The user does not have SE_TCB_NAME.\n");
goto done;
}
err = EnablePrivilege(SE_ASSIGNPRIMARYTOKEN_NAME);
if( err != ERROR_SUCCESS ) {
fwprintf(stdout, L"INFO: The user does not have SE_ASSIGNPRIMARYTOKEN_NAME.\n");
goto done;
}
err = EnablePrivilege(SE_INCREASE_QUOTA_NAME);
if( err != ERROR_SUCCESS ) {
fwprintf(stdout, L"INFO: The user does not have SE_INCREASE_QUOTA_NAME.\n");
goto done;
}
err = EnablePrivilege(SE_RESTORE_NAME);
if( err != ERROR_SUCCESS ) {
fwprintf(stdout, L"INFO: The user does not have SE_RESTORE_NAME.\n");
ReportErrorCode(L"EnableImpersonatePrivileges", err);
goto done;
}
err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle);
if( err != ERROR_SUCCESS ) goto done;
if( err != ERROR_SUCCESS ) {
ReportErrorCode(L"RegisterWithLsa", err);
goto done;
}
err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId );
if( err != ERROR_SUCCESS ) goto done;
if( err != ERROR_SUCCESS ) {
ReportErrorCode(L"LookupKerberosAuthenticationPackageId", err);
goto done;
}
err = CreateLogonForUser(lsaHandle,
err = CreateLogonTokenForUser(lsaHandle,
LOGON_PROCESS_NAME,
TOKEN_SOURCE_NAME,
authnPkgId,
user,
&logonHandle);
if( err != ERROR_SUCCESS ) goto done;
if( err != ERROR_SUCCESS ) {
ReportErrorCode(L"CreateLogonTokenForUser", err);
goto done;
}
err = LoadUserProfileForLogon(logonHandle, &pi);
if( err != ERROR_SUCCESS ) goto done;
if( err != ERROR_SUCCESS ) {
ReportErrorCode(L"LoadUserProfileForLogon", err);
goto done;
}
profileIsLoaded = TRUE;
// Create the PID file
if (!(pidFile = _wfopen(pidFilePath, "w"))) {
err = GetLastError();
ReportErrorCode(L"_wfopen:pidFilePath", err);
goto done;
}
@ -371,10 +885,11 @@ DWORD CreateTaskAsUser(__in PCWSTR jobObjName,__in PWSTR user, __in PWSTR pidFil
fclose(pidFile);
if (err != ERROR_SUCCESS) {
goto done;
ReportErrorCode(L"fprintf_s:pidFilePath", err);
goto done;
}
err = CreateTaskImpl(logonHandle, jobObjName, cmdLine);
err = CreateTaskImpl(logonHandle, jobObjName, cmdLine, user);
done:
if( profileIsLoaded ) {
@ -392,7 +907,6 @@ done:
return err;
}
//----------------------------------------------------------------------------
// Function: IsTaskAlive
//
@ -408,10 +922,27 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
HANDLE jobObject = NULL;
int numProcs = 100;
WCHAR secureJobNameBuffer[MAX_PATH];
*isAlive = FALSE;
jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
if(jobObject == NULL)
{
// Try Global\...
DWORD err = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer);
if (err) {
ReportErrorCode(L"GetSecureJobObjectName", err);
return err;
}
jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, secureJobNameBuffer);
}
if(jobObject == NULL)
{
DWORD err = GetLastError();
return err;
}
if(jobObject == NULL)
{
@ -453,39 +984,6 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob)
return ERROR_SUCCESS;
}
//----------------------------------------------------------------------------
// Function: KillTask
//
// Description:
// Kills a task via a jobobject. Outputs the
// appropriate information to stdout on success, or stderr on failure.
//
// Returns:
// ERROR_SUCCESS: On success
// GetLastError: otherwise
DWORD KillTask(PCWSTR jobObjName)
{
HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName);
if(jobObject == NULL)
{
DWORD err = GetLastError();
if(err == ERROR_FILE_NOT_FOUND)
{
// job object does not exist. assume its not alive
return ERROR_SUCCESS;
}
return err;
}
if(TerminateJobObject(jobObject, KILLED_PROCESS_EXIT_CODE) == 0)
{
return GetLastError();
}
CloseHandle(jobObject);
return ERROR_SUCCESS;
}
//----------------------------------------------------------------------------
// Function: PrintTaskProcessList
//
@ -500,7 +998,21 @@ DWORD PrintTaskProcessList(const WCHAR* jobObjName)
DWORD i;
PJOBOBJECT_BASIC_PROCESS_ID_LIST procList;
int numProcs = 100;
HANDLE jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
WCHAR secureJobNameBuffer[MAX_PATH];
HANDLE jobObject = NULL;
jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName);
if(jobObject == NULL)
{
// Try Global\...
DWORD err = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer);
if (err) {
ReportErrorCode(L"GetSecureJobObjectName", err);
return err;
}
jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, secureJobNameBuffer);
}
if(jobObject == NULL)
{
DWORD err = GetLastError();
@ -581,7 +1093,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
TaskCommandOption command = TaskInvalid;
wchar_t* cmdLine = NULL;
wchar_t buffer[16*1024] = L""; // 32K max command line
size_t charCountBufferLeft = sizeof (buffer)/sizeof(wchar_t);
size_t charCountBufferLeft = sizeof(buffer)/sizeof(wchar_t);
int crtArgIndex = 0;
size_t argLen = 0;
size_t wscatErr = 0;
@ -712,6 +1224,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[])
}
TaskExit:
ReportErrorCode(L"TaskExit:", dwErrorCode);
return dwErrorCode;
}

64
hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc Normal file
View File

@ -0,0 +1,64 @@
;/*
; * Licensed to the Apache Software Foundation (ASF) under one
; * or more contributor license agreements. See the NOTICE file
; * distributed with this work for additional information
; * regarding copyright ownership. The ASF licenses this file
; * to you under the Apache License, Version 2.0 (the
; * "License"); you may not use this file except in compliance
; * with the License. You may obtain a copy of the License at
; *
; * http://www.apache.org/licenses/LICENSE-2.0
; *
; * Unless required by applicable law or agreed to in writing, software
; * distributed under the License is distributed on an "AS IS" BASIS,
; * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
; * See the License for the specific language governing permissions and
; * limitations under the License.
; */
; // winutils.mc
; // EventLog messages for Hadoop winutils service.
LanguageNames=(English=0x409:MSG00409)
; // The following are the categories of events.
MessageIdTypedef=WORD
MessageId=0x1
SymbolicName=SERVICE_CATEGORY
Language=English
Service Events
.
MessageId=0x2
SymbolicName=LOG_CATEGORY
Language=English
Task Events
.
; // The following are the message definitions.
MessageIdTypedef=DWORD
MessageId=0x80
SymbolicName=MSG_CHECK_ERROR
Language=English
%1. Error %2: %3.
.
MessageId=0x100
SymbolicName=MSG_RPC_SERVICE_HAS_STARTED
Language=English
The LPC server is listenning.
.
MessageId=0x200
SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED
Language=English
The LPC server has stopped listenning.
.

18
hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln
View File

@ -26,26 +26,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils.
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Win32 = Debug|Win32
Debug|x64 = Debug|x64
Release|Win32 = Release|Win32
Release|x64 = Release|x64
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64
{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64
{12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64
EndGlobalSection

108
hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj
View File

@ -19,18 +19,10 @@
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|Win32">
<Configuration>Debug</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|Win32">
<Configuration>Release</Configuration>
<Platform>Win32</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
@ -42,22 +34,11 @@
<RootNamespace>winutils</RootNamespace>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
<ConfigurationType>Application</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
@ -67,15 +48,9 @@
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
@ -83,74 +58,32 @@
<PropertyGroup>
<IncludePath>include;$(IncludePath)</IncludePath>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<OutDir />
<IntDir>..\..\..\target\winutils\$(Configuration)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IntDir>..\..\..\target\winutils\$(Platform)\$(Configuration)\</IntDir>
<OutDir>..\..\..\target\bin\</OutDir>
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<ClCompile>
<PrecompiledHeader>
</PrecompiledHeader>
<WarningLevel>Level4</WarningLevel>
<Optimization>Disabled</Optimization>
<PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>WIN32;_CONSOLE;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
<GenerateDebugInformation>true</GenerateDebugInformation>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<PrecompiledHeader>
</PrecompiledHeader>
<Optimization>MaxSpeed</Optimization>
<!-- <Optimization>MaxSpeed</Optimization> -->
<Optimization>Disabled</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions>WIN32;_CONSOLE;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions>
</ClCompile>
<Link>
<SubSystem>Console</SubSystem>
@ -159,7 +92,40 @@
<OptimizeReferences>true</OptimizeReferences>
</Link>
</ItemDefinitionGroup>
<ItemDefinitionGroup>
<ClCompile>
<AdditionalIncludeDirectories>$(IntermediateOutputPath)</AdditionalIncludeDirectories>
</ClCompile>
<CustomBuildStep>
<Message>Compiling Messages</Message>
<Command>mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U</Command>
<Outputs>$(IntermediateOutputPath)$(TargetName)_msg.rc;$(IntermediateOutputPath)$(TargetName)_msg.h</Outputs>
</CustomBuildStep>
<Midl>
<ApplicationConfigurationMode>true</ApplicationConfigurationMode>
<TargetEnvironment>X64</TargetEnvironment>
<OutputDirectory>$(IntermediateOutputPath)</OutputDirectory>
<GenerateStublessProxies>true</GenerateStublessProxies>
<ValidateAllParameters>true</ValidateAllParameters>
<WarnAsError>true</WarnAsError>
<WarningLevel>2</WarningLevel>
</Midl>
</ItemDefinitionGroup>
<PropertyGroup>
<CustomBuildAfterTargets>Midl</CustomBuildAfterTargets>
<CustomBuildBeforeTargets>ClCompile,ResourceCompile</CustomBuildBeforeTargets>
</PropertyGroup>
<ItemGroup>
<Midl Include="hadoopwinutilsvc.idl" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="$(IntermediateOutputPath)$(TargetName)_msg.rc" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="$(IntermediateOutputPath)\hadoopwinutilsvc_s.c" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="service.c" />
<ClCompile Include="readlink.c" />
<ClCompile Include="symlink.c" />
<ClCompile Include="systeminfo.c" />

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java
View File

@ -296,7 +296,7 @@ public class ProcessTree {
return false;
} catch (IOException ioe) {
LOG.warn("Error executing shell command "
+ Arrays.toString(shexec.getExecString()) + ioe);
+ shexec.toString() + ioe);
return false;
}
return (shexec.getExitCode() == 0 ? true : false);
@ -321,7 +321,7 @@ public class ProcessTree {
return false;
} catch (IOException ioe) {
LOG.warn("Error executing shell command "
+ Arrays.toString(shexec.getExecString()) + ioe);
+ shexec.toString() + ioe);
return false;
}
return (shexec.getExitCode() == 0 ? true : false);

3
hadoop-yarn-project/CHANGES.txt
View File

@ -178,6 +178,9 @@ Release 2.6.0 - UNRELEASED
YARN-2504. Enhanced RM Admin CLI to support management of node-labels.
(Wangda Tan via vinodkv)
YARN-2198. Remove the need to run NodeManager as privileged account for
Windows Secure Container Executor. (Remus Rusanu via jianhe)
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
View File

@ -79,19 +79,23 @@ public abstract class ContainerExecutor implements Configurable {
public abstract void init() throws IOException;
/**
* On Windows the ContainerLaunch creates a temporary empty jar to workaround the CLASSPATH length
* In a secure cluster this jar must be localized so that the container has access to it
* On Windows the ContainerLaunch creates a temporary special jar manifest of
* other jars to workaround the CLASSPATH length. In a secure cluster this
* jar must be localized so that the container has access to it.
* This function localizes on-demand the jar.
*
* @param classPathJar
* @param owner
* @throws IOException
*/
public void localizeClasspathJar(Path classPathJar, String owner) throws IOException {
// For the default container this is a no-op
// The WindowsSecureContainerExecutor overrides this
public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner)
throws IOException {
// Non-secure executor simply use the classpath created
// in the NM fprivate folder
return classPathJar;
}
/**
* Prepare the environment for containers in this application to execute.
* For $x in local.dirs
@ -105,14 +109,13 @@ public abstract class ContainerExecutor implements Configurable {
* @param appId id of the application
* @param nmPrivateContainerTokens path to localized credentials, rsrc by NM
* @param nmAddr RPC address to contact NM
* @param localDirs nm-local-dirs
* @param logDirs nm-log-dirs
* @param dirsHandler NM local dirs service, for nm-local-dirs and nm-log-dirs
* @throws IOException For most application init failures
* @throws InterruptedException If application init thread is halted by NM
*/
public abstract void startLocalizer(Path nmPrivateContainerTokens,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<String> localDirs, List<String> logDirs)
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException;
@ -132,8 +135,8 @@ public abstract class ContainerExecutor implements Configurable {
*/
public abstract int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
String user, String appId, Path containerWorkDir, List<String> localDirs,
List<String> logDirs) throws IOException;
String user, String appId, Path containerWorkDir,
List<String> localDirs, List<String> logDirs) throws IOException;
public abstract boolean signalContainer(String user, String pid,
Signal signal)

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
@ -32,9 +33,11 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.Random;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@ -42,6 +45,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.CommandExecutor;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
@ -92,9 +96,12 @@ public class DefaultContainerExecutor extends ContainerExecutor {
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<String> localDirs, List<String> logDirs)
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
ContainerLocalizer localizer =
new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
@ -120,7 +127,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
@Override
public int launchContainer(Container container,
Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath,
String userName, String appId, Path containerWorkDir,
String user, String appId, Path containerWorkDir,
List<String> localDirs, List<String> logDirs) throws IOException {
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
@ -134,29 +141,30 @@ public class DefaultContainerExecutor extends ContainerExecutor {
getApplicationId());
for (String sLocalDir : localDirs) {
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
Path userdir = new Path(usersdir, userName);
Path userdir = new Path(usersdir, user);
Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE);
Path appDir = new Path(appCacheDir, appIdStr);
Path containerDir = new Path(appDir, containerIdStr);
createDir(containerDir, dirPerm, true, userName);
createDir(containerDir, dirPerm, true, user);
}
// Create the container log-dirs on all disks
createContainerLogDirs(appIdStr, containerIdStr, logDirs, userName);
createContainerLogDirs(appIdStr, containerIdStr, logDirs, user);
Path tmpDir = new Path(containerWorkDir,
YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
createDir(tmpDir, dirPerm, false, userName);
createDir(tmpDir, dirPerm, false, user);
// copy launch script to work dir
Path launchDst =
new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
copyFile(nmPrivateContainerScriptPath, launchDst, userName);
// copy container tokens to work dir
Path tokenDst =
new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE);
copyFile(nmPrivateTokensPath, tokenDst, userName);
copyFile(nmPrivateTokensPath, tokenDst, user);
// copy launch script to work dir
Path launchDst =
new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
copyFile(nmPrivateContainerScriptPath, launchDst, user);
// Create new local launch wrapper script
LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder(
@ -184,20 +192,16 @@ public class DefaultContainerExecutor extends ContainerExecutor {
// create log dir under app
// fork script
ShellCommandExecutor shExec = null;
Shell.CommandExecutor shExec = null;
try {
setScriptExecutable(launchDst, userName);
setScriptExecutable(sb.getWrapperScriptPath(), userName);
setScriptExecutable(launchDst, user);
setScriptExecutable(sb.getWrapperScriptPath(), user);
// Setup command to run
String[] command = getRunCommand(sb.getWrapperScriptPath().toString(),
containerIdStr, userName, pidFile, this.getConf());
LOG.info("launchContainer: " + Arrays.toString(command));
shExec = new ShellCommandExecutor(
command,
shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(),
containerIdStr, user, pidFile,
new File(containerWorkDir.toUri().getPath()),
container.getLaunchContext().getEnvironment()); // sanitized env
container.getLaunchContext().getEnvironment());
if (isContainerActive(containerId)) {
shExec.execute();
}
@ -242,11 +246,26 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
return exitCode;
} finally {
; //
if (shExec != null) shExec.close();
}
return 0;
}
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath,
String containerIdStr, String user, Path pidFile, File wordDir,
Map<String, String> environment)
throws IOException {
String[] command = getRunCommand(wrapperScriptPath,
containerIdStr, user, pidFile, this.getConf());
LOG.info("launchContainer: " + Arrays.toString(command));
return new ShellCommandExecutor(
command,
wordDir,
environment);
}
protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
String containerIdStr, Path containerWorkDir) {
return Shell.WINDOWS ?
@ -421,7 +440,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
* @param signal signal to send
* (for logging).
*/
private void killContainer(String pid, Signal signal) throws IOException {
protected void killContainer(String pid, Signal signal) throws IOException {
new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid))
.execute();
}

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -196,9 +196,12 @@ public class LinuxContainerExecutor extends ContainerExecutor {
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<String> localDirs, List<String> logDirs)
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
verifyUsernamePattern(user);
String runAsUser = getRunAsUser(user);
List<String> command = new ArrayList<String>();

639
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
View File

@ -17,36 +17,272 @@
*/
package org.apache.hadoop.yarn.server.nodemanager;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileDescriptor;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DelegateToFileSystem;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.nativeio.NativeIO.Windows;
import org.apache.hadoop.io.nativeio.NativeIOException;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.Shell.CommandExecutor;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.LocalWrapperScriptBuilder;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
/**
* Windows secure container executor. Uses winutils task createAsUser.
*
* Windows secure container executor (WSCE).
* This class offers a secure container executor on Windows, similar to the
* LinuxContainerExecutor. As the NM does not run on a high privileged context,
* this class delegates elevated operations to the helper hadoopwintuilsvc,
* implemented by the winutils.exe running as a service.
* JNI and LRPC is used to communicate with the privileged service.
*/
public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
private static final Log LOG = LogFactory
.getLog(WindowsSecureContainerExecutor.class);
public static final String LOCALIZER_PID_FORMAT = "STAR_LOCALIZER_%s";
/**
* This class is a container for the JNI Win32 native methods used by WSCE.
*/
private static class Native {
private static boolean nativeLoaded = false;
static {
if (NativeCodeLoader.isNativeCodeLoaded()) {
try {
initWsceNative();
nativeLoaded = true;
} catch (Throwable t) {
LOG.info("Unable to initialize WSCE Native libraries", t);
}
}
}
/** Initialize the JNI method ID and class ID cache */
private static native void initWsceNative();
/**
* This class contains methods used by the WindowsSecureContainerExecutor
* file system operations.
*/
public static class Elevated {
private static final int MOVE_FILE = 1;
private static final int COPY_FILE = 2;
public static void mkdir(Path dirName) throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for mkdir");
}
elevatedMkDirImpl(dirName.toString());
}
private static native void elevatedMkDirImpl(String dirName)
throws IOException;
public static void chown(Path fileName, String user, String group)
throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for chown");
}
elevatedChownImpl(fileName.toString(), user, group);
}
private static native void elevatedChownImpl(String fileName, String user,
String group) throws IOException;
public static void move(Path src, Path dst, boolean replaceExisting)
throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for move");
}
elevatedCopyImpl(MOVE_FILE, src.toString(), dst.toString(),
replaceExisting);
}
public static void copy(Path src, Path dst, boolean replaceExisting)
throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for copy");
}
elevatedCopyImpl(COPY_FILE, src.toString(), dst.toString(),
replaceExisting);
}
private static native void elevatedCopyImpl(int operation, String src,
String dst, boolean replaceExisting) throws IOException;
public static void chmod(Path fileName, int mode) throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for chmod");
}
elevatedChmodImpl(fileName.toString(), mode);
}
private static native void elevatedChmodImpl(String path, int mode)
throws IOException;
public static void killTask(String containerName) throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for killTask");
}
elevatedKillTaskImpl(containerName);
}
private static native void elevatedKillTaskImpl(String containerName)
throws IOException;
public static OutputStream create(Path f, boolean append)
throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for create");
}
long desiredAccess = Windows.GENERIC_WRITE;
long shareMode = 0L;
long creationDisposition = append ?
Windows.OPEN_ALWAYS : Windows.CREATE_ALWAYS;
long flags = Windows.FILE_ATTRIBUTE_NORMAL;
String fileName = f.toString();
fileName = fileName.replace('/', '\\');
long hFile = elevatedCreateImpl(
fileName, desiredAccess, shareMode, creationDisposition, flags);
return new FileOutputStream(
WinutilsProcessStub.getFileDescriptorFromHandle(hFile));
}
private static native long elevatedCreateImpl(String path,
long desiredAccess, long shareMode,
long creationDisposition, long flags) throws IOException;
public static boolean deleteFile(Path path) throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for deleteFile");
}
return elevatedDeletePathImpl(path.toString(), false);
}
public static boolean deleteDirectory(Path path) throws IOException {
if (!nativeLoaded) {
throw new IOException("Native WSCE libraries are required for deleteDirectory");
}
return elevatedDeletePathImpl(path.toString(), true);
}
public native static boolean elevatedDeletePathImpl(String path,
boolean isDir) throws IOException;
}
/**
* Wraps a process started by the winutils service helper.
*
*/
public static class WinutilsProcessStub extends Process {
private final long hProcess;
private final long hThread;
private boolean disposed = false;
private final InputStream stdErr;
private final InputStream stdOut;
private final OutputStream stdIn;
public WinutilsProcessStub(long hProcess, long hThread, long hStdIn,
long hStdOut, long hStdErr) {
this.hProcess = hProcess;
this.hThread = hThread;
this.stdIn = new FileOutputStream(getFileDescriptorFromHandle(hStdIn));
this.stdOut = new FileInputStream(getFileDescriptorFromHandle(hStdOut));
this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr));
}
public static native FileDescriptor getFileDescriptorFromHandle(long handle);
@Override
public native void destroy();
@Override
public native int exitValue();
@Override
public InputStream getErrorStream() {
return stdErr;
}
@Override
public InputStream getInputStream() {
return stdOut;
}
@Override
public OutputStream getOutputStream() {
return stdIn;
}
@Override
public native int waitFor() throws InterruptedException;
public synchronized native void dispose();
public native void resume() throws NativeIOException;
}
public synchronized static WinutilsProcessStub createTaskAsUser(
String cwd, String jobName, String user, String pidFile, String cmdLine)
throws IOException {
if (!nativeLoaded) {
throw new IOException(
"Native WSCE libraries are required for createTaskAsUser");
}
synchronized(Shell.WindowsProcessLaunchLock) {
return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine);
}
}
private static native WinutilsProcessStub createTaskAsUser0(
String cwd, String jobName, String user, String pidFile, String cmdLine)
throws NativeIOException;
}
/**
* A shell script wrapper builder for WSCE.
* Overwrites the default behavior to remove the creation of the PID file in
* the script wrapper. WSCE creates the pid file as part of launching the
* task in winutils.
*/
private class WindowsSecureWrapperScriptBuilder
extends LocalWrapperScriptBuilder {
@ -60,19 +296,276 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
}
}
/**
* This is a skeleton file system used to elevate certain operations.
* WSCE has to create container dirs under local/userchache/$user but
* this dir itself is owned by $user, with chmod 750. As ther NM has no
* write access, it must delegate the write operations to the privileged
* hadoopwintuilsvc.
*/
private static class ElevatedFileSystem extends DelegateToFileSystem {
/**
* This overwrites certain RawLocalSystem operations to be performed by a
* privileged process.
*
*/
private static class ElevatedRawLocalFilesystem extends RawLocalFileSystem {
@Override
protected boolean mkOneDir(File p2f) throws IOException {
Path path = new Path(p2f.getAbsolutePath());
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:mkOneDir: %s", path));
}
boolean ret = false;
// File.mkdir returns false, does not throw. Must mimic it.
try {
Native.Elevated.mkdir(path);
ret = true;
}
catch(Throwable e) {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:mkOneDir: %s",
org.apache.hadoop.util.StringUtils.stringifyException(e)));
}
}
return ret;
}
@Override
public void setPermission(Path p, FsPermission permission)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:setPermission: %s %s", p, permission));
}
Native.Elevated.chmod(p, permission.toShort());
}
@Override
public void setOwner(Path p, String username, String groupname)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:setOwner: %s %s %s",
p, username, groupname));
}
Native.Elevated.chown(p, username, groupname);
}
@Override
protected OutputStream createOutputStream(Path f, boolean append)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:create: %s %b", f, append));
}
return Native.Elevated.create(f, append);
}
@Override
public boolean delete(Path p, boolean recursive) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("EFS:delete: %s %b", p, recursive));
}
// The super delete uses the FileUtil.fullyDelete,
// but we cannot rely on that because we need to use the elevated
// operations to remove the files
//
File f = pathToFile(p);
if (!f.exists()) {
//no path, return false "nothing to delete"
return false;
}
else if (f.isFile()) {
return Native.Elevated.deleteFile(p);
}
else if (f.isDirectory()) {
// This is a best-effort attempt. There are race conditions in that
// child files can be created/deleted after we snapped the list.
// No need to protect against that case.
File[] files = FileUtil.listFiles(f);
int childCount = files.length;
if (recursive) {
for(File child:files) {
if (delete(new Path(child.getPath()), recursive)) {
--childCount;
}
}
}
if (childCount == 0) {
return Native.Elevated.deleteDirectory(p);
}
else {
throw new IOException("Directory " + f.toString() + " is not empty");
}
}
else {
// This can happen under race conditions if an external agent
// is messing with the file type between IFs
throw new IOException("Path " + f.toString() +
" exists, but is neither a file nor a directory");
}
}
}
protected ElevatedFileSystem() throws IOException, URISyntaxException {
super(FsConstants.LOCAL_FS_URI,
new ElevatedRawLocalFilesystem(),
new Configuration(),
FsConstants.LOCAL_FS_URI.getScheme(),
false);
}
}
private static class WintuilsProcessStubExecutor
implements Shell.CommandExecutor {
private Native.WinutilsProcessStub processStub;
private StringBuilder output = new StringBuilder();
private int exitCode;
private enum State {
INIT,
RUNNING,
COMPLETE
};
private State state;;
private final String cwd;
private final String jobName;
private final String userName;
private final String pidFile;
private final String cmdLine;
public WintuilsProcessStubExecutor(
String cwd,
String jobName,
String userName,
String pidFile,
String cmdLine) {
this.cwd = cwd;
this.jobName = jobName;
this.userName = userName;
this.pidFile = pidFile;
this.cmdLine = cmdLine;
this.state = State.INIT;
}
private void assertComplete() throws IOException {
if (state != State.COMPLETE) {
throw new IOException("Process is not complete");
}
}
public String getOutput () throws IOException {
assertComplete();
return output.toString();
}
public int getExitCode() throws IOException {
assertComplete();
return exitCode;
}
public void validateResult() throws IOException {
assertComplete();
if (0 != exitCode) {
LOG.warn(output.toString());
throw new IOException("Processs exit code is:" + exitCode);
}
}
private Thread startStreamReader(final InputStream stream)
throws IOException {
Thread streamReaderThread = new Thread() {
@Override
public void run() {
try
{
BufferedReader lines = new BufferedReader(
new InputStreamReader(stream));
char[] buf = new char[512];
int nRead;
while ((nRead = lines.read(buf, 0, buf.length)) > 0) {
output.append(buf, 0, nRead);
}
}
catch(Throwable t) {
LOG.error("Error occured reading the process stdout", t);
}
}
};
streamReaderThread.start();
return streamReaderThread;
}
public void execute() throws IOException {
if (state != State.INIT) {
throw new IOException("Process is already started");
}
processStub = Native.createTaskAsUser(cwd,
jobName, userName, pidFile, cmdLine);
state = State.RUNNING;
Thread stdOutReader = startStreamReader(processStub.getInputStream());
Thread stdErrReader = startStreamReader(processStub.getErrorStream());
try {
processStub.resume();
processStub.waitFor();
stdOutReader.join();
stdErrReader.join();
}
catch(InterruptedException ie) {
throw new IOException(ie);
}
exitCode = processStub.exitValue();
state = State.COMPLETE;
}
@Override
public void close() {
if (processStub != null) {
processStub.dispose();
}
}
}
private String nodeManagerGroup;
/**
* Permissions for user WSCE dirs.
*/
static final short DIR_PERM = (short)0750;
public WindowsSecureContainerExecutor()
throws IOException, URISyntaxException {
super(FileContext.getFileContext(new ElevatedFileSystem(),
new Configuration()));
}
@Override
public void setConf(Configuration conf) {
super.setConf(conf);
nodeManagerGroup = conf.get(YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP);
nodeManagerGroup = conf.get(
YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP);
}
@Override
protected String[] getRunCommand(String command, String groupId,
String userName, Path pidFile, Configuration conf) {
return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, userName,
pidFile.toString(), "cmd /c " + command };
File f = new File(command);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("getRunCommand: %s exists:%b",
command, f.exists()));
}
return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId,
userName, pidFile.toString(), "cmd /c " + command };
}
@Override
@ -83,34 +576,68 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
@Override
protected void copyFile(Path src, Path dst, String owner) throws IOException {
super.copyFile(src, dst, owner);
lfs.setOwner(dst, owner, nodeManagerGroup);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("copyFile: %s -> %s owner:%s", src.toString(),
dst.toString(), owner));
}
Native.Elevated.copy(src, dst, true);
Native.Elevated.chown(dst, owner, nodeManagerGroup);
}
@Override
protected void createDir(Path dirPath, FsPermission perms,
boolean createParent, String owner) throws IOException {
// WSCE requires dirs to be 750, not 710 as DCE.
// This is similar to how LCE creates dirs
//
perms = new FsPermission(DIR_PERM);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("createDir: %s perm:%s owner:%s",
dirPath.toString(), perms.toString(), owner));
}
super.createDir(dirPath, perms, createParent, owner);
lfs.setOwner(dirPath, owner, nodeManagerGroup);
}
@Override
protected void setScriptExecutable(Path script, String owner) throws IOException {
super.setScriptExecutable(script, null);
lfs.setOwner(script, owner, nodeManagerGroup);
protected void setScriptExecutable(Path script, String owner)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("setScriptExecutable: %s owner:%s",
script.toString(), owner));
}
super.setScriptExecutable(script, owner);
Native.Elevated.chown(script, owner, nodeManagerGroup);
}
@Override
public void localizeClasspathJar(Path classpathJar, String owner) throws IOException {
lfs.setOwner(classpathJar, owner, nodeManagerGroup);
public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("localizeClasspathJar: %s %s o:%s",
classPathJar, pwd, owner));
}
createDir(pwd, new FsPermission(DIR_PERM), true, owner);
String fileName = classPathJar.getName();
Path dst = new Path(pwd, fileName);
Native.Elevated.move(classPathJar, dst, true);
Native.Elevated.chown(dst, owner, nodeManagerGroup);
return dst;
}
@Override
public void startLocalizer(Path nmPrivateContainerTokens,
InetSocketAddress nmAddr, String user, String appId, String locId,
List<String> localDirs, List<String> logDirs) throws IOException,
LocalDirsHandlerService dirsHandler) throws IOException,
InterruptedException {
List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs();
Path classpathJarPrivateDir = dirsHandler.getLocalPathForWrite(
ResourceLocalizationService.NM_PRIVATE_DIR);
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@ -118,55 +645,87 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor {
Path appStorageDir = getWorkingDir(localDirs, user, appId);
String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
String tokenFn = String.format(
ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
LOG.info("Copying from " + nmPrivateContainerTokens + " to " + tokenDst);
copyFile(nmPrivateContainerTokens, tokenDst, user);
List<String> command ;
String[] commandArray;
ShellCommandExecutor shExec;
File cwdApp = new File(appStorageDir.toString());
LOG.info(String.format("cwdApp: %s", cwdApp));
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("cwdApp: %s", cwdApp));
}
List<String> command ;
command = new ArrayList<String>();
command.add(Shell.WINUTILS);
command.add("task");
command.add("createAsUser");
command.add("START_LOCALIZER_" + locId);
command.add(user);
command.add("nul:"); // PID file
//use same jvm as parent
File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java.exe");
File jvm = new File(
new File(System.getProperty("java.home"), "bin"), "java.exe");
command.add(jvm.toString());
Path cwdPath = new Path(cwdApp.getPath());
// Build a temp classpath jar. See ContainerLaunch.sanitizeEnv().
// Passing CLASSPATH explicitly is *way* too long for command line.
String classPath = System.getProperty("java.class.path");
Map<String, String> env = new HashMap<String, String>(System.getenv());
String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
String classPathJar = jarCp[0];
localizeClasspathJar(new Path(classPathJar), user);
String replacementClassPath = classPathJar + jarCp[1];
String jarCp[] = FileUtil.createJarWithClassPath(classPath,
classpathJarPrivateDir, cwdPath, env);
String classPathJar = localizeClasspathJar(
new Path(jarCp[0]), cwdPath, user).toString();
command.add("-classpath");
command.add(replacementClassPath);
command.add(classPathJar + jarCp[1]);
String javaLibPath = System.getProperty("java.library.path");
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
commandArray = command.toArray(new String[command.size()]);
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
localDirs);
shExec = new ShellCommandExecutor(
commandArray, cwdApp);
String cmdLine = StringUtils.join(command, " ");
shExec.execute();
String localizerPid = String.format(LOCALIZER_PID_FORMAT, locId);
WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor(
cwdApp.getAbsolutePath(),
localizerPid, user, "nul:", cmdLine);
try {
stubExecutor.execute();
stubExecutor.validateResult();
}
finally {
stubExecutor.close();
try
{
killContainer(localizerPid, Signal.KILL);
}
catch(Throwable e) {
LOG.warn(String.format(
"An exception occured during the cleanup of localizer job %s:\n%s",
localizerPid,
org.apache.hadoop.util.StringUtils.stringifyException(e)));
}
}
}
@Override
protected CommandExecutor buildCommandExecutor(String wrapperScriptPath,
String containerIdStr,
String userName, Path pidFile,File wordDir, Map<String, String> environment)
throws IOException {
return new WintuilsProcessStubExecutor(
wordDir.toString(),
containerIdStr, userName, pidFile.toString(),
"cmd /c " + wrapperScriptPath);
}
@Override
protected void killContainer(String pid, Signal signal) throws IOException {
Native.Elevated.killTask(pid);
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -212,7 +212,9 @@ public class ContainerLaunch implements Callable<Integer> {
+ Path.SEPARATOR
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
containerIdStr));
Path nmPrivateClasspathJarDir =
dirsHandler.getLocalPathForWrite(
getContainerPrivateDir(appIdStr, containerIdStr));
DataOutputStream containerScriptOutStream = null;
DataOutputStream tokensOutStream = null;
@ -263,7 +265,7 @@ public class ContainerLaunch implements Callable<Integer> {
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs,
localResources);
localResources, nmPrivateClasspathJarDir);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@ -658,7 +660,8 @@ public class ContainerLaunch implements Callable<Integer> {
public void sanitizeEnv(Map<String, String> environment, Path pwd,
List<Path> appDirs, List<String> containerLogDirs,
Map<Path, List<String>> resources) throws IOException {
Map<Path, List<String>> resources,
Path nmPrivateClasspathJarDir) throws IOException {
/**
* Non-modifiable environment variables
*/
@ -722,6 +725,7 @@ public class ContainerLaunch implements Callable<Integer> {
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
String inputClassPath = environment.get(Environment.CLASSPATH.name());
if (inputClassPath != null && !inputClassPath.isEmpty()) {
StringBuilder newClassPath = new StringBuilder(inputClassPath);
@ -765,11 +769,11 @@ public class ContainerLaunch implements Callable<Integer> {
mergedEnv.putAll(environment);
String[] jarCp = FileUtil.createJarWithClassPath(
newClassPath.toString(), pwd, mergedEnv);
String classPathJar = jarCp[0];
newClassPath.toString(), nmPrivateClasspathJarDir, pwd, mergedEnv);
// In a secure cluster the classpath jar must be localized to grant access
this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
String replacementClassPath = classPathJar + jarCp[1];
Path localizedClassPathJar = exec.localizeClasspathJar(
new Path(jarCp[0]), pwd, container.getUser());
String replacementClassPath = localizedClassPathJar.toString() + jarCp[1];
environment.put(Environment.CLASSPATH.name(), replacementClassPath);
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -369,10 +369,15 @@ public class ContainerLocalizer {
new ContainerLocalizer(FileContext.getLocalFSFileContext(), user,
appId, locId, localDirs,
RecordFactoryProvider.getRecordFactory(null));
System.exit(localizer.runLocalization(nmAddr));
int nRet = localizer.runLocalization(nmAddr);
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("nRet: %d", nRet));
}
System.exit(nRet);
} catch (Throwable e) {
// Print error to stdout so that LCE can use it.
e.printStackTrace(System.out);
LOG.error("Exception in main:", e);
throw e;
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -1088,7 +1088,8 @@ public class ResourceLocalizationService extends CompositeService
ConverterUtils.toString(
context.getContainerId().
getApplicationAttemptId().getApplicationId()),
localizerId, localDirs, logDirs);
localizerId,
dirsHandler);
} else {
throw new IOException("All disks failed. "
+ dirsHandler.getDisksHealthReport(false));

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
View File

@ -303,6 +303,7 @@ public class TestDefaultContainerExecutor {
public void testStartLocalizer()
throws IOException, InterruptedException {
InetSocketAddress localizationServerAddress;
final Path firstDir = new Path(BASE_TMP_PATH, "localDir1");
List<String> localDirs = new ArrayList<String>();
final Path secondDir = new Path(BASE_TMP_PATH, "localDir2");
@ -383,9 +384,14 @@ public class TestDefaultContainerExecutor {
String appSubmitter = "nobody";
String appId = "APP_ID";
String locId = "LOC_ID";
LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
when(dirsHandler.getLocalDirs()).thenReturn(localDirs);
when(dirsHandler.getLogDirs()).thenReturn(logDirs);
try {
mockExec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
appSubmitter, appId, locId, localDirs, logDirs);
appSubmitter, appId, locId, dirsHandler);
} catch (IOException e) {
Assert.fail("StartLocalizer failed to copy token file " + e);
} finally {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
View File

@ -266,7 +266,7 @@ public class TestLinuxContainerExecutor {
exec.setConf(conf);
exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
appId, locId, localDirs, logDirs);
appId, locId, dirsHandler);
String locId2 = "container_01_02";
Path nmPrivateContainerTokensPath2 =
@ -276,7 +276,7 @@ public class TestLinuxContainerExecutor {
+ String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
appId, locId2, localDirs, logDirs);
appId, locId2, dirsHandler);
}
@Test

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View File

@ -185,7 +185,7 @@ public class TestLinuxContainerExecutorWithMocks {
Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
try {
mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs());
mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
List<String> result=readMockParams();
Assert.assertEquals(result.size(), 17);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
View File

@ -850,7 +850,7 @@ public class TestResourceLocalizationService {
ArgumentCaptor<Path> tokenPathCaptor = ArgumentCaptor.forClass(Path.class);
verify(exec).startLocalizer(tokenPathCaptor.capture(),
isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr),
isA(List.class), isA(List.class));
isA(LocalDirsHandlerService.class));
Path localizationTokenPath = tokenPathCaptor.getValue();
// heartbeat from localizer

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm
View File

@ -83,11 +83,15 @@ min.user.id=1000#Prevent other super-users
+---+
** Windows Secure Container Executor
** Windows Secure Container Executor (WSCE)
The Windows environment secure container executor is the <<<WindowsSecureContainerExecutor>>>.
It uses the Windows S4U infrastructure to launch the container as the
YARN application user.
YARN application user. The WSCE requires the presense of the <<<hadoopwinutilsvc>>> service. This services
is hosted by <<<%HADOOP_HOME%\bin\winutils.exe>>> started with the <<<service>>> command line argument. This
service offers some privileged operations that require LocalSystem authority so that the NM is not required
to run the entire JVM and all the NM code in an elevated context. The NM interacts with the <<<hadoopwintulsvc>>>
service by means of Local RPC (LRPC) via calls JNI to the RCP client hosted in <<<hadoop.dll>>>.
*** Configuration
@ -102,17 +106,71 @@ min.user.id=1000#Prevent other super-users
<property>
<name>yarn.nodemanager.windows-secure-container-executor.group</name>
<value>hadoop</value>
<value>yarn</value>
</property>
+---+
*** wsce-site.xml
The NodeManager must run as a member of the local <<<Administrators>>> group or as
<<<LocalSystem>>>. It is not enough for the NodeManager to simply impersonate such an user.
The hadoopwinutilsvc uses <<<%HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations.
+---+
<property>
<name>yarn.nodemanager.windows-secure-container-executor.impersonate.allowed</name>
<value>HadoopUsers</value>
</property>
<property>
<name>yarn.nodemanager.windows-secure-container-executor.impersonate.denied</name>
<value>HadoopServices,Administrators</value>
</property>
<property>
<name>yarn.nodemanager.windows-secure-container-executor.allowed</name>
<value>nodemanager</value>
</property>
<property>
<name>yarn.nodemanager.windows-secure-container-executor.local-dirs</name>
<value>nm-local-dir, nm-log-dirs</value>
</property>
<property>
<name>yarn.nodemanager.windows-secure-container-executor.job-name</name>
<value>nodemanager-job-name</value>
</property>
+---+
<<<yarn.nodemanager.windows-secure-container-executor.allowed>>> should contain the name of the service account running the
nodemanager. This user will be allowed to access the hadoopwintuilsvc functions.
<<<yarn.nodemanager.windows-secure-container-executor.impersonate.allowed>>> should contain the users that are allowed to create
containers in the cluster. These users will be allowed to be impersonated by hadoopwinutilsvc.
<<<yarn.nodemanager.windows-secure-container-executor.impersonate.denied>>> should contain users that are explictly forbiden from
creating containers. hadoopwinutilsvc will refuse to impersonate these users.
<<<yarn.nodemanager.windows-secure-container-executor.local-dirs>>> should contain the nodemanager local dirs. hadoopwinutilsvc will
allow only file operations under these directories. This should contain the same values as <<<${yarn.nodemanager.local-dirs}, ${yarn.nodemanager.log-dirs}>>>
but note that hadoopwinutilsvc XML configuration processing does not do substitutions so the value must be the final value. All paths
must be absolute and no environment variable substitution will be performed. The paths are compared LOCAL_INVARIANT case insensitive string comparison,
the file path validated must start with one of the paths listed in local-dirs configuration. Use comma as path separator:<<<,>>>
<<<yarn.nodemanager.windows-secure-container-executor.job-name>>> should contain an Windows NT job name that all containers should be added to.
This configuration is optional. If not set, the container is not added to a global NodeManager job. Normally this should be set to the job that the NM is assigned to,
so that killing the NM kills also all containers. Hadoopwinutilsvc will not attempt to create this job, the job must exists when the container is launched.
If the value is set and the job does not exists, container launch will fail with error 2 <<<The system cannot find the file specified>>>.
Note that this global NM job is not related to the container job, which always gets created for each container and is named after the container ID.
This setting controls a global job that spans all containers and the parent NM, and as such it requires nested jobs.
Nested jobs are available only post Windows 8 and Windows Server 2012.
*** Useful Links
* {{{http://msdn.microsoft.com/en-us/magazine/cc188757.aspx}Exploring S4U Kerberos Extensions in Windows Server 2003}}
* {{{http://msdn.microsoft.com/en-us/library/windows/desktop/hh448388(v=vs.85).aspx}Nested Jobs}}
* {{{https://issues.apache.org/jira/browse/YARN-1063}Winutils needs ability to create task as domain user}}
* {{{https://issues.apache.org/jira/browse/YARN-1972}Implement secure Windows Container Executor}}
* {{{https://issues.apache.org/jira/browse/YARN-2198}Remove the need to run NodeManager as privileged account for Windows Secure Container Executor}}