HADOOP-11996. Improve and restructure native ISAL support (Kai Zheng via cmccabe)
This commit is contained in:
Colin Patrick Mccabe
parent
6e9a582eb1
commit
658ee95ff3
|
|
@ -103,12 +103,24 @@ find_library(ISAL_LIBRARY
|
||||||
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
|
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
|
||||||
if (ISAL_LIBRARY)
|
if (ISAL_LIBRARY)
|
||||||
GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME)
|
GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME)
|
||||||
set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode/include)
|
set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode)
|
||||||
set(ISAL_SOURCE_FILES
|
set(ISAL_SOURCE_FILES
|
||||||
${SRC}/io/erasurecode/coder/erasure_code_native.c
|
${SRC}/io/erasurecode/isal_load.c
|
||||||
${SRC}/io/erasurecode/erasure_code.c)
|
|
||||||
add_executable(erasure_code_test
|
|
||||||
${SRC}/io/erasurecode/erasure_code.c
|
${SRC}/io/erasurecode/erasure_code.c
|
||||||
|
${SRC}/io/erasurecode/gf_util.c
|
||||||
|
${SRC}/io/erasurecode/dump.c
|
||||||
|
${SRC}/io/erasurecode/erasure_coder.c
|
||||||
|
${SRC}/io/erasurecode/jni_erasure_code_native.c
|
||||||
|
${SRC}/io/erasurecode/jni_common.c
|
||||||
|
${SRC}/io/erasurecode/jni_rs_encoder.c
|
||||||
|
${SRC}/io/erasurecode/jni_rs_decoder.c)
|
||||||
|
|
||||||
|
add_executable(erasure_code_test
|
||||||
|
${SRC}/io/erasurecode/isal_load.c
|
||||||
|
${SRC}/io/erasurecode/erasure_code.c
|
||||||
|
${SRC}/io/erasurecode/gf_util.c
|
||||||
|
${SRC}/io/erasurecode/dump.c
|
||||||
|
${SRC}/io/erasurecode/erasure_coder.c
|
||||||
${TST}/io/erasurecode/erasure_code_test.c
|
${TST}/io/erasurecode/erasure_code_test.c
|
||||||
)
|
)
|
||||||
target_link_libraries(erasure_code_test ${CMAKE_DL_LIBS})
|
target_link_libraries(erasure_code_test ${CMAKE_DL_LIBS})
|
||||||
|
|
|
||||||
|
|
@ -162,10 +162,17 @@
|
||||||
<ClCompile Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.c">
|
<ClCompile Include="src\org\apache\hadoop\yarn\server\nodemanager\windows_secure_container_executor.c">
|
||||||
<AdditionalIncludeDirectories>src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
<AdditionalIncludeDirectories>src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_code.c" Condition="'$(IsalEnabled)' == 'true'">
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\isal_load.c" Condition="'$(IsalEnabled)' == 'true'">
|
||||||
<AdditionalOptions>/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"</AdditionalOptions>
|
<AdditionalOptions>/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"</AdditionalOptions>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\coder\erasure_code_native.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_code.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\gf_util.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\erasure_coder.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\dump.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_erasure_code_native.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_common.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_rs_encoder.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
|
<ClCompile Include="src\org\apache\hadoop\io\erasurecode\jni_rs_decoder.c" Condition="'$(IsalEnabled)' == 'true'"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" />
|
<ClInclude Include="..\src\org\apache\hadoop\util\crc32c_tables.h" />
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "erasure_code.h"
|
||||||
|
#include "gf_util.h"
|
||||||
|
#include "erasure_coder.h"
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
void dump(unsigned char* buf, int len) {
|
||||||
|
int i;
|
||||||
|
for (i = 0; i < len;) {
|
||||||
|
printf(" %2x", 0xff & buf[i++]);
|
||||||
|
if (i % 32 == 0)
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void dumpMatrix(unsigned char** buf, int n1, int n2) {
|
||||||
|
int i, j;
|
||||||
|
for (i = 0; i < n1; i++) {
|
||||||
|
for (j = 0; j < n2; j++) {
|
||||||
|
printf(" %2x", buf[i][j]);
|
||||||
|
}
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
void dumpCodingMatrix(unsigned char* buf, int n1, int n2) {
|
||||||
|
int i, j;
|
||||||
|
for (i = 0; i < n1; i++) {
|
||||||
|
for (j = 0; j < n2; j++) {
|
||||||
|
printf(" %d", 0xff & buf[j + (i * n2)]);
|
||||||
|
}
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
printf("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
void dumpEncoder(IsalEncoder* pCoder) {
|
||||||
|
int numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
int numParityUnits = pCoder->coder.numDataUnits;
|
||||||
|
int numAllUnits = pCoder->coder.numAllUnits;
|
||||||
|
|
||||||
|
printf("Encoding (numAlnumParityUnitslUnits = %d, numDataUnits = %d)\n",
|
||||||
|
numParityUnits, numDataUnits);
|
||||||
|
|
||||||
|
printf("\n\nEncodeMatrix:\n");
|
||||||
|
dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix,
|
||||||
|
numDataUnits, numAllUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
void dumpDecoder(IsalDecoder* pCoder) {
|
||||||
|
int i, j;
|
||||||
|
int numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
int numAllUnits = pCoder->coder.numAllUnits;
|
||||||
|
|
||||||
|
printf("Recovering (numAllUnits = %d, numDataUnits = %d, numErased = %d)\n",
|
||||||
|
numAllUnits, numDataUnits, pCoder->numErased);
|
||||||
|
|
||||||
|
printf(" - ErasedIndexes = ");
|
||||||
|
for (j = 0; j < pCoder->numErased; j++) {
|
||||||
|
printf(" %d", pCoder->erasedIndexes[j]);
|
||||||
|
}
|
||||||
|
printf(" - DecodeIndex = ");
|
||||||
|
for (i = 0; i < numDataUnits; i++) {
|
||||||
|
printf(" %d", pCoder->decodeIndex[i]);
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("\n\nEncodeMatrix:\n");
|
||||||
|
dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix,
|
||||||
|
numDataUnits, numAllUnits);
|
||||||
|
|
||||||
|
printf("InvertMatrix:\n");
|
||||||
|
dumpCodingMatrix((unsigned char*) pCoder->invertMatrix,
|
||||||
|
numDataUnits, numDataUnits);
|
||||||
|
|
||||||
|
printf("DecodeMatrix:\n");
|
||||||
|
dumpCodingMatrix((unsigned char*) pCoder->decodeMatrix,
|
||||||
|
numDataUnits, numAllUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dump utilities for erasure coders.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef _DUMP_H_
|
||||||
|
#define _DUMP_H_
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
void dumpEncoder(IsalEncoder* pCoder);
|
||||||
|
|
||||||
|
void dumpDecoder(IsalDecoder* pCoder);
|
||||||
|
|
||||||
|
void dump(unsigned char* buf, int len);
|
||||||
|
|
||||||
|
void dumpMatrix(unsigned char** s, int k, int m);
|
||||||
|
|
||||||
|
void dumpCodingMatrix(unsigned char* s, int n1, int n2);
|
||||||
|
|
||||||
|
#endif //_DUMP_H_
|
||||||
|
|
@ -20,252 +20,25 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include "org_apache_hadoop.h"
|
#include "isal_load.h"
|
||||||
#include "../include/gf_util.h"
|
#include "erasure_code.h"
|
||||||
#include "../include/erasure_code.h"
|
|
||||||
|
|
||||||
#ifdef UNIX
|
|
||||||
#include <sys/time.h>
|
|
||||||
#include <sys/types.h>
|
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <dlfcn.h>
|
|
||||||
|
|
||||||
#include "config.h"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef WINDOWS
|
|
||||||
#include <Windows.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* erasure_code.c
|
* erasure_code.c
|
||||||
* Implementation erasure code utilities based on lib of erasure_code.so.
|
* Implementation erasure code utilities based on ISA-L library.
|
||||||
* Building of this codes won't rely on any ISA-L source codes, but running
|
|
||||||
* into this will rely on successfully loading of the dynamic library.
|
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
* The loaded library handle.
|
|
||||||
*/
|
|
||||||
static void* libec = NULL;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* A helper function to dlsym a 'symbol' from a given library-handle.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#ifdef UNIX
|
|
||||||
|
|
||||||
static __attribute__ ((unused))
|
|
||||||
void *my_dlsym(void *handle, const char *symbol) {
|
|
||||||
void *func_ptr = dlsym(handle, symbol);
|
|
||||||
return func_ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
|
|
||||||
#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, handle, symbol) \
|
|
||||||
if ((func_ptr = my_dlsym(handle, symbol)) == NULL) { \
|
|
||||||
return "Failed to load symbol" symbol; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef WINDOWS
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static FARPROC WINAPI my_dlsym(HMODULE handle, LPCSTR symbol) {
|
|
||||||
FARPROC func_ptr = GetProcAddress(handle, symbol);
|
|
||||||
return func_ptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
|
|
||||||
#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, handle, symbol) \
|
|
||||||
if ((func_ptr = (func_type)my_dlsym(handle, symbol)) == NULL) { \
|
|
||||||
return "Failed to load symbol" symbol; \
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
|
|
||||||
#ifdef UNIX
|
|
||||||
// For gf_util.h
|
|
||||||
static unsigned char (*d_gf_mul)(unsigned char, unsigned char);
|
|
||||||
static unsigned char (*d_gf_inv)(unsigned char);
|
|
||||||
static void (*d_gf_gen_rs_matrix)(unsigned char *, int, int);
|
|
||||||
static void (*d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
|
|
||||||
static int (*d_gf_invert_matrix)(unsigned char *, unsigned char *, const int);
|
|
||||||
static int (*d_gf_vect_mul)(int, unsigned char *, void *, void *);
|
|
||||||
|
|
||||||
// For erasure_code.h
|
|
||||||
static void (*d_ec_init_tables)(int, int, unsigned char*, unsigned char*);
|
|
||||||
static void (*d_ec_encode_data)(int, int, int, unsigned char*,
|
|
||||||
unsigned char**, unsigned char**);
|
|
||||||
static void (*d_ec_encode_data_update)(int, int, int, int, unsigned char*,
|
|
||||||
unsigned char*, unsigned char**);
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef WINDOWS
|
|
||||||
// For erasure_code.h
|
|
||||||
typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char);
|
|
||||||
static __d_gf_mul d_gf_mul;
|
|
||||||
typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char);
|
|
||||||
static __d_gf_inv d_gf_inv;
|
|
||||||
typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int);
|
|
||||||
static __d_gf_gen_rs_matrix d_gf_gen_rs_matrix;
|
|
||||||
typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
|
|
||||||
static __d_gf_gen_cauchy_matrix d_gf_gen_cauchy_matrix;
|
|
||||||
typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *,
|
|
||||||
unsigned char *, const int);
|
|
||||||
static __d_gf_invert_matrix d_gf_invert_matrix;
|
|
||||||
typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *);
|
|
||||||
static __d_gf_vect_mul d_gf_vect_mul;
|
|
||||||
|
|
||||||
// For erasure_code.h
|
|
||||||
typedef void (__cdecl *__d_ec_init_tables)(int, int,
|
|
||||||
unsigned char*, unsigned char*);
|
|
||||||
static __d_ec_init_tables d_ec_init_tables;
|
|
||||||
typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*,
|
|
||||||
unsigned char**, unsigned char**);
|
|
||||||
static __d_ec_encode_data d_ec_encode_data;
|
|
||||||
typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
|
|
||||||
unsigned char*, unsigned char**);
|
|
||||||
static __d_ec_encode_data_update d_ec_encode_data_update;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static const char* load_functions(void* libec) {
|
|
||||||
#ifdef UNIX
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_mul, libec, "gf_mul");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_inv, libec, "gf_inv");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_invert_matrix, libec, "gf_invert_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_gf_vect_mul, libec, "gf_vect_mul");
|
|
||||||
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_ec_init_tables, libec, "ec_init_tables");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data, libec, "ec_encode_data");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data_update, libec, "ec_encode_data_update");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef WINDOWS
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, d_gf_mul, libec, "gf_mul");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, d_gf_inv, libec, "gf_inv");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, d_gf_invert_matrix, libec, "gf_invert_matrix");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, d_gf_vect_mul, libec, "gf_vect_mul");
|
|
||||||
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, d_ec_init_tables, libec, "ec_init_tables");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, d_ec_encode_data, libec, "ec_encode_data");
|
|
||||||
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, d_ec_encode_data_update, libec, "ec_encode_data_update");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
void load_erasurecode_lib(char* err, size_t err_len) {
|
|
||||||
const char* errMsg;
|
|
||||||
|
|
||||||
err[0] = '\0';
|
|
||||||
|
|
||||||
if (libec != NULL) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load Intel ISA-L
|
|
||||||
#ifdef UNIX
|
|
||||||
libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
|
|
||||||
if (libec == NULL) {
|
|
||||||
snprintf(err, err_len, "Failed to load %s (%s)",
|
|
||||||
HADOOP_ISAL_LIBRARY, dlerror());
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Clear any existing error
|
|
||||||
dlerror();
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#ifdef WINDOWS
|
|
||||||
libec = LoadLibrary(HADOOP_ISAL_LIBRARY);
|
|
||||||
if (libec == NULL) {
|
|
||||||
snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
errMsg = load_functions(libec);
|
|
||||||
if (errMsg != NULL) {
|
|
||||||
snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
int build_support_erasurecode() {
|
|
||||||
#ifdef HADOOP_ISAL_LIBRARY
|
|
||||||
return 1;
|
|
||||||
#else
|
|
||||||
return 0;
|
|
||||||
#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
const char* get_library_name() {
|
|
||||||
#ifdef UNIX
|
|
||||||
Dl_info dl_info;
|
|
||||||
|
|
||||||
if (d_ec_encode_data == NULL) {
|
|
||||||
return HADOOP_ISAL_LIBRARY;
|
|
||||||
}
|
|
||||||
|
|
||||||
if(dladdr(d_ec_encode_data, &dl_info)) {
|
|
||||||
return dl_info.dli_fname;
|
|
||||||
}
|
|
||||||
#else
|
|
||||||
LPTSTR filename = NULL;
|
|
||||||
|
|
||||||
if (libec == NULL) {
|
|
||||||
return HADOOP_ISAL_LIBRARY;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (GetModuleFileName(libec, filename, 256) > 0) {
|
|
||||||
return filename;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned char h_gf_mul(unsigned char a, unsigned char b) {
|
|
||||||
return d_gf_mul(a, b);
|
|
||||||
}
|
|
||||||
|
|
||||||
unsigned char h_gf_inv(unsigned char a) {
|
|
||||||
return d_gf_inv(a);
|
|
||||||
}
|
|
||||||
|
|
||||||
void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) {
|
|
||||||
d_gf_gen_rs_matrix(a, m, k);
|
|
||||||
}
|
|
||||||
|
|
||||||
void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) {
|
|
||||||
d_gf_gen_cauchy_matrix(a, m, k);
|
|
||||||
}
|
|
||||||
|
|
||||||
int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) {
|
|
||||||
return d_gf_invert_matrix(in, out, n);
|
|
||||||
}
|
|
||||||
|
|
||||||
int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) {
|
|
||||||
return d_gf_vect_mul(len, gftbl, src, dest);
|
|
||||||
}
|
|
||||||
|
|
||||||
void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls) {
|
void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls) {
|
||||||
d_ec_init_tables(k, rows, a, gftbls);
|
isaLoader->ec_init_tables(k, rows, a, gftbls);
|
||||||
}
|
}
|
||||||
|
|
||||||
void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls,
|
void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls,
|
||||||
unsigned char **data, unsigned char **coding) {
|
unsigned char **data, unsigned char **coding) {
|
||||||
d_ec_encode_data(len, k, rows, gftbls, data, coding);
|
isaLoader->ec_encode_data(len, k, rows, gftbls, data, coding);
|
||||||
}
|
}
|
||||||
|
|
||||||
void h_ec_encode_data_update(int len, int k, int rows, int vec_i,
|
void h_ec_encode_data_update(int len, int k, int rows, int vec_i,
|
||||||
unsigned char *gftbls, unsigned char *data, unsigned char **coding) {
|
unsigned char *gftbls, unsigned char *data, unsigned char **coding) {
|
||||||
d_ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding);
|
isaLoader->ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding);
|
||||||
}
|
}
|
||||||
|
|
@ -37,24 +37,6 @@
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
|
||||||
* Return 0 if not support, 1 otherwise.
|
|
||||||
*/
|
|
||||||
int build_support_erasurecode();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get the library name possibly of full path.
|
|
||||||
*/
|
|
||||||
const char* get_library_name();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Initialize and load erasure code library, returning error message if any.
|
|
||||||
*
|
|
||||||
* @param err The err message buffer.
|
|
||||||
* @param err_len The length of the message buffer.
|
|
||||||
*/
|
|
||||||
void load_erasurecode_lib(char* err, size_t err_len);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Initialize tables for fast Erasure Code encode and decode.
|
* Initialize tables for fast Erasure Code encode and decode.
|
||||||
*
|
*
|
||||||
|
|
@ -0,0 +1,229 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "erasure_code.h"
|
||||||
|
#include "gf_util.h"
|
||||||
|
#include "erasure_coder.h"
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits) {
|
||||||
|
pCoder->verbose = 0;
|
||||||
|
pCoder->numParityUnits = numParityUnits;
|
||||||
|
pCoder->numDataUnits = numDataUnits;
|
||||||
|
pCoder->numAllUnits = numDataUnits + numParityUnits;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 0 not to verbose, 1 to verbose
|
||||||
|
void allowVerbose(IsalCoder* pCoder, int flag) {
|
||||||
|
pCoder->verbose = flag;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void initEncodeMatrix(int numDataUnits, int numParityUnits,
|
||||||
|
unsigned char* encodeMatrix) {
|
||||||
|
// Generate encode matrix, always invertible
|
||||||
|
h_gf_gen_cauchy_matrix(encodeMatrix,
|
||||||
|
numDataUnits + numParityUnits, numDataUnits);
|
||||||
|
}
|
||||||
|
|
||||||
|
void initEncoder(IsalEncoder* pCoder, int numDataUnits,
|
||||||
|
int numParityUnits) {
|
||||||
|
initCoder(&pCoder->coder, numDataUnits, numParityUnits);
|
||||||
|
|
||||||
|
initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix);
|
||||||
|
|
||||||
|
// Generate gftbls from encode matrix
|
||||||
|
h_ec_init_tables(numDataUnits, numParityUnits,
|
||||||
|
&pCoder->encodeMatrix[numDataUnits * numDataUnits],
|
||||||
|
pCoder->gftbls);
|
||||||
|
|
||||||
|
if (pCoder->coder.verbose > 0) {
|
||||||
|
dumpEncoder(pCoder);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void initDecoder(IsalDecoder* pCoder, int numDataUnits,
|
||||||
|
int numParityUnits) {
|
||||||
|
initCoder(&pCoder->coder, numDataUnits, numParityUnits);
|
||||||
|
|
||||||
|
initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix);
|
||||||
|
}
|
||||||
|
|
||||||
|
int encode(IsalEncoder* pCoder, unsigned char** dataUnits,
|
||||||
|
unsigned char** parityUnits, int chunkSize) {
|
||||||
|
int numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
int numParityUnits = pCoder->coder.numParityUnits;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
for (i = 0; i < numParityUnits; i++) {
|
||||||
|
memset(parityUnits[i], 0, chunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
h_ec_encode_data(chunkSize, numDataUnits, numParityUnits,
|
||||||
|
pCoder->gftbls, dataUnits, parityUnits);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return 1 when diff, 0 otherwise
|
||||||
|
static int compare(int* arr1, int len1, int* arr2, int len2) {
|
||||||
|
int i;
|
||||||
|
|
||||||
|
if (len1 == len2) {
|
||||||
|
for (i = 0; i < len1; i++) {
|
||||||
|
if (arr1[i] != arr2[i]) {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int processErasures(IsalDecoder* pCoder, unsigned char** inputs,
|
||||||
|
int* erasedIndexes, int numErased) {
|
||||||
|
int i, r, ret, index;
|
||||||
|
int numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
int isChanged = 0;
|
||||||
|
|
||||||
|
for (i = 0, r = 0; i < numDataUnits; i++, r++) {
|
||||||
|
while (inputs[r] == NULL) {
|
||||||
|
r++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pCoder->decodeIndex[i] != r) {
|
||||||
|
pCoder->decodeIndex[i] = r;
|
||||||
|
isChanged = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (i = 0; i < numDataUnits; i++) {
|
||||||
|
pCoder->realInputs[i] = inputs[pCoder->decodeIndex[i]];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isChanged == 0 &&
|
||||||
|
compare(pCoder->erasedIndexes, pCoder->numErased,
|
||||||
|
erasedIndexes, numErased) == 0) {
|
||||||
|
return 0; // Optimization, nothing to do
|
||||||
|
}
|
||||||
|
|
||||||
|
clearDecoder(pCoder);
|
||||||
|
|
||||||
|
for (i = 0; i < numErased; i++) {
|
||||||
|
index = erasedIndexes[i];
|
||||||
|
pCoder->erasedIndexes[i] = index;
|
||||||
|
pCoder->erasureFlags[index] = 1;
|
||||||
|
if (index < numDataUnits) {
|
||||||
|
pCoder->numErasedDataUnits++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pCoder->numErased = numErased;
|
||||||
|
|
||||||
|
ret = generateDecodeMatrix(pCoder);
|
||||||
|
if (ret != 0) {
|
||||||
|
printf("Failed to generate decode matrix\n");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
h_ec_init_tables(numDataUnits, pCoder->numErased,
|
||||||
|
pCoder->decodeMatrix, pCoder->gftbls);
|
||||||
|
|
||||||
|
if (pCoder->coder.verbose > 0) {
|
||||||
|
dumpDecoder(pCoder);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int decode(IsalDecoder* pCoder, unsigned char** inputs,
|
||||||
|
int* erasedIndexes, int numErased,
|
||||||
|
unsigned char** outputs, int chunkSize) {
|
||||||
|
int numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
processErasures(pCoder, inputs, erasedIndexes, numErased);
|
||||||
|
|
||||||
|
for (i = 0; i < numErased; i++) {
|
||||||
|
memset(outputs[i], 0, chunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
h_ec_encode_data(chunkSize, numDataUnits, pCoder->numErased,
|
||||||
|
pCoder->gftbls, pCoder->realInputs, outputs);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear variables used per decode call
|
||||||
|
void clearDecoder(IsalDecoder* decoder) {
|
||||||
|
decoder->numErasedDataUnits = 0;
|
||||||
|
decoder->numErased = 0;
|
||||||
|
memset(decoder->gftbls, 0, sizeof(decoder->gftbls));
|
||||||
|
memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix));
|
||||||
|
memset(decoder->tmpMatrix, 0, sizeof(decoder->tmpMatrix));
|
||||||
|
memset(decoder->invertMatrix, 0, sizeof(decoder->invertMatrix));
|
||||||
|
memset(decoder->erasureFlags, 0, sizeof(decoder->erasureFlags));
|
||||||
|
memset(decoder->erasedIndexes, 0, sizeof(decoder->erasedIndexes));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate decode matrix from encode matrix
|
||||||
|
int generateDecodeMatrix(IsalDecoder* pCoder) {
|
||||||
|
int i, j, r, p;
|
||||||
|
unsigned char s;
|
||||||
|
int numDataUnits;
|
||||||
|
|
||||||
|
numDataUnits = pCoder->coder.numDataUnits;
|
||||||
|
|
||||||
|
// Construct matrix b by removing error rows
|
||||||
|
for (i = 0; i < numDataUnits; i++) {
|
||||||
|
r = pCoder->decodeIndex[i];
|
||||||
|
for (j = 0; j < numDataUnits; j++) {
|
||||||
|
pCoder->tmpMatrix[numDataUnits * i + j] =
|
||||||
|
pCoder->encodeMatrix[numDataUnits * r + j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
h_gf_invert_matrix(pCoder->tmpMatrix,
|
||||||
|
pCoder->invertMatrix, numDataUnits);
|
||||||
|
|
||||||
|
for (i = 0; i < pCoder->numErasedDataUnits; i++) {
|
||||||
|
for (j = 0; j < numDataUnits; j++) {
|
||||||
|
pCoder->decodeMatrix[numDataUnits * i + j] =
|
||||||
|
pCoder->invertMatrix[numDataUnits *
|
||||||
|
pCoder->erasedIndexes[i] + j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) {
|
||||||
|
for (i = 0; i < numDataUnits; i++) {
|
||||||
|
s = 0;
|
||||||
|
for (j = 0; j < numDataUnits; j++) {
|
||||||
|
s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
|
||||||
|
pCoder->encodeMatrix[numDataUnits *
|
||||||
|
pCoder->erasedIndexes[p] + j]);
|
||||||
|
}
|
||||||
|
|
||||||
|
pCoder->decodeMatrix[numDataUnits * p + i] = s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is a sample program illustrating how to use the Intel ISA-L library.
|
||||||
|
* Note it's adapted from erasure_code_test.c test program, but trying to use
|
||||||
|
* variable names and styles we're more familiar with already similar to Java
|
||||||
|
* coders.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef _ERASURE_CODER_H_
|
||||||
|
#define _ERASURE_CODER_H_
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#define MMAX 14
|
||||||
|
#define KMAX 10
|
||||||
|
|
||||||
|
typedef struct _IsalCoder {
|
||||||
|
int verbose;
|
||||||
|
int numParityUnits;
|
||||||
|
int numDataUnits;
|
||||||
|
int numAllUnits;
|
||||||
|
} IsalCoder;
|
||||||
|
|
||||||
|
typedef struct _IsalEncoder {
|
||||||
|
IsalCoder coder;
|
||||||
|
|
||||||
|
unsigned char gftbls[MMAX * KMAX * 32];
|
||||||
|
|
||||||
|
unsigned char encodeMatrix[MMAX * KMAX];
|
||||||
|
} IsalEncoder;
|
||||||
|
|
||||||
|
typedef struct _IsalDecoder {
|
||||||
|
IsalCoder coder;
|
||||||
|
|
||||||
|
unsigned char encodeMatrix[MMAX * KMAX];
|
||||||
|
|
||||||
|
// Below are per decode call
|
||||||
|
unsigned char gftbls[MMAX * KMAX * 32];
|
||||||
|
unsigned int decodeIndex[MMAX];
|
||||||
|
unsigned char tmpMatrix[MMAX * KMAX];
|
||||||
|
unsigned char invertMatrix[MMAX * KMAX];
|
||||||
|
unsigned char decodeMatrix[MMAX * KMAX];
|
||||||
|
unsigned char erasureFlags[MMAX];
|
||||||
|
int erasedIndexes[MMAX];
|
||||||
|
int numErased;
|
||||||
|
int numErasedDataUnits;
|
||||||
|
unsigned char* realInputs[MMAX];
|
||||||
|
} IsalDecoder;
|
||||||
|
|
||||||
|
void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits);
|
||||||
|
|
||||||
|
void allowVerbose(IsalCoder* pCoder, int flag);
|
||||||
|
|
||||||
|
void initEncoder(IsalEncoder* encoder, int numDataUnits, int numParityUnits);
|
||||||
|
|
||||||
|
void initDecoder(IsalDecoder* decoder, int numDataUnits, int numParityUnits);
|
||||||
|
|
||||||
|
void clearDecoder(IsalDecoder* decoder);
|
||||||
|
|
||||||
|
int encode(IsalEncoder* encoder, unsigned char** dataUnits,
|
||||||
|
unsigned char** parityUnits, int chunkSize);
|
||||||
|
|
||||||
|
int decode(IsalDecoder* decoder, unsigned char** allUnits,
|
||||||
|
int* erasedIndexes, int numErased,
|
||||||
|
unsigned char** recoveredUnits, int chunkSize);
|
||||||
|
|
||||||
|
int generateDecodeMatrix(IsalDecoder* pCoder);
|
||||||
|
|
||||||
|
#endif //_ERASURE_CODER_H_
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "isal_load.h"
|
||||||
|
#include "gf_util.h"
|
||||||
|
|
||||||
|
/**
|
||||||
|
* gf_util.c
|
||||||
|
* Implementation GF utilities based on ISA-L library.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
unsigned char h_gf_mul(unsigned char a, unsigned char b) {
|
||||||
|
return isaLoader->gf_mul(a, b);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned char h_gf_inv(unsigned char a) {
|
||||||
|
return isaLoader->gf_inv(a);
|
||||||
|
}
|
||||||
|
|
||||||
|
void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) {
|
||||||
|
isaLoader->gf_gen_rs_matrix(a, m, k);
|
||||||
|
}
|
||||||
|
|
||||||
|
void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) {
|
||||||
|
isaLoader->gf_gen_cauchy_matrix(a, m, k);
|
||||||
|
}
|
||||||
|
|
||||||
|
int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) {
|
||||||
|
return isaLoader->gf_invert_matrix(in, out, n);
|
||||||
|
}
|
||||||
|
|
||||||
|
int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) {
|
||||||
|
return isaLoader->gf_vect_mul(len, gftbl, src, dest);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,148 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
#include "isal_load.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <dlfcn.h>
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
#include <Windows.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
IsaLibLoader* isaLoader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* isal_load.c
|
||||||
|
* Utility of loading the ISA-L library and the required functions.
|
||||||
|
* Building of this codes won't rely on any ISA-L source codes, but running
|
||||||
|
* into this will rely on successfully loading of the dynamic library.
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
|
||||||
|
static const char* load_functions() {
|
||||||
|
#ifdef UNIX
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_mul), "gf_mul");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_inv), "gf_inv");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_invert_matrix), "gf_invert_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_vect_mul), "gf_vect_mul");
|
||||||
|
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_init_tables), "ec_init_tables");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data), "ec_encode_data");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data_update), "ec_encode_data_update");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, (isaLoader->gf_mul), "gf_mul");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, (isaLoader->gf_inv), "gf_inv");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, (isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, (isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, (isaLoader->gf_invert_matrix), "gf_invert_matrix");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, (isaLoader->gf_vect_mul), "gf_vect_mul");
|
||||||
|
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, (isaLoader->ec_init_tables), "ec_init_tables");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, (isaLoader->ec_encode_data), "ec_encode_data");
|
||||||
|
EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, (isaLoader->ec_encode_data_update), "ec_encode_data_update");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
void load_erasurecode_lib(char* err, size_t err_len) {
|
||||||
|
const char* errMsg;
|
||||||
|
|
||||||
|
err[0] = '\0';
|
||||||
|
|
||||||
|
if (isaLoader != NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
isaLoader = calloc(1, sizeof(IsaLibLoader));
|
||||||
|
memset(isaLoader, 0, sizeof(IsaLibLoader));
|
||||||
|
|
||||||
|
// Load Intel ISA-L
|
||||||
|
#ifdef UNIX
|
||||||
|
isaLoader->libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
|
||||||
|
if (isaLoader->libec == NULL) {
|
||||||
|
snprintf(err, err_len, "Failed to load %s (%s)",
|
||||||
|
HADOOP_ISAL_LIBRARY, dlerror());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Clear any existing error
|
||||||
|
dlerror();
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
isaLoader->libec = LoadLibrary(HADOOP_ISAL_LIBRARY);
|
||||||
|
if (isaLoader->libec == NULL) {
|
||||||
|
snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
errMsg = load_functions(isaLoader->libec);
|
||||||
|
if (errMsg != NULL) {
|
||||||
|
snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int build_support_erasurecode() {
|
||||||
|
#ifdef HADOOP_ISAL_LIBRARY
|
||||||
|
return 1;
|
||||||
|
#else
|
||||||
|
return 0;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* get_library_name() {
|
||||||
|
#ifdef UNIX
|
||||||
|
Dl_info dl_info;
|
||||||
|
|
||||||
|
if (isaLoader->ec_encode_data == NULL) {
|
||||||
|
return HADOOP_ISAL_LIBRARY;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dladdr(isaLoader->ec_encode_data, &dl_info)) {
|
||||||
|
return dl_info.dli_fname;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
LPTSTR filename = NULL;
|
||||||
|
|
||||||
|
if (isaLoader->libec == NULL) {
|
||||||
|
return HADOOP_ISAL_LIBRARY;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (GetModuleFileName(isaLoader->libec, filename, 256) > 0) {
|
||||||
|
return filename;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,149 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <dlfcn.h>
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
#include <Windows.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifndef _ISAL_LOAD_H_
|
||||||
|
#define _ISAL_LOAD_H_
|
||||||
|
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
// For gf_util.h
|
||||||
|
typedef unsigned char (*__d_gf_mul)(unsigned char, unsigned char);
|
||||||
|
typedef unsigned char (*__d_gf_inv)(unsigned char);
|
||||||
|
typedef void (*__d_gf_gen_rs_matrix)(unsigned char *, int, int);
|
||||||
|
typedef void (*__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
|
||||||
|
typedef int (*__d_gf_invert_matrix)(unsigned char *, unsigned char *, const int);
|
||||||
|
typedef int (*__d_gf_vect_mul)(int, unsigned char *, void *, void *);
|
||||||
|
|
||||||
|
// For erasure_code.h
|
||||||
|
typedef void (*__d_ec_init_tables)(int, int, unsigned char*, unsigned char*);
|
||||||
|
typedef void (*__d_ec_encode_data)(int, int, int, unsigned char*,
|
||||||
|
unsigned char**, unsigned char**);
|
||||||
|
typedef void (*__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
|
||||||
|
unsigned char*, unsigned char**);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
// For erasure_code.h
|
||||||
|
typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char);
|
||||||
|
typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char);
|
||||||
|
typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int);
|
||||||
|
typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
|
||||||
|
typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *,
|
||||||
|
unsigned char *, const int);
|
||||||
|
typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *);
|
||||||
|
|
||||||
|
// For erasure_code.h
|
||||||
|
typedef void (__cdecl *__d_ec_init_tables)(int, int,
|
||||||
|
unsigned char*, unsigned char*);
|
||||||
|
typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*,
|
||||||
|
unsigned char**, unsigned char**);
|
||||||
|
typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
|
||||||
|
unsigned char*, unsigned char**);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
typedef struct __IsaLibLoader {
|
||||||
|
// The loaded library handle
|
||||||
|
void* libec;
|
||||||
|
|
||||||
|
__d_gf_mul gf_mul;
|
||||||
|
__d_gf_inv gf_inv;
|
||||||
|
__d_gf_gen_rs_matrix gf_gen_rs_matrix;
|
||||||
|
__d_gf_gen_cauchy_matrix gf_gen_cauchy_matrix;
|
||||||
|
__d_gf_invert_matrix gf_invert_matrix;
|
||||||
|
__d_gf_vect_mul gf_vect_mul;
|
||||||
|
__d_ec_init_tables ec_init_tables;
|
||||||
|
__d_ec_encode_data ec_encode_data;
|
||||||
|
__d_ec_encode_data_update ec_encode_data_update;
|
||||||
|
} IsaLibLoader;
|
||||||
|
|
||||||
|
extern IsaLibLoader* isaLoader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A helper function to dlsym a 'symbol' from a given library-handle.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifdef UNIX
|
||||||
|
|
||||||
|
static __attribute__ ((unused))
|
||||||
|
void *myDlsym(void *handle, const char *symbol) {
|
||||||
|
void *func_ptr = dlsym(handle, symbol);
|
||||||
|
return func_ptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
|
||||||
|
#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, symbol) \
|
||||||
|
if ((func_ptr = myDlsym(isaLoader->libec, symbol)) == NULL) { \
|
||||||
|
return "Failed to load symbol" symbol; \
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#ifdef WINDOWS
|
||||||
|
|
||||||
|
|
||||||
|
static FARPROC WINAPI myDlsym(HMODULE handle, LPCSTR symbol) {
|
||||||
|
FARPROC func_ptr = GetProcAddress(handle, symbol);
|
||||||
|
return func_ptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
|
||||||
|
#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, symbol) \
|
||||||
|
if ((func_ptr = (func_type)myDlsym(isaLoader->libec, symbol)) == NULL) { \
|
||||||
|
return "Failed to load symbol" symbol; \
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return 0 if not support, 1 otherwise.
|
||||||
|
*/
|
||||||
|
int build_support_erasurecode();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the library name possibly of full path.
|
||||||
|
*/
|
||||||
|
const char* get_library_name();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Initialize and load erasure code library, returning error message if any.
|
||||||
|
*
|
||||||
|
* @param err The err message buffer.
|
||||||
|
* @param err_len The length of the message buffer.
|
||||||
|
*/
|
||||||
|
void load_erasurecode_lib(char* err, size_t err_len);
|
||||||
|
|
||||||
|
#endif //_ISAL_LOAD_H_
|
||||||
|
|
@ -0,0 +1,98 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
#include "isal_load.h"
|
||||||
|
#include "erasure_code.h"
|
||||||
|
#include "jni_common.h"
|
||||||
|
|
||||||
|
void loadLib(JNIEnv *env) {
|
||||||
|
char errMsg[1024];
|
||||||
|
load_erasurecode_lib(errMsg, sizeof(errMsg));
|
||||||
|
if (strlen(errMsg) > 0) {
|
||||||
|
THROW(env, "java/lang/UnsatisfiedLinkError", errMsg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void setCoder(JNIEnv* env, jobject thiz, IsalCoder* pCoder) {
|
||||||
|
jclass clazz = (*env)->GetObjectClass(env, thiz);
|
||||||
|
jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J");
|
||||||
|
(*env)->SetLongField(env, thiz, __coderState, (jlong) pCoder);
|
||||||
|
}
|
||||||
|
|
||||||
|
IsalCoder* getCoder(JNIEnv* env, jobject thiz) {
|
||||||
|
jclass clazz = (*env)->GetObjectClass(env, thiz);
|
||||||
|
|
||||||
|
jfieldID __verbose = (*env)->GetFieldID(env, clazz, "__native_verbose", "J");
|
||||||
|
int verbose = (int)(*env)->GetIntField(env, thiz, __verbose);
|
||||||
|
|
||||||
|
jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J");
|
||||||
|
IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env,
|
||||||
|
thiz, __coderState);
|
||||||
|
pCoder->verbose = verbose;
|
||||||
|
|
||||||
|
return pCoder;
|
||||||
|
}
|
||||||
|
|
||||||
|
void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets,
|
||||||
|
unsigned char** destInputs, int num) {
|
||||||
|
int numInputs = (*env)->GetArrayLength(env, inputs);
|
||||||
|
int* tmpInputOffsets;
|
||||||
|
int i;
|
||||||
|
jobject byteBuffer;
|
||||||
|
|
||||||
|
if (numInputs != num) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Invalid inputs");
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpInputOffsets = (int*)(*env)->GetIntArrayElements(env,
|
||||||
|
inputOffsets, NULL);
|
||||||
|
for (i = 0; i < numInputs; i++) {
|
||||||
|
byteBuffer = (*env)->GetObjectArrayElement(env, inputs, i);
|
||||||
|
if (byteBuffer != NULL) {
|
||||||
|
destInputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env,
|
||||||
|
byteBuffer));
|
||||||
|
destInputs[i] += tmpInputOffsets[i];
|
||||||
|
} else {
|
||||||
|
destInputs[i] = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets,
|
||||||
|
unsigned char** destOutputs, int num) {
|
||||||
|
int numOutputs = (*env)->GetArrayLength(env, outputs);
|
||||||
|
int i, *tmpOutputOffsets;
|
||||||
|
jobject byteBuffer;
|
||||||
|
|
||||||
|
if (numOutputs != num) {
|
||||||
|
THROW(env, "java/lang/InternalError", "Invalid outputs");
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpOutputOffsets = (int*)(*env)->GetIntArrayElements(env,
|
||||||
|
outputOffsets, NULL);
|
||||||
|
for (i = 0; i < numOutputs; i++) {
|
||||||
|
byteBuffer = (*env)->GetObjectArrayElement(env, outputs, i);
|
||||||
|
destOutputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env,
|
||||||
|
byteBuffer));
|
||||||
|
destOutputs[i] += tmpOutputOffsets[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef _JNI_CODER_COMMON_H_
|
||||||
|
#define _JNI_CODER_COMMON_H_
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include <jni.h>
|
||||||
|
|
||||||
|
#include "erasure_coder.h"
|
||||||
|
|
||||||
|
void loadLib(JNIEnv *env);
|
||||||
|
|
||||||
|
void setCoder(JNIEnv* env, jobject thiz, IsalCoder* coder);
|
||||||
|
|
||||||
|
IsalCoder* getCoder(JNIEnv* env, jobject thiz);
|
||||||
|
|
||||||
|
void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets,
|
||||||
|
unsigned char** destInputs, int num);
|
||||||
|
|
||||||
|
void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets,
|
||||||
|
unsigned char** destOutputs, int num);
|
||||||
|
|
||||||
|
#endif //_JNI_CODER_COMMON_H_
|
||||||
|
|
@ -21,7 +21,7 @@
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#include "org_apache_hadoop.h"
|
#include "org_apache_hadoop.h"
|
||||||
#include "../include/erasure_code.h"
|
#include "jni_common.h"
|
||||||
#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h"
|
#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h"
|
||||||
|
|
||||||
#ifdef UNIX
|
#ifdef UNIX
|
||||||
|
|
@ -31,11 +31,7 @@
|
||||||
JNIEXPORT void JNICALL
|
JNIEXPORT void JNICALL
|
||||||
Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary
|
Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary
|
||||||
(JNIEnv *env, jclass myclass) {
|
(JNIEnv *env, jclass myclass) {
|
||||||
char errMsg[1024];
|
loadLib(env);
|
||||||
load_erasurecode_lib(errMsg, sizeof(errMsg));
|
|
||||||
if (strlen(errMsg) > 0) {
|
|
||||||
THROW(env, "java/lang/UnsatisfiedLinkError", errMsg);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
JNIEXPORT jstring JNICALL
|
JNIEXPORT jstring JNICALL
|
||||||
|
|
@ -0,0 +1,72 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
#include "erasure_code.h"
|
||||||
|
#include "gf_util.h"
|
||||||
|
#include "jni_common.h"
|
||||||
|
#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h"
|
||||||
|
|
||||||
|
typedef struct _RSDecoder {
|
||||||
|
IsalDecoder decoder;
|
||||||
|
unsigned char* inputs[MMAX];
|
||||||
|
unsigned char* outputs[MMAX];
|
||||||
|
} RSDecoder;
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl(
|
||||||
|
JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) {
|
||||||
|
RSDecoder* rsDecoder = (RSDecoder*)malloc(sizeof(RSDecoder));
|
||||||
|
memset(rsDecoder, 0, sizeof(*rsDecoder));
|
||||||
|
initDecoder(&rsDecoder->decoder, (int)numDataUnits, (int)numParityUnits);
|
||||||
|
|
||||||
|
setCoder(env, thiz, &rsDecoder->decoder.coder);
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl(
|
||||||
|
JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
|
||||||
|
jint dataLen, jintArray erasedIndexes, jobjectArray outputs,
|
||||||
|
jintArray outputOffsets) {
|
||||||
|
RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
|
||||||
|
|
||||||
|
int numDataUnits = rsDecoder->decoder.coder.numDataUnits;
|
||||||
|
int numParityUnits = rsDecoder->decoder.coder.numParityUnits;
|
||||||
|
int chunkSize = (int)dataLen;
|
||||||
|
|
||||||
|
int* tmpErasedIndexes = (int*)(*env)->GetIntArrayElements(env,
|
||||||
|
erasedIndexes, NULL);
|
||||||
|
int numErased = (*env)->GetArrayLength(env, erasedIndexes);
|
||||||
|
getInputs(env, inputs, inputOffsets, rsDecoder->inputs,
|
||||||
|
numDataUnits + numParityUnits);
|
||||||
|
getOutputs(env, outputs, outputOffsets, rsDecoder->outputs, numErased);
|
||||||
|
|
||||||
|
decode(&rsDecoder->decoder, rsDecoder->inputs, tmpErasedIndexes,
|
||||||
|
numErased, rsDecoder->outputs, chunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl(
|
||||||
|
JNIEnv *env, jobject thiz) {
|
||||||
|
RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
|
||||||
|
free(rsDecoder);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "org_apache_hadoop.h"
|
||||||
|
#include "erasure_code.h"
|
||||||
|
#include "gf_util.h"
|
||||||
|
#include "jni_common.h"
|
||||||
|
#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h"
|
||||||
|
|
||||||
|
typedef struct _RSEncoder {
|
||||||
|
IsalEncoder encoder;
|
||||||
|
unsigned char* inputs[MMAX];
|
||||||
|
unsigned char* outputs[MMAX];
|
||||||
|
} RSEncoder;
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl(
|
||||||
|
JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) {
|
||||||
|
RSEncoder* rsEncoder = (RSEncoder*)malloc(sizeof(RSEncoder));
|
||||||
|
memset(rsEncoder, 0, sizeof(*rsEncoder));
|
||||||
|
initEncoder(&rsEncoder->encoder, (int)numDataUnits, (int)numParityUnits);
|
||||||
|
|
||||||
|
setCoder(env, thiz, &rsEncoder->encoder.coder);
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl(
|
||||||
|
JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
|
||||||
|
jint dataLen, jobjectArray outputs, jintArray outputOffsets) {
|
||||||
|
RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
|
||||||
|
|
||||||
|
int numDataUnits = rsEncoder->encoder.coder.numDataUnits;
|
||||||
|
int numParityUnits = rsEncoder->encoder.coder.numParityUnits;
|
||||||
|
int chunkSize = (int)dataLen;
|
||||||
|
|
||||||
|
getInputs(env, inputs, inputOffsets, rsEncoder->inputs, numDataUnits);
|
||||||
|
getOutputs(env, outputs, outputOffsets, rsEncoder->outputs, numParityUnits);
|
||||||
|
|
||||||
|
encode(&rsEncoder->encoder, rsEncoder->inputs, rsEncoder->outputs, chunkSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
JNIEXPORT void JNICALL
|
||||||
|
Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl(
|
||||||
|
JNIEnv *env, jobject thiz) {
|
||||||
|
RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
|
||||||
|
free(rsEncoder);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
/* DO NOT EDIT THIS FILE - it is machine generated */
|
||||||
|
#include <jni.h>
|
||||||
|
/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder */
|
||||||
|
|
||||||
|
#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
|
||||||
|
#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
|
||||||
|
#ifdef __cplusplus
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
|
||||||
|
* Method: initImpl
|
||||||
|
* Signature: (II[I)V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl
|
||||||
|
(JNIEnv *, jobject, jint, jint);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
|
||||||
|
* Method: decodeImpl
|
||||||
|
* Signature: ([Ljava/nio/ByteBuffer;[II[I[Ljava/nio/ByteBuffer;[I)V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl
|
||||||
|
(JNIEnv *, jobject, jobjectArray, jintArray, jint, jintArray, jobjectArray, jintArray);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
|
||||||
|
* Method: destroyImpl
|
||||||
|
* Signature: ()V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl
|
||||||
|
(JNIEnv *, jobject);
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
/* DO NOT EDIT THIS FILE - it is machine generated */
|
||||||
|
#include <jni.h>
|
||||||
|
/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder */
|
||||||
|
|
||||||
|
#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
|
||||||
|
#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
|
||||||
|
#ifdef __cplusplus
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
|
||||||
|
* Method: initImpl
|
||||||
|
* Signature: (II[I)V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl
|
||||||
|
(JNIEnv *, jobject, jint, jint);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
|
||||||
|
* Method: encodeImpl
|
||||||
|
* Signature: ([Ljava/nio/ByteBuffer;[II[Ljava/nio/ByteBuffer;[I)V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl
|
||||||
|
(JNIEnv *, jobject, jobjectArray, jintArray, jint, jobjectArray, jintArray);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
|
||||||
|
* Method: destroyImpl
|
||||||
|
* Signature: ()V
|
||||||
|
*/
|
||||||
|
JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl
|
||||||
|
(JNIEnv *, jobject);
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
@ -17,185 +17,36 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This is a lightweight version of the same file in Intel ISA-L library to test
|
* This is a sample program illustrating how to use the Intel ISA-L library.
|
||||||
* and verify the basic functions of ISA-L integration. Note it's not serving as
|
* Note it's adapted from erasure_code_test.c test program, but trying to use
|
||||||
* a complete ISA-L library test nor as any sample to write an erasure coder
|
* variable names and styles we're more familiar with already similar to Java
|
||||||
* using the library. A sample is to be written and provided separately.
|
* coders.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "org_apache_hadoop.h"
|
#include "isal_load.h"
|
||||||
#include "erasure_code.h"
|
#include "erasure_code.h"
|
||||||
#include "gf_util.h"
|
#include "gf_util.h"
|
||||||
|
#include "erasure_coder.h"
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#define TEST_LEN 8192
|
int main(int argc, char *argv[]) {
|
||||||
#define TEST_SOURCES 127
|
|
||||||
#define MMAX TEST_SOURCES
|
|
||||||
#define KMAX TEST_SOURCES
|
|
||||||
#define TEST_SEED 11
|
|
||||||
|
|
||||||
static void dump(unsigned char *buf, int len)
|
|
||||||
{
|
|
||||||
int i;
|
|
||||||
for (i = 0; i < len;) {
|
|
||||||
printf(" %2x", 0xff & buf[i++]);
|
|
||||||
if (i % 32 == 0)
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
static void dump_matrix(unsigned char **s, int k, int m)
|
|
||||||
{
|
|
||||||
int i, j;
|
int i, j;
|
||||||
for (i = 0; i < k; i++) {
|
|
||||||
for (j = 0; j < m; j++) {
|
|
||||||
printf(" %2x", s[i][j]);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
static void dump_u8xu8(unsigned char *s, int k, int m)
|
|
||||||
{
|
|
||||||
int i, j;
|
|
||||||
for (i = 0; i < k; i++) {
|
|
||||||
for (j = 0; j < m; j++) {
|
|
||||||
printf(" %2x", 0xff & s[j + (i * m)]);
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
printf("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate Random errors
|
|
||||||
static void gen_err_list(unsigned char *src_err_list,
|
|
||||||
unsigned char *src_in_err, int *pnerrs, int *pnsrcerrs, int k, int m)
|
|
||||||
{
|
|
||||||
int i, err;
|
|
||||||
int nerrs = 0, nsrcerrs = 0;
|
|
||||||
|
|
||||||
for (i = 0, nerrs = 0, nsrcerrs = 0; i < m && nerrs < m - k; i++) {
|
|
||||||
err = 1 & rand();
|
|
||||||
src_in_err[i] = err;
|
|
||||||
if (err) {
|
|
||||||
src_err_list[nerrs++] = i;
|
|
||||||
if (i < k) {
|
|
||||||
nsrcerrs++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (nerrs == 0) { // should have at least one error
|
|
||||||
while ((err = (rand() % KMAX)) >= m) ;
|
|
||||||
src_err_list[nerrs++] = err;
|
|
||||||
src_in_err[err] = 1;
|
|
||||||
if (err < k)
|
|
||||||
nsrcerrs = 1;
|
|
||||||
}
|
|
||||||
*pnerrs = nerrs;
|
|
||||||
*pnsrcerrs = nsrcerrs;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
#define NO_INVERT_MATRIX -2
|
|
||||||
// Generate decode matrix from encode matrix
|
|
||||||
static int gf_gen_decode_matrix(unsigned char *encode_matrix,
|
|
||||||
unsigned char *decode_matrix,
|
|
||||||
unsigned char *invert_matrix,
|
|
||||||
unsigned int *decode_index,
|
|
||||||
unsigned char *src_err_list,
|
|
||||||
unsigned char *src_in_err,
|
|
||||||
int nerrs, int nsrcerrs, int k, int m)
|
|
||||||
{
|
|
||||||
int i, j, p;
|
|
||||||
int r;
|
|
||||||
unsigned char *backup, *b, s;
|
|
||||||
int incr = 0;
|
|
||||||
|
|
||||||
b = malloc(MMAX * KMAX);
|
|
||||||
backup = malloc(MMAX * KMAX);
|
|
||||||
|
|
||||||
if (b == NULL || backup == NULL) {
|
|
||||||
printf("Test failure! Error with malloc\n");
|
|
||||||
free(b);
|
|
||||||
free(backup);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
// Construct matrix b by removing error rows
|
|
||||||
for (i = 0, r = 0; i < k; i++, r++) {
|
|
||||||
while (src_in_err[r])
|
|
||||||
r++;
|
|
||||||
for (j = 0; j < k; j++) {
|
|
||||||
b[k * i + j] = encode_matrix[k * r + j];
|
|
||||||
backup[k * i + j] = encode_matrix[k * r + j];
|
|
||||||
}
|
|
||||||
decode_index[i] = r;
|
|
||||||
}
|
|
||||||
incr = 0;
|
|
||||||
while (h_gf_invert_matrix(b, invert_matrix, k) < 0) {
|
|
||||||
if (nerrs == (m - k)) {
|
|
||||||
free(b);
|
|
||||||
free(backup);
|
|
||||||
printf("BAD MATRIX\n");
|
|
||||||
return NO_INVERT_MATRIX;
|
|
||||||
}
|
|
||||||
incr++;
|
|
||||||
memcpy(b, backup, MMAX * KMAX);
|
|
||||||
for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) {
|
|
||||||
if (src_err_list[i] == (decode_index[k - 1] + incr)) {
|
|
||||||
// skip the erased parity line
|
|
||||||
incr++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (decode_index[k - 1] + incr >= m) {
|
|
||||||
free(b);
|
|
||||||
free(backup);
|
|
||||||
printf("BAD MATRIX\n");
|
|
||||||
return NO_INVERT_MATRIX;
|
|
||||||
}
|
|
||||||
decode_index[k - 1] += incr;
|
|
||||||
for (j = 0; j < k; j++)
|
|
||||||
b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j];
|
|
||||||
|
|
||||||
};
|
|
||||||
|
|
||||||
for (i = 0; i < nsrcerrs; i++) {
|
|
||||||
for (j = 0; j < k; j++) {
|
|
||||||
decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/* src_err_list from encode_matrix * invert of b for parity decoding */
|
|
||||||
for (p = nsrcerrs; p < nerrs; p++) {
|
|
||||||
for (i = 0; i < k; i++) {
|
|
||||||
s = 0;
|
|
||||||
for (j = 0; j < k; j++)
|
|
||||||
s ^= h_gf_mul(invert_matrix[j * k + i],
|
|
||||||
encode_matrix[k * src_err_list[p] + j]);
|
|
||||||
|
|
||||||
decode_matrix[k * p + i] = s;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(b);
|
|
||||||
free(backup);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc, char *argv[])
|
|
||||||
{
|
|
||||||
char err[256];
|
char err[256];
|
||||||
size_t err_len = sizeof(err);
|
size_t err_len = sizeof(err);
|
||||||
int re, i, j, p, m, k;
|
int chunkSize = 1024;
|
||||||
int nerrs, nsrcerrs;
|
int numDataUnits = 6;
|
||||||
unsigned int decode_index[MMAX];
|
int numParityUnits = 3;
|
||||||
unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
|
unsigned char** dataUnits;
|
||||||
unsigned char *encode_matrix, *decode_matrix, *invert_matrix, *g_tbls;
|
unsigned char** parityUnits;
|
||||||
unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
|
IsalEncoder* pEncoder;
|
||||||
unsigned char *recov[TEST_SOURCES];
|
int erasedIndexes[2];
|
||||||
|
unsigned char* allUnits[MMAX];
|
||||||
|
IsalDecoder* pDecoder;
|
||||||
|
unsigned char* decodingOutput[2];
|
||||||
|
unsigned char** backupUnits;
|
||||||
|
|
||||||
if (0 == build_support_erasurecode()) {
|
if (0 == build_support_erasurecode()) {
|
||||||
printf("The native library isn't available, skipping this test\n");
|
printf("The native library isn't available, skipping this test\n");
|
||||||
|
|
@ -209,102 +60,66 @@ int main(int argc, char *argv[])
|
||||||
}
|
}
|
||||||
|
|
||||||
printf("Performing erasure code test\n");
|
printf("Performing erasure code test\n");
|
||||||
srand(TEST_SEED);
|
|
||||||
|
|
||||||
// Allocate the arrays
|
dataUnits = calloc(numDataUnits, sizeof(unsigned char*));
|
||||||
for (i = 0; i < TEST_SOURCES; i++) {
|
parityUnits = calloc(numParityUnits, sizeof(unsigned char*));
|
||||||
buffs[i] = malloc(TEST_LEN);
|
backupUnits = calloc(numParityUnits, sizeof(unsigned char*));
|
||||||
|
|
||||||
|
// Allocate and generate data units
|
||||||
|
srand(135);
|
||||||
|
for (i = 0; i < numDataUnits; i++) {
|
||||||
|
dataUnits[i] = calloc(chunkSize, sizeof(unsigned char));
|
||||||
|
for (j = 0; j < chunkSize; j++) {
|
||||||
|
dataUnits[i][j] = rand();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (i = 0; i < TEST_SOURCES; i++) {
|
// Allocate and initialize parity units
|
||||||
temp_buffs[i] = malloc(TEST_LEN);
|
for (i = 0; i < numParityUnits; i++) {
|
||||||
|
parityUnits[i] = calloc(chunkSize, sizeof(unsigned char));
|
||||||
|
for (j = 0; j < chunkSize; j++) {
|
||||||
|
parityUnits[i][j] = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test erasure code by encode and recovery
|
pEncoder = (IsalEncoder*)malloc(sizeof(IsalEncoder));
|
||||||
|
memset(pEncoder, 0, sizeof(*pEncoder));
|
||||||
|
initEncoder(pEncoder, numDataUnits, numParityUnits);
|
||||||
|
encode(pEncoder, dataUnits, parityUnits, chunkSize);
|
||||||
|
|
||||||
encode_matrix = malloc(MMAX * KMAX);
|
pDecoder = (IsalDecoder*)malloc(sizeof(IsalDecoder));
|
||||||
decode_matrix = malloc(MMAX * KMAX);
|
memset(pDecoder, 0, sizeof(*pDecoder));
|
||||||
invert_matrix = malloc(MMAX * KMAX);
|
initDecoder(pDecoder, numDataUnits, numParityUnits);
|
||||||
g_tbls = malloc(KMAX * TEST_SOURCES * 32);
|
|
||||||
if (encode_matrix == NULL || decode_matrix == NULL
|
|
||||||
|| invert_matrix == NULL || g_tbls == NULL) {
|
|
||||||
snprintf(err, err_len, "%s", "allocating test matrix buffers error");
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
m = 9;
|
memcpy(allUnits, dataUnits, numDataUnits * (sizeof (unsigned char*)));
|
||||||
k = 5;
|
memcpy(allUnits + numDataUnits, parityUnits,
|
||||||
if (m > MMAX || k > KMAX)
|
numParityUnits * (sizeof (unsigned char*)));
|
||||||
return -1;
|
|
||||||
|
|
||||||
// Make random data
|
erasedIndexes[0] = 1;
|
||||||
for (i = 0; i < k; i++)
|
erasedIndexes[1] = 7;
|
||||||
for (j = 0; j < TEST_LEN; j++)
|
|
||||||
buffs[i][j] = rand();
|
|
||||||
|
|
||||||
// The matrix generated by gf_gen_cauchy1_matrix
|
backupUnits[0] = allUnits[1];
|
||||||
// is always invertable.
|
backupUnits[1] = allUnits[7];
|
||||||
h_gf_gen_cauchy_matrix(encode_matrix, m, k);
|
|
||||||
|
|
||||||
// Generate g_tbls from encode matrix encode_matrix
|
allUnits[0] = NULL; // Not to read
|
||||||
h_ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
|
allUnits[1] = NULL;
|
||||||
|
allUnits[7] = NULL;
|
||||||
|
|
||||||
// Perform matrix dot_prod for EC encoding
|
decodingOutput[0] = malloc(chunkSize);
|
||||||
// using g_tbls from encode matrix encode_matrix
|
decodingOutput[1] = malloc(chunkSize);
|
||||||
h_ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]);
|
|
||||||
|
|
||||||
// Choose random buffers to be in erasure
|
decode(pDecoder, allUnits, erasedIndexes, 2, decodingOutput, chunkSize);
|
||||||
memset(src_in_err, 0, TEST_SOURCES);
|
|
||||||
gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
|
|
||||||
|
|
||||||
// Generate decode matrix
|
|
||||||
re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
|
|
||||||
invert_matrix, decode_index, src_err_list, src_in_err,
|
|
||||||
nerrs, nsrcerrs, k, m);
|
|
||||||
if (re != 0) {
|
|
||||||
snprintf(err, err_len, "%s", "gf_gen_decode_matrix failed");
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
// Pack recovery array as list of valid sources
|
|
||||||
// Its order must be the same as the order
|
|
||||||
// to generate matrix b in gf_gen_decode_matrix
|
|
||||||
for (i = 0; i < k; i++) {
|
|
||||||
recov[i] = buffs[decode_index[i]];
|
|
||||||
}
|
|
||||||
|
|
||||||
// Recover data
|
|
||||||
h_ec_init_tables(k, nerrs, decode_matrix, g_tbls);
|
|
||||||
h_ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
|
|
||||||
for (i = 0; i < nerrs; i++) {
|
|
||||||
if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) {
|
|
||||||
snprintf(err, err_len, "%s", "Error recovery failed");
|
|
||||||
printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
|
|
||||||
|
|
||||||
printf(" - erase list = ");
|
|
||||||
for (j = 0; j < nerrs; j++) {
|
|
||||||
printf(" %d", src_err_list[j]);
|
|
||||||
}
|
|
||||||
|
|
||||||
printf(" - Index = ");
|
|
||||||
for (p = 0; p < k; p++) {
|
|
||||||
printf(" %d", decode_index[p]);
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("\nencode_matrix:\n");
|
|
||||||
dump_u8xu8((unsigned char *) encode_matrix, m, k);
|
|
||||||
printf("inv b:\n");
|
|
||||||
dump_u8xu8((unsigned char *) invert_matrix, k, k);
|
|
||||||
printf("\ndecode_matrix:\n");
|
|
||||||
dump_u8xu8((unsigned char *) decode_matrix, m, k);
|
|
||||||
printf("recov %d:", src_err_list[i]);
|
|
||||||
dump(temp_buffs[k + i], 25);
|
|
||||||
printf("orig :");
|
|
||||||
dump(buffs[src_err_list[i]], 25);
|
|
||||||
|
|
||||||
|
for (i = 0; i < pDecoder->numErased; i++) {
|
||||||
|
if (0 != memcmp(decodingOutput[i], backupUnits[i], chunkSize)) {
|
||||||
|
fprintf(stderr, "Decoding failed\n\n");
|
||||||
|
dumpDecoder(pDecoder);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
printf("done EC tests: Pass\n");
|
dumpDecoder(pDecoder);
|
||||||
|
fprintf(stdout, "Successfully done, passed!\n\n");
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue