diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index 51fb0fe49c3..8026eb54cf4 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
@@ -103,13 +103,25 @@ find_library(ISAL_LIBRARY
set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES})
if (ISAL_LIBRARY)
GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME)
- set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode/include)
+ set(ISAL_INCLUDE_DIR ${SRC}/io/erasurecode)
set(ISAL_SOURCE_FILES
- ${SRC}/io/erasurecode/coder/erasure_code_native.c
- ${SRC}/io/erasurecode/erasure_code.c)
+ ${SRC}/io/erasurecode/isal_load.c
+ ${SRC}/io/erasurecode/erasure_code.c
+ ${SRC}/io/erasurecode/gf_util.c
+ ${SRC}/io/erasurecode/dump.c
+ ${SRC}/io/erasurecode/erasure_coder.c
+ ${SRC}/io/erasurecode/jni_erasure_code_native.c
+ ${SRC}/io/erasurecode/jni_common.c
+ ${SRC}/io/erasurecode/jni_rs_encoder.c
+ ${SRC}/io/erasurecode/jni_rs_decoder.c)
+
add_executable(erasure_code_test
- ${SRC}/io/erasurecode/erasure_code.c
- ${TST}/io/erasurecode/erasure_code_test.c
+ ${SRC}/io/erasurecode/isal_load.c
+ ${SRC}/io/erasurecode/erasure_code.c
+ ${SRC}/io/erasurecode/gf_util.c
+ ${SRC}/io/erasurecode/dump.c
+ ${SRC}/io/erasurecode/erasure_coder.c
+ ${TST}/io/erasurecode/erasure_code_test.c
)
target_link_libraries(erasure_code_test ${CMAKE_DL_LIBS})
else (ISAL_LIBRARY)
diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
index 17149f7bfb5..2274c413319 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
+++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj
@@ -162,10 +162,17 @@
src\org\apache\hadoop\io\nativeio;%(AdditionalIncludeDirectories)
-
+
/D HADOOP_ISAL_LIBRARY=\"isa-l.dll\"
-
+
+
+
+
+
+
+
+
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c
new file mode 100644
index 00000000000..20bd1893fd0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.c
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "erasure_code.h"
+#include "gf_util.h"
+#include "erasure_coder.h"
+
+#include
+#include
+#include
+
+void dump(unsigned char* buf, int len) {
+ int i;
+ for (i = 0; i < len;) {
+ printf(" %2x", 0xff & buf[i++]);
+ if (i % 32 == 0)
+ printf("\n");
+ }
+}
+
+void dumpMatrix(unsigned char** buf, int n1, int n2) {
+ int i, j;
+ for (i = 0; i < n1; i++) {
+ for (j = 0; j < n2; j++) {
+ printf(" %2x", buf[i][j]);
+ }
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void dumpCodingMatrix(unsigned char* buf, int n1, int n2) {
+ int i, j;
+ for (i = 0; i < n1; i++) {
+ for (j = 0; j < n2; j++) {
+ printf(" %d", 0xff & buf[j + (i * n2)]);
+ }
+ printf("\n");
+ }
+ printf("\n");
+}
+
+void dumpEncoder(IsalEncoder* pCoder) {
+ int numDataUnits = pCoder->coder.numDataUnits;
+ int numParityUnits = pCoder->coder.numDataUnits;
+ int numAllUnits = pCoder->coder.numAllUnits;
+
+ printf("Encoding (numAlnumParityUnitslUnits = %d, numDataUnits = %d)\n",
+ numParityUnits, numDataUnits);
+
+ printf("\n\nEncodeMatrix:\n");
+ dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix,
+ numDataUnits, numAllUnits);
+}
+
+void dumpDecoder(IsalDecoder* pCoder) {
+ int i, j;
+ int numDataUnits = pCoder->coder.numDataUnits;
+ int numAllUnits = pCoder->coder.numAllUnits;
+
+ printf("Recovering (numAllUnits = %d, numDataUnits = %d, numErased = %d)\n",
+ numAllUnits, numDataUnits, pCoder->numErased);
+
+ printf(" - ErasedIndexes = ");
+ for (j = 0; j < pCoder->numErased; j++) {
+ printf(" %d", pCoder->erasedIndexes[j]);
+ }
+ printf(" - DecodeIndex = ");
+ for (i = 0; i < numDataUnits; i++) {
+ printf(" %d", pCoder->decodeIndex[i]);
+ }
+
+ printf("\n\nEncodeMatrix:\n");
+ dumpCodingMatrix((unsigned char*) pCoder->encodeMatrix,
+ numDataUnits, numAllUnits);
+
+ printf("InvertMatrix:\n");
+ dumpCodingMatrix((unsigned char*) pCoder->invertMatrix,
+ numDataUnits, numDataUnits);
+
+ printf("DecodeMatrix:\n");
+ dumpCodingMatrix((unsigned char*) pCoder->decodeMatrix,
+ numDataUnits, numAllUnits);
+}
+
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h
new file mode 100644
index 00000000000..f58d960fd0d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/dump.h
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Dump utilities for erasure coders.
+ */
+
+#ifndef _DUMP_H_
+#define _DUMP_H_
+
+#include
+#include
+#include
+
+void dumpEncoder(IsalEncoder* pCoder);
+
+void dumpDecoder(IsalDecoder* pCoder);
+
+void dump(unsigned char* buf, int len);
+
+void dumpMatrix(unsigned char** s, int k, int m);
+
+void dumpCodingMatrix(unsigned char* s, int n1, int n2);
+
+#endif //_DUMP_H_
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c
index a6c099ad72c..125068877be 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.c
@@ -20,252 +20,25 @@
#include
#include
-#include "org_apache_hadoop.h"
-#include "../include/gf_util.h"
-#include "../include/erasure_code.h"
-
-#ifdef UNIX
-#include
-#include
-#include
-#include
-
-#include "config.h"
-#endif
-
-#ifdef WINDOWS
-#include
-#endif
+#include "isal_load.h"
+#include "erasure_code.h"
/**
* erasure_code.c
- * Implementation erasure code utilities based on lib of erasure_code.so.
- * Building of this codes won't rely on any ISA-L source codes, but running
- * into this will rely on successfully loading of the dynamic library.
+ * Implementation erasure code utilities based on ISA-L library.
*
*/
-/**
- * The loaded library handle.
- */
-static void* libec = NULL;
-
-/**
- * A helper function to dlsym a 'symbol' from a given library-handle.
- */
-
-#ifdef UNIX
-
-static __attribute__ ((unused))
-void *my_dlsym(void *handle, const char *symbol) {
- void *func_ptr = dlsym(handle, symbol);
- return func_ptr;
-}
-
-/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
-#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, handle, symbol) \
- if ((func_ptr = my_dlsym(handle, symbol)) == NULL) { \
- return "Failed to load symbol" symbol; \
- }
-
-#endif
-
-#ifdef WINDOWS
-
-
-
-static FARPROC WINAPI my_dlsym(HMODULE handle, LPCSTR symbol) {
- FARPROC func_ptr = GetProcAddress(handle, symbol);
- return func_ptr;
-}
-
-/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
-#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, handle, symbol) \
- if ((func_ptr = (func_type)my_dlsym(handle, symbol)) == NULL) { \
- return "Failed to load symbol" symbol; \
- }
-
-#endif
-
-
-#ifdef UNIX
-// For gf_util.h
-static unsigned char (*d_gf_mul)(unsigned char, unsigned char);
-static unsigned char (*d_gf_inv)(unsigned char);
-static void (*d_gf_gen_rs_matrix)(unsigned char *, int, int);
-static void (*d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
-static int (*d_gf_invert_matrix)(unsigned char *, unsigned char *, const int);
-static int (*d_gf_vect_mul)(int, unsigned char *, void *, void *);
-
-// For erasure_code.h
-static void (*d_ec_init_tables)(int, int, unsigned char*, unsigned char*);
-static void (*d_ec_encode_data)(int, int, int, unsigned char*,
- unsigned char**, unsigned char**);
-static void (*d_ec_encode_data_update)(int, int, int, int, unsigned char*,
- unsigned char*, unsigned char**);
-#endif
-
-#ifdef WINDOWS
-// For erasure_code.h
-typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char);
-static __d_gf_mul d_gf_mul;
-typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char);
-static __d_gf_inv d_gf_inv;
-typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int);
-static __d_gf_gen_rs_matrix d_gf_gen_rs_matrix;
-typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
-static __d_gf_gen_cauchy_matrix d_gf_gen_cauchy_matrix;
-typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *,
- unsigned char *, const int);
-static __d_gf_invert_matrix d_gf_invert_matrix;
-typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *);
-static __d_gf_vect_mul d_gf_vect_mul;
-
-// For erasure_code.h
-typedef void (__cdecl *__d_ec_init_tables)(int, int,
- unsigned char*, unsigned char*);
-static __d_ec_init_tables d_ec_init_tables;
-typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*,
- unsigned char**, unsigned char**);
-static __d_ec_encode_data d_ec_encode_data;
-typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
- unsigned char*, unsigned char**);
-static __d_ec_encode_data_update d_ec_encode_data_update;
-#endif
-
-static const char* load_functions(void* libec) {
-#ifdef UNIX
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_mul, libec, "gf_mul");
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_inv, libec, "gf_inv");
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_invert_matrix, libec, "gf_invert_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(d_gf_vect_mul, libec, "gf_vect_mul");
-
- EC_LOAD_DYNAMIC_SYMBOL(d_ec_init_tables, libec, "ec_init_tables");
- EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data, libec, "ec_encode_data");
- EC_LOAD_DYNAMIC_SYMBOL(d_ec_encode_data_update, libec, "ec_encode_data_update");
-#endif
-
-#ifdef WINDOWS
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, d_gf_mul, libec, "gf_mul");
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, d_gf_inv, libec, "gf_inv");
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, d_gf_gen_rs_matrix, libec, "gf_gen_rs_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, d_gf_gen_cauchy_matrix, libec, "gf_gen_cauchy1_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, d_gf_invert_matrix, libec, "gf_invert_matrix");
- EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, d_gf_vect_mul, libec, "gf_vect_mul");
-
- EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, d_ec_init_tables, libec, "ec_init_tables");
- EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, d_ec_encode_data, libec, "ec_encode_data");
- EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, d_ec_encode_data_update, libec, "ec_encode_data_update");
-#endif
-
- return NULL;
-}
-
-void load_erasurecode_lib(char* err, size_t err_len) {
- const char* errMsg;
-
- err[0] = '\0';
-
- if (libec != NULL) {
- return;
- }
-
- // Load Intel ISA-L
- #ifdef UNIX
- libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
- if (libec == NULL) {
- snprintf(err, err_len, "Failed to load %s (%s)",
- HADOOP_ISAL_LIBRARY, dlerror());
- return;
- }
- // Clear any existing error
- dlerror();
- #endif
-
- #ifdef WINDOWS
- libec = LoadLibrary(HADOOP_ISAL_LIBRARY);
- if (libec == NULL) {
- snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY);
- return;
- }
- #endif
-
- errMsg = load_functions(libec);
- if (errMsg != NULL) {
- snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg);
- }
-}
-
-int build_support_erasurecode() {
-#ifdef HADOOP_ISAL_LIBRARY
- return 1;
-#else
- return 0;
-#endif
-}
-
-const char* get_library_name() {
-#ifdef UNIX
- Dl_info dl_info;
-
- if (d_ec_encode_data == NULL) {
- return HADOOP_ISAL_LIBRARY;
- }
-
- if(dladdr(d_ec_encode_data, &dl_info)) {
- return dl_info.dli_fname;
- }
-#else
- LPTSTR filename = NULL;
-
- if (libec == NULL) {
- return HADOOP_ISAL_LIBRARY;
- }
-
- if (GetModuleFileName(libec, filename, 256) > 0) {
- return filename;
- }
-#endif
-
- return NULL;
-}
-
-unsigned char h_gf_mul(unsigned char a, unsigned char b) {
- return d_gf_mul(a, b);
-}
-
-unsigned char h_gf_inv(unsigned char a) {
- return d_gf_inv(a);
-}
-
-void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) {
- d_gf_gen_rs_matrix(a, m, k);
-}
-
-void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) {
- d_gf_gen_cauchy_matrix(a, m, k);
-}
-
-int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) {
- return d_gf_invert_matrix(in, out, n);
-}
-
-int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) {
- return d_gf_vect_mul(len, gftbl, src, dest);
-}
-
void h_ec_init_tables(int k, int rows, unsigned char* a, unsigned char* gftbls) {
- d_ec_init_tables(k, rows, a, gftbls);
+ isaLoader->ec_init_tables(k, rows, a, gftbls);
}
void h_ec_encode_data(int len, int k, int rows, unsigned char *gftbls,
unsigned char **data, unsigned char **coding) {
- d_ec_encode_data(len, k, rows, gftbls, data, coding);
+ isaLoader->ec_encode_data(len, k, rows, gftbls, data, coding);
}
void h_ec_encode_data_update(int len, int k, int rows, int vec_i,
unsigned char *gftbls, unsigned char *data, unsigned char **coding) {
- d_ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding);
+ isaLoader->ec_encode_data_update(len, k, rows, vec_i, gftbls, data, coding);
}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h
similarity index 92%
rename from hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h
rename to hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h
index 123085ecb51..a6cfd587900 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/erasure_code.h
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_code.h
@@ -37,24 +37,6 @@
*
*/
-/**
- * Return 0 if not support, 1 otherwise.
- */
-int build_support_erasurecode();
-
-/**
- * Get the library name possibly of full path.
- */
-const char* get_library_name();
-
-/**
- * Initialize and load erasure code library, returning error message if any.
- *
- * @param err The err message buffer.
- * @param err_len The length of the message buffer.
- */
-void load_erasurecode_lib(char* err, size_t err_len);
-
/**
* Initialize tables for fast Erasure Code encode and decode.
*
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
new file mode 100644
index 00000000000..b3479bbd712
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "erasure_code.h"
+#include "gf_util.h"
+#include "erasure_coder.h"
+
+#include
+#include
+#include
+
+void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits) {
+ pCoder->verbose = 0;
+ pCoder->numParityUnits = numParityUnits;
+ pCoder->numDataUnits = numDataUnits;
+ pCoder->numAllUnits = numDataUnits + numParityUnits;
+}
+
+// 0 not to verbose, 1 to verbose
+void allowVerbose(IsalCoder* pCoder, int flag) {
+ pCoder->verbose = flag;
+}
+
+static void initEncodeMatrix(int numDataUnits, int numParityUnits,
+ unsigned char* encodeMatrix) {
+ // Generate encode matrix, always invertible
+ h_gf_gen_cauchy_matrix(encodeMatrix,
+ numDataUnits + numParityUnits, numDataUnits);
+}
+
+void initEncoder(IsalEncoder* pCoder, int numDataUnits,
+ int numParityUnits) {
+ initCoder(&pCoder->coder, numDataUnits, numParityUnits);
+
+ initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix);
+
+ // Generate gftbls from encode matrix
+ h_ec_init_tables(numDataUnits, numParityUnits,
+ &pCoder->encodeMatrix[numDataUnits * numDataUnits],
+ pCoder->gftbls);
+
+ if (pCoder->coder.verbose > 0) {
+ dumpEncoder(pCoder);
+ }
+}
+
+void initDecoder(IsalDecoder* pCoder, int numDataUnits,
+ int numParityUnits) {
+ initCoder(&pCoder->coder, numDataUnits, numParityUnits);
+
+ initEncodeMatrix(numDataUnits, numParityUnits, pCoder->encodeMatrix);
+}
+
+int encode(IsalEncoder* pCoder, unsigned char** dataUnits,
+ unsigned char** parityUnits, int chunkSize) {
+ int numDataUnits = pCoder->coder.numDataUnits;
+ int numParityUnits = pCoder->coder.numParityUnits;
+ int i;
+
+ for (i = 0; i < numParityUnits; i++) {
+ memset(parityUnits[i], 0, chunkSize);
+ }
+
+ h_ec_encode_data(chunkSize, numDataUnits, numParityUnits,
+ pCoder->gftbls, dataUnits, parityUnits);
+
+ return 0;
+}
+
+// Return 1 when diff, 0 otherwise
+static int compare(int* arr1, int len1, int* arr2, int len2) {
+ int i;
+
+ if (len1 == len2) {
+ for (i = 0; i < len1; i++) {
+ if (arr1[i] != arr2[i]) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ return 1;
+}
+
+static int processErasures(IsalDecoder* pCoder, unsigned char** inputs,
+ int* erasedIndexes, int numErased) {
+ int i, r, ret, index;
+ int numDataUnits = pCoder->coder.numDataUnits;
+ int isChanged = 0;
+
+ for (i = 0, r = 0; i < numDataUnits; i++, r++) {
+ while (inputs[r] == NULL) {
+ r++;
+ }
+
+ if (pCoder->decodeIndex[i] != r) {
+ pCoder->decodeIndex[i] = r;
+ isChanged = 1;
+ }
+ }
+
+ for (i = 0; i < numDataUnits; i++) {
+ pCoder->realInputs[i] = inputs[pCoder->decodeIndex[i]];
+ }
+
+ if (isChanged == 0 &&
+ compare(pCoder->erasedIndexes, pCoder->numErased,
+ erasedIndexes, numErased) == 0) {
+ return 0; // Optimization, nothing to do
+ }
+
+ clearDecoder(pCoder);
+
+ for (i = 0; i < numErased; i++) {
+ index = erasedIndexes[i];
+ pCoder->erasedIndexes[i] = index;
+ pCoder->erasureFlags[index] = 1;
+ if (index < numDataUnits) {
+ pCoder->numErasedDataUnits++;
+ }
+ }
+
+ pCoder->numErased = numErased;
+
+ ret = generateDecodeMatrix(pCoder);
+ if (ret != 0) {
+ printf("Failed to generate decode matrix\n");
+ return -1;
+ }
+
+ h_ec_init_tables(numDataUnits, pCoder->numErased,
+ pCoder->decodeMatrix, pCoder->gftbls);
+
+ if (pCoder->coder.verbose > 0) {
+ dumpDecoder(pCoder);
+ }
+
+ return 0;
+}
+
+int decode(IsalDecoder* pCoder, unsigned char** inputs,
+ int* erasedIndexes, int numErased,
+ unsigned char** outputs, int chunkSize) {
+ int numDataUnits = pCoder->coder.numDataUnits;
+ int i;
+
+ processErasures(pCoder, inputs, erasedIndexes, numErased);
+
+ for (i = 0; i < numErased; i++) {
+ memset(outputs[i], 0, chunkSize);
+ }
+
+ h_ec_encode_data(chunkSize, numDataUnits, pCoder->numErased,
+ pCoder->gftbls, pCoder->realInputs, outputs);
+
+ return 0;
+}
+
+// Clear variables used per decode call
+void clearDecoder(IsalDecoder* decoder) {
+ decoder->numErasedDataUnits = 0;
+ decoder->numErased = 0;
+ memset(decoder->gftbls, 0, sizeof(decoder->gftbls));
+ memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix));
+ memset(decoder->tmpMatrix, 0, sizeof(decoder->tmpMatrix));
+ memset(decoder->invertMatrix, 0, sizeof(decoder->invertMatrix));
+ memset(decoder->erasureFlags, 0, sizeof(decoder->erasureFlags));
+ memset(decoder->erasedIndexes, 0, sizeof(decoder->erasedIndexes));
+}
+
+// Generate decode matrix from encode matrix
+int generateDecodeMatrix(IsalDecoder* pCoder) {
+ int i, j, r, p;
+ unsigned char s;
+ int numDataUnits;
+
+ numDataUnits = pCoder->coder.numDataUnits;
+
+ // Construct matrix b by removing error rows
+ for (i = 0; i < numDataUnits; i++) {
+ r = pCoder->decodeIndex[i];
+ for (j = 0; j < numDataUnits; j++) {
+ pCoder->tmpMatrix[numDataUnits * i + j] =
+ pCoder->encodeMatrix[numDataUnits * r + j];
+ }
+ }
+
+ h_gf_invert_matrix(pCoder->tmpMatrix,
+ pCoder->invertMatrix, numDataUnits);
+
+ for (i = 0; i < pCoder->numErasedDataUnits; i++) {
+ for (j = 0; j < numDataUnits; j++) {
+ pCoder->decodeMatrix[numDataUnits * i + j] =
+ pCoder->invertMatrix[numDataUnits *
+ pCoder->erasedIndexes[i] + j];
+ }
+ }
+
+ for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) {
+ for (i = 0; i < numDataUnits; i++) {
+ s = 0;
+ for (j = 0; j < numDataUnits; j++) {
+ s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
+ pCoder->encodeMatrix[numDataUnits *
+ pCoder->erasedIndexes[p] + j]);
+ }
+
+ pCoder->decodeMatrix[numDataUnits * p + i] = s;
+ }
+ }
+
+ return 0;
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h
new file mode 100644
index 00000000000..8f5bf8a3ca7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is a sample program illustrating how to use the Intel ISA-L library.
+ * Note it's adapted from erasure_code_test.c test program, but trying to use
+ * variable names and styles we're more familiar with already similar to Java
+ * coders.
+ */
+
+#ifndef _ERASURE_CODER_H_
+#define _ERASURE_CODER_H_
+
+#include
+#include
+#include
+
+#define MMAX 14
+#define KMAX 10
+
+typedef struct _IsalCoder {
+ int verbose;
+ int numParityUnits;
+ int numDataUnits;
+ int numAllUnits;
+} IsalCoder;
+
+typedef struct _IsalEncoder {
+ IsalCoder coder;
+
+ unsigned char gftbls[MMAX * KMAX * 32];
+
+ unsigned char encodeMatrix[MMAX * KMAX];
+} IsalEncoder;
+
+typedef struct _IsalDecoder {
+ IsalCoder coder;
+
+ unsigned char encodeMatrix[MMAX * KMAX];
+
+ // Below are per decode call
+ unsigned char gftbls[MMAX * KMAX * 32];
+ unsigned int decodeIndex[MMAX];
+ unsigned char tmpMatrix[MMAX * KMAX];
+ unsigned char invertMatrix[MMAX * KMAX];
+ unsigned char decodeMatrix[MMAX * KMAX];
+ unsigned char erasureFlags[MMAX];
+ int erasedIndexes[MMAX];
+ int numErased;
+ int numErasedDataUnits;
+ unsigned char* realInputs[MMAX];
+} IsalDecoder;
+
+void initCoder(IsalCoder* pCoder, int numDataUnits, int numParityUnits);
+
+void allowVerbose(IsalCoder* pCoder, int flag);
+
+void initEncoder(IsalEncoder* encoder, int numDataUnits, int numParityUnits);
+
+void initDecoder(IsalDecoder* decoder, int numDataUnits, int numParityUnits);
+
+void clearDecoder(IsalDecoder* decoder);
+
+int encode(IsalEncoder* encoder, unsigned char** dataUnits,
+ unsigned char** parityUnits, int chunkSize);
+
+int decode(IsalDecoder* decoder, unsigned char** allUnits,
+ int* erasedIndexes, int numErased,
+ unsigned char** recoveredUnits, int chunkSize);
+
+int generateDecodeMatrix(IsalDecoder* pCoder);
+
+#endif //_ERASURE_CODER_H_
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c
new file mode 100644
index 00000000000..804dcec61cc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.c
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "isal_load.h"
+#include "gf_util.h"
+
+/**
+ * gf_util.c
+ * Implementation GF utilities based on ISA-L library.
+ *
+ */
+
+unsigned char h_gf_mul(unsigned char a, unsigned char b) {
+ return isaLoader->gf_mul(a, b);
+}
+
+unsigned char h_gf_inv(unsigned char a) {
+ return isaLoader->gf_inv(a);
+}
+
+void h_gf_gen_rs_matrix(unsigned char *a, int m, int k) {
+ isaLoader->gf_gen_rs_matrix(a, m, k);
+}
+
+void h_gf_gen_cauchy_matrix(unsigned char *a, int m, int k) {
+ isaLoader->gf_gen_cauchy_matrix(a, m, k);
+}
+
+int h_gf_invert_matrix(unsigned char *in, unsigned char *out, const int n) {
+ return isaLoader->gf_invert_matrix(in, out, n);
+}
+
+int h_gf_vect_mul(int len, unsigned char *gftbl, void *src, void *dest) {
+ return isaLoader->gf_vect_mul(len, gftbl, src, dest);
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/include/gf_util.h
rename to hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/gf_util.h
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
new file mode 100644
index 00000000000..55e8efda014
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.c
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "org_apache_hadoop.h"
+#include "isal_load.h"
+
+#ifdef UNIX
+#include
+#include
+#include
+#include
+
+#include "config.h"
+#endif
+
+#ifdef WINDOWS
+#include
+#endif
+
+IsaLibLoader* isaLoader;
+
+/**
+ * isal_load.c
+ * Utility of loading the ISA-L library and the required functions.
+ * Building of this codes won't rely on any ISA-L source codes, but running
+ * into this will rely on successfully loading of the dynamic library.
+ *
+ */
+
+static const char* load_functions() {
+#ifdef UNIX
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_mul), "gf_mul");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_inv), "gf_inv");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_invert_matrix), "gf_invert_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->gf_vect_mul), "gf_vect_mul");
+
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_init_tables), "ec_init_tables");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data), "ec_encode_data");
+ EC_LOAD_DYNAMIC_SYMBOL((isaLoader->ec_encode_data_update), "ec_encode_data_update");
+#endif
+
+#ifdef WINDOWS
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_mul, (isaLoader->gf_mul), "gf_mul");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_inv, (isaLoader->gf_inv), "gf_inv");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_rs_matrix, (isaLoader->gf_gen_rs_matrix), "gf_gen_rs_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_gen_cauchy_matrix, (isaLoader->gf_gen_cauchy_matrix), "gf_gen_cauchy1_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_invert_matrix, (isaLoader->gf_invert_matrix), "gf_invert_matrix");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_gf_vect_mul, (isaLoader->gf_vect_mul), "gf_vect_mul");
+
+ EC_LOAD_DYNAMIC_SYMBOL(__d_ec_init_tables, (isaLoader->ec_init_tables), "ec_init_tables");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data, (isaLoader->ec_encode_data), "ec_encode_data");
+ EC_LOAD_DYNAMIC_SYMBOL(__d_ec_encode_data_update, (isaLoader->ec_encode_data_update), "ec_encode_data_update");
+#endif
+
+ return NULL;
+}
+
+void load_erasurecode_lib(char* err, size_t err_len) {
+ const char* errMsg;
+
+ err[0] = '\0';
+
+ if (isaLoader != NULL) {
+ return;
+ }
+ isaLoader = calloc(1, sizeof(IsaLibLoader));
+ memset(isaLoader, 0, sizeof(IsaLibLoader));
+
+ // Load Intel ISA-L
+ #ifdef UNIX
+ isaLoader->libec = dlopen(HADOOP_ISAL_LIBRARY, RTLD_LAZY | RTLD_GLOBAL);
+ if (isaLoader->libec == NULL) {
+ snprintf(err, err_len, "Failed to load %s (%s)",
+ HADOOP_ISAL_LIBRARY, dlerror());
+ return;
+ }
+ // Clear any existing error
+ dlerror();
+ #endif
+
+ #ifdef WINDOWS
+ isaLoader->libec = LoadLibrary(HADOOP_ISAL_LIBRARY);
+ if (isaLoader->libec == NULL) {
+ snprintf(err, err_len, "Failed to load %s", HADOOP_ISAL_LIBRARY);
+ return;
+ }
+ #endif
+
+ errMsg = load_functions(isaLoader->libec);
+ if (errMsg != NULL) {
+ snprintf(err, err_len, "Loading functions from ISA-L failed: %s", errMsg);
+ }
+}
+
+int build_support_erasurecode() {
+#ifdef HADOOP_ISAL_LIBRARY
+ return 1;
+#else
+ return 0;
+#endif
+}
+
+const char* get_library_name() {
+#ifdef UNIX
+ Dl_info dl_info;
+
+ if (isaLoader->ec_encode_data == NULL) {
+ return HADOOP_ISAL_LIBRARY;
+ }
+
+ if(dladdr(isaLoader->ec_encode_data, &dl_info)) {
+ return dl_info.dli_fname;
+ }
+#else
+ LPTSTR filename = NULL;
+
+ if (isaLoader->libec == NULL) {
+ return HADOOP_ISAL_LIBRARY;
+ }
+
+ if (GetModuleFileName(isaLoader->libec, filename, 256) > 0) {
+ return filename;
+ }
+#endif
+
+ return NULL;
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
new file mode 100644
index 00000000000..635706d1b22
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/isal_load.h
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+#include
+
+#include "org_apache_hadoop.h"
+
+#ifdef UNIX
+#include
+#include
+#include
+#include
+
+#include "config.h"
+#endif
+
+#ifdef WINDOWS
+#include
+#endif
+
+#ifndef _ISAL_LOAD_H_
+#define _ISAL_LOAD_H_
+
+
+#ifdef UNIX
+// For gf_util.h
+typedef unsigned char (*__d_gf_mul)(unsigned char, unsigned char);
+typedef unsigned char (*__d_gf_inv)(unsigned char);
+typedef void (*__d_gf_gen_rs_matrix)(unsigned char *, int, int);
+typedef void (*__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
+typedef int (*__d_gf_invert_matrix)(unsigned char *, unsigned char *, const int);
+typedef int (*__d_gf_vect_mul)(int, unsigned char *, void *, void *);
+
+// For erasure_code.h
+typedef void (*__d_ec_init_tables)(int, int, unsigned char*, unsigned char*);
+typedef void (*__d_ec_encode_data)(int, int, int, unsigned char*,
+ unsigned char**, unsigned char**);
+typedef void (*__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
+ unsigned char*, unsigned char**);
+#endif
+
+#ifdef WINDOWS
+// For erasure_code.h
+typedef unsigned char (__cdecl *__d_gf_mul)(unsigned char, unsigned char);
+typedef unsigned char (__cdecl *__d_gf_inv)(unsigned char);
+typedef void (__cdecl *__d_gf_gen_rs_matrix)(unsigned char *, int, int);
+typedef void (__cdecl *__d_gf_gen_cauchy_matrix)(unsigned char *, int, int);
+typedef int (__cdecl *__d_gf_invert_matrix)(unsigned char *,
+ unsigned char *, const int);
+typedef int (__cdecl *__d_gf_vect_mul)(int, unsigned char *, void *, void *);
+
+// For erasure_code.h
+typedef void (__cdecl *__d_ec_init_tables)(int, int,
+ unsigned char*, unsigned char*);
+typedef void (__cdecl *__d_ec_encode_data)(int, int, int, unsigned char*,
+ unsigned char**, unsigned char**);
+typedef void (__cdecl *__d_ec_encode_data_update)(int, int, int, int, unsigned char*,
+ unsigned char*, unsigned char**);
+#endif
+
+typedef struct __IsaLibLoader {
+ // The loaded library handle
+ void* libec;
+
+ __d_gf_mul gf_mul;
+ __d_gf_inv gf_inv;
+ __d_gf_gen_rs_matrix gf_gen_rs_matrix;
+ __d_gf_gen_cauchy_matrix gf_gen_cauchy_matrix;
+ __d_gf_invert_matrix gf_invert_matrix;
+ __d_gf_vect_mul gf_vect_mul;
+ __d_ec_init_tables ec_init_tables;
+ __d_ec_encode_data ec_encode_data;
+ __d_ec_encode_data_update ec_encode_data_update;
+} IsaLibLoader;
+
+extern IsaLibLoader* isaLoader;
+
+/**
+ * A helper function to dlsym a 'symbol' from a given library-handle.
+ */
+
+#ifdef UNIX
+
+static __attribute__ ((unused))
+void *myDlsym(void *handle, const char *symbol) {
+ void *func_ptr = dlsym(handle, symbol);
+ return func_ptr;
+}
+
+/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
+#define EC_LOAD_DYNAMIC_SYMBOL(func_ptr, symbol) \
+ if ((func_ptr = myDlsym(isaLoader->libec, symbol)) == NULL) { \
+ return "Failed to load symbol" symbol; \
+ }
+
+#endif
+
+#ifdef WINDOWS
+
+
+static FARPROC WINAPI myDlsym(HMODULE handle, LPCSTR symbol) {
+ FARPROC func_ptr = GetProcAddress(handle, symbol);
+ return func_ptr;
+}
+
+/* A helper macro to dlsym the requisite dynamic symbol in NON-JNI env. */
+#define EC_LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, symbol) \
+ if ((func_ptr = (func_type)myDlsym(isaLoader->libec, symbol)) == NULL) { \
+ return "Failed to load symbol" symbol; \
+ }
+
+#endif
+
+/**
+ * Return 0 if not support, 1 otherwise.
+ */
+int build_support_erasurecode();
+
+/**
+ * Get the library name possibly of full path.
+ */
+const char* get_library_name();
+
+/**
+ * Initialize and load erasure code library, returning error message if any.
+ *
+ * @param err The err message buffer.
+ * @param err_len The length of the message buffer.
+ */
+void load_erasurecode_lib(char* err, size_t err_len);
+
+#endif //_ISAL_LOAD_H_
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
new file mode 100644
index 00000000000..8126e9af404
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.c
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include
+#include
+#include
+
+#include "org_apache_hadoop.h"
+#include "isal_load.h"
+#include "erasure_code.h"
+#include "jni_common.h"
+
+void loadLib(JNIEnv *env) {
+ char errMsg[1024];
+ load_erasurecode_lib(errMsg, sizeof(errMsg));
+ if (strlen(errMsg) > 0) {
+ THROW(env, "java/lang/UnsatisfiedLinkError", errMsg);
+ }
+}
+
+void setCoder(JNIEnv* env, jobject thiz, IsalCoder* pCoder) {
+ jclass clazz = (*env)->GetObjectClass(env, thiz);
+ jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J");
+ (*env)->SetLongField(env, thiz, __coderState, (jlong) pCoder);
+}
+
+IsalCoder* getCoder(JNIEnv* env, jobject thiz) {
+ jclass clazz = (*env)->GetObjectClass(env, thiz);
+
+ jfieldID __verbose = (*env)->GetFieldID(env, clazz, "__native_verbose", "J");
+ int verbose = (int)(*env)->GetIntField(env, thiz, __verbose);
+
+ jfieldID __coderState = (*env)->GetFieldID(env, clazz, "__native_coder", "J");
+ IsalCoder* pCoder = (IsalCoder*)(*env)->GetLongField(env,
+ thiz, __coderState);
+ pCoder->verbose = verbose;
+
+ return pCoder;
+}
+
+void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets,
+ unsigned char** destInputs, int num) {
+ int numInputs = (*env)->GetArrayLength(env, inputs);
+ int* tmpInputOffsets;
+ int i;
+ jobject byteBuffer;
+
+ if (numInputs != num) {
+ THROW(env, "java/lang/InternalError", "Invalid inputs");
+ }
+
+ tmpInputOffsets = (int*)(*env)->GetIntArrayElements(env,
+ inputOffsets, NULL);
+ for (i = 0; i < numInputs; i++) {
+ byteBuffer = (*env)->GetObjectArrayElement(env, inputs, i);
+ if (byteBuffer != NULL) {
+ destInputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env,
+ byteBuffer));
+ destInputs[i] += tmpInputOffsets[i];
+ } else {
+ destInputs[i] = NULL;
+ }
+ }
+}
+
+void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets,
+ unsigned char** destOutputs, int num) {
+ int numOutputs = (*env)->GetArrayLength(env, outputs);
+ int i, *tmpOutputOffsets;
+ jobject byteBuffer;
+
+ if (numOutputs != num) {
+ THROW(env, "java/lang/InternalError", "Invalid outputs");
+ }
+
+ tmpOutputOffsets = (int*)(*env)->GetIntArrayElements(env,
+ outputOffsets, NULL);
+ for (i = 0; i < numOutputs; i++) {
+ byteBuffer = (*env)->GetObjectArrayElement(env, outputs, i);
+ destOutputs[i] = (unsigned char *)((*env)->GetDirectBufferAddress(env,
+ byteBuffer));
+ destOutputs[i] += tmpOutputOffsets[i];
+ }
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h
new file mode 100644
index 00000000000..a9f62a09086
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_common.h
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _JNI_CODER_COMMON_H_
+#define _JNI_CODER_COMMON_H_
+
+#include
+#include
+#include
+
+#include
+
+#include "erasure_coder.h"
+
+void loadLib(JNIEnv *env);
+
+void setCoder(JNIEnv* env, jobject thiz, IsalCoder* coder);
+
+IsalCoder* getCoder(JNIEnv* env, jobject thiz);
+
+void getInputs(JNIEnv *env, jobjectArray inputs, jintArray inputOffsets,
+ unsigned char** destInputs, int num);
+
+void getOutputs(JNIEnv *env, jobjectArray outputs, jintArray outputOffsets,
+ unsigned char** destOutputs, int num);
+
+#endif //_JNI_CODER_COMMON_H_
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
similarity index 87%
rename from hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c
rename to hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
index e84df9a7310..eb09e73ae5e 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/erasure_code_native.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_erasure_code_native.c
@@ -21,7 +21,7 @@
#include
#include "org_apache_hadoop.h"
-#include "../include/erasure_code.h"
+#include "jni_common.h"
#include "org_apache_hadoop_io_erasurecode_ErasureCodeNative.h"
#ifdef UNIX
@@ -31,11 +31,7 @@
JNIEXPORT void JNICALL
Java_org_apache_hadoop_io_erasurecode_ErasureCodeNative_loadLibrary
(JNIEnv *env, jclass myclass) {
- char errMsg[1024];
- load_erasurecode_lib(errMsg, sizeof(errMsg));
- if (strlen(errMsg) > 0) {
- THROW(env, "java/lang/UnsatisfiedLinkError", errMsg);
- }
+ loadLib(env);
}
JNIEXPORT jstring JNICALL
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
new file mode 100644
index 00000000000..eb4b903da42
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_decoder.c
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+
+#include "org_apache_hadoop.h"
+#include "erasure_code.h"
+#include "gf_util.h"
+#include "jni_common.h"
+#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h"
+
+typedef struct _RSDecoder {
+ IsalDecoder decoder;
+ unsigned char* inputs[MMAX];
+ unsigned char* outputs[MMAX];
+} RSDecoder;
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl(
+JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) {
+ RSDecoder* rsDecoder = (RSDecoder*)malloc(sizeof(RSDecoder));
+ memset(rsDecoder, 0, sizeof(*rsDecoder));
+ initDecoder(&rsDecoder->decoder, (int)numDataUnits, (int)numParityUnits);
+
+ setCoder(env, thiz, &rsDecoder->decoder.coder);
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl(
+JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
+jint dataLen, jintArray erasedIndexes, jobjectArray outputs,
+jintArray outputOffsets) {
+ RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
+
+ int numDataUnits = rsDecoder->decoder.coder.numDataUnits;
+ int numParityUnits = rsDecoder->decoder.coder.numParityUnits;
+ int chunkSize = (int)dataLen;
+
+ int* tmpErasedIndexes = (int*)(*env)->GetIntArrayElements(env,
+ erasedIndexes, NULL);
+ int numErased = (*env)->GetArrayLength(env, erasedIndexes);
+ getInputs(env, inputs, inputOffsets, rsDecoder->inputs,
+ numDataUnits + numParityUnits);
+ getOutputs(env, outputs, outputOffsets, rsDecoder->outputs, numErased);
+
+ decode(&rsDecoder->decoder, rsDecoder->inputs, tmpErasedIndexes,
+ numErased, rsDecoder->outputs, chunkSize);
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl(
+JNIEnv *env, jobject thiz) {
+ RSDecoder* rsDecoder = (RSDecoder*)getCoder(env, thiz);
+ free(rsDecoder);
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
new file mode 100644
index 00000000000..6c477ed4560
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/jni_rs_encoder.c
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include
+#include
+#include
+
+#include "org_apache_hadoop.h"
+#include "erasure_code.h"
+#include "gf_util.h"
+#include "jni_common.h"
+#include "org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h"
+
+typedef struct _RSEncoder {
+ IsalEncoder encoder;
+ unsigned char* inputs[MMAX];
+ unsigned char* outputs[MMAX];
+} RSEncoder;
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl(
+JNIEnv *env, jobject thiz, jint numDataUnits, jint numParityUnits) {
+ RSEncoder* rsEncoder = (RSEncoder*)malloc(sizeof(RSEncoder));
+ memset(rsEncoder, 0, sizeof(*rsEncoder));
+ initEncoder(&rsEncoder->encoder, (int)numDataUnits, (int)numParityUnits);
+
+ setCoder(env, thiz, &rsEncoder->encoder.coder);
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl(
+JNIEnv *env, jobject thiz, jobjectArray inputs, jintArray inputOffsets,
+jint dataLen, jobjectArray outputs, jintArray outputOffsets) {
+ RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
+
+ int numDataUnits = rsEncoder->encoder.coder.numDataUnits;
+ int numParityUnits = rsEncoder->encoder.coder.numParityUnits;
+ int chunkSize = (int)dataLen;
+
+ getInputs(env, inputs, inputOffsets, rsEncoder->inputs, numDataUnits);
+ getOutputs(env, outputs, outputOffsets, rsEncoder->outputs, numParityUnits);
+
+ encode(&rsEncoder->encoder, rsEncoder->inputs, rsEncoder->outputs, chunkSize);
+}
+
+JNIEXPORT void JNICALL
+Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl(
+JNIEnv *env, jobject thiz) {
+ RSEncoder* rsEncoder = (RSEncoder*)getCoder(env, thiz);
+ free(rsEncoder);
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h
similarity index 100%
rename from hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/coder/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h
rename to hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_ErasureCodeNative.h
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h
new file mode 100644
index 00000000000..40da4e1f59b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder.h
@@ -0,0 +1,37 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include
+/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder */
+
+#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
+#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
+ * Method: initImpl
+ * Signature: (II[I)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_initImpl
+ (JNIEnv *, jobject, jint, jint);
+
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
+ * Method: decodeImpl
+ * Signature: ([Ljava/nio/ByteBuffer;[II[I[Ljava/nio/ByteBuffer;[I)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_decodeImpl
+ (JNIEnv *, jobject, jobjectArray, jintArray, jint, jintArray, jobjectArray, jintArray);
+
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder
+ * Method: destroyImpl
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawDecoder_destroyImpl
+ (JNIEnv *, jobject);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h
new file mode 100644
index 00000000000..db094cf1f66
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder.h
@@ -0,0 +1,37 @@
+/* DO NOT EDIT THIS FILE - it is machine generated */
+#include
+/* Header for class org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder */
+
+#ifndef _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
+#define _Included_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
+ * Method: initImpl
+ * Signature: (II[I)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_initImpl
+ (JNIEnv *, jobject, jint, jint);
+
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
+ * Method: encodeImpl
+ * Signature: ([Ljava/nio/ByteBuffer;[II[Ljava/nio/ByteBuffer;[I)V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_encodeImpl
+ (JNIEnv *, jobject, jobjectArray, jintArray, jint, jobjectArray, jintArray);
+
+/*
+ * Class: org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder
+ * Method: destroyImpl
+ * Signature: ()V
+ */
+JNIEXPORT void JNICALL Java_org_apache_hadoop_io_erasurecode_rawcoder_NativeRSRawEncoder_destroyImpl
+ (JNIEnv *, jobject);
+
+#ifdef __cplusplus
+}
+#endif
+#endif
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c
index 9817a7685ba..331bb219b7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c
@@ -17,185 +17,36 @@
*/
/**
- * This is a lightweight version of the same file in Intel ISA-L library to test
- * and verify the basic functions of ISA-L integration. Note it's not serving as
- * a complete ISA-L library test nor as any sample to write an erasure coder
- * using the library. A sample is to be written and provided separately.
+ * This is a sample program illustrating how to use the Intel ISA-L library.
+ * Note it's adapted from erasure_code_test.c test program, but trying to use
+ * variable names and styles we're more familiar with already similar to Java
+ * coders.
*/
-#include "org_apache_hadoop.h"
+#include "isal_load.h"
#include "erasure_code.h"
#include "gf_util.h"
+#include "erasure_coder.h"
#include
#include
#include
-#define TEST_LEN 8192
-#define TEST_SOURCES 127
-#define MMAX TEST_SOURCES
-#define KMAX TEST_SOURCES
-#define TEST_SEED 11
-
-static void dump(unsigned char *buf, int len)
-{
- int i;
- for (i = 0; i < len;) {
- printf(" %2x", 0xff & buf[i++]);
- if (i % 32 == 0)
- printf("\n");
- }
- printf("\n");
-}
-
-static void dump_matrix(unsigned char **s, int k, int m)
-{
+int main(int argc, char *argv[]) {
int i, j;
- for (i = 0; i < k; i++) {
- for (j = 0; j < m; j++) {
- printf(" %2x", s[i][j]);
- }
- printf("\n");
- }
- printf("\n");
-}
-
-static void dump_u8xu8(unsigned char *s, int k, int m)
-{
- int i, j;
- for (i = 0; i < k; i++) {
- for (j = 0; j < m; j++) {
- printf(" %2x", 0xff & s[j + (i * m)]);
- }
- printf("\n");
- }
- printf("\n");
-}
-
-// Generate Random errors
-static void gen_err_list(unsigned char *src_err_list,
- unsigned char *src_in_err, int *pnerrs, int *pnsrcerrs, int k, int m)
-{
- int i, err;
- int nerrs = 0, nsrcerrs = 0;
-
- for (i = 0, nerrs = 0, nsrcerrs = 0; i < m && nerrs < m - k; i++) {
- err = 1 & rand();
- src_in_err[i] = err;
- if (err) {
- src_err_list[nerrs++] = i;
- if (i < k) {
- nsrcerrs++;
- }
- }
- }
- if (nerrs == 0) { // should have at least one error
- while ((err = (rand() % KMAX)) >= m) ;
- src_err_list[nerrs++] = err;
- src_in_err[err] = 1;
- if (err < k)
- nsrcerrs = 1;
- }
- *pnerrs = nerrs;
- *pnsrcerrs = nsrcerrs;
- return;
-}
-
-#define NO_INVERT_MATRIX -2
-// Generate decode matrix from encode matrix
-static int gf_gen_decode_matrix(unsigned char *encode_matrix,
- unsigned char *decode_matrix,
- unsigned char *invert_matrix,
- unsigned int *decode_index,
- unsigned char *src_err_list,
- unsigned char *src_in_err,
- int nerrs, int nsrcerrs, int k, int m)
-{
- int i, j, p;
- int r;
- unsigned char *backup, *b, s;
- int incr = 0;
-
- b = malloc(MMAX * KMAX);
- backup = malloc(MMAX * KMAX);
-
- if (b == NULL || backup == NULL) {
- printf("Test failure! Error with malloc\n");
- free(b);
- free(backup);
- return -1;
- }
- // Construct matrix b by removing error rows
- for (i = 0, r = 0; i < k; i++, r++) {
- while (src_in_err[r])
- r++;
- for (j = 0; j < k; j++) {
- b[k * i + j] = encode_matrix[k * r + j];
- backup[k * i + j] = encode_matrix[k * r + j];
- }
- decode_index[i] = r;
- }
- incr = 0;
- while (h_gf_invert_matrix(b, invert_matrix, k) < 0) {
- if (nerrs == (m - k)) {
- free(b);
- free(backup);
- printf("BAD MATRIX\n");
- return NO_INVERT_MATRIX;
- }
- incr++;
- memcpy(b, backup, MMAX * KMAX);
- for (i = nsrcerrs; i < nerrs - nsrcerrs; i++) {
- if (src_err_list[i] == (decode_index[k - 1] + incr)) {
- // skip the erased parity line
- incr++;
- continue;
- }
- }
- if (decode_index[k - 1] + incr >= m) {
- free(b);
- free(backup);
- printf("BAD MATRIX\n");
- return NO_INVERT_MATRIX;
- }
- decode_index[k - 1] += incr;
- for (j = 0; j < k; j++)
- b[k * (k - 1) + j] = encode_matrix[k * decode_index[k - 1] + j];
-
- };
-
- for (i = 0; i < nsrcerrs; i++) {
- for (j = 0; j < k; j++) {
- decode_matrix[k * i + j] = invert_matrix[k * src_err_list[i] + j];
- }
- }
- /* src_err_list from encode_matrix * invert of b for parity decoding */
- for (p = nsrcerrs; p < nerrs; p++) {
- for (i = 0; i < k; i++) {
- s = 0;
- for (j = 0; j < k; j++)
- s ^= h_gf_mul(invert_matrix[j * k + i],
- encode_matrix[k * src_err_list[p] + j]);
-
- decode_matrix[k * p + i] = s;
- }
- }
- free(b);
- free(backup);
- return 0;
-}
-
-int main(int argc, char *argv[])
-{
char err[256];
size_t err_len = sizeof(err);
- int re, i, j, p, m, k;
- int nerrs, nsrcerrs;
- unsigned int decode_index[MMAX];
- unsigned char *temp_buffs[TEST_SOURCES], *buffs[TEST_SOURCES];
- unsigned char *encode_matrix, *decode_matrix, *invert_matrix, *g_tbls;
- unsigned char src_in_err[TEST_SOURCES], src_err_list[TEST_SOURCES];
- unsigned char *recov[TEST_SOURCES];
+ int chunkSize = 1024;
+ int numDataUnits = 6;
+ int numParityUnits = 3;
+ unsigned char** dataUnits;
+ unsigned char** parityUnits;
+ IsalEncoder* pEncoder;
+ int erasedIndexes[2];
+ unsigned char* allUnits[MMAX];
+ IsalDecoder* pDecoder;
+ unsigned char* decodingOutput[2];
+ unsigned char** backupUnits;
if (0 == build_support_erasurecode()) {
printf("The native library isn't available, skipping this test\n");
@@ -209,102 +60,66 @@ int main(int argc, char *argv[])
}
printf("Performing erasure code test\n");
- srand(TEST_SEED);
- // Allocate the arrays
- for (i = 0; i < TEST_SOURCES; i++) {
- buffs[i] = malloc(TEST_LEN);
+ dataUnits = calloc(numDataUnits, sizeof(unsigned char*));
+ parityUnits = calloc(numParityUnits, sizeof(unsigned char*));
+ backupUnits = calloc(numParityUnits, sizeof(unsigned char*));
+
+ // Allocate and generate data units
+ srand(135);
+ for (i = 0; i < numDataUnits; i++) {
+ dataUnits[i] = calloc(chunkSize, sizeof(unsigned char));
+ for (j = 0; j < chunkSize; j++) {
+ dataUnits[i][j] = rand();
+ }
}
- for (i = 0; i < TEST_SOURCES; i++) {
- temp_buffs[i] = malloc(TEST_LEN);
+ // Allocate and initialize parity units
+ for (i = 0; i < numParityUnits; i++) {
+ parityUnits[i] = calloc(chunkSize, sizeof(unsigned char));
+ for (j = 0; j < chunkSize; j++) {
+ parityUnits[i][j] = 0;
+ }
}
- // Test erasure code by encode and recovery
+ pEncoder = (IsalEncoder*)malloc(sizeof(IsalEncoder));
+ memset(pEncoder, 0, sizeof(*pEncoder));
+ initEncoder(pEncoder, numDataUnits, numParityUnits);
+ encode(pEncoder, dataUnits, parityUnits, chunkSize);
- encode_matrix = malloc(MMAX * KMAX);
- decode_matrix = malloc(MMAX * KMAX);
- invert_matrix = malloc(MMAX * KMAX);
- g_tbls = malloc(KMAX * TEST_SOURCES * 32);
- if (encode_matrix == NULL || decode_matrix == NULL
- || invert_matrix == NULL || g_tbls == NULL) {
- snprintf(err, err_len, "%s", "allocating test matrix buffers error");
- return -1;
- }
+ pDecoder = (IsalDecoder*)malloc(sizeof(IsalDecoder));
+ memset(pDecoder, 0, sizeof(*pDecoder));
+ initDecoder(pDecoder, numDataUnits, numParityUnits);
- m = 9;
- k = 5;
- if (m > MMAX || k > KMAX)
- return -1;
+ memcpy(allUnits, dataUnits, numDataUnits * (sizeof (unsigned char*)));
+ memcpy(allUnits + numDataUnits, parityUnits,
+ numParityUnits * (sizeof (unsigned char*)));
- // Make random data
- for (i = 0; i < k; i++)
- for (j = 0; j < TEST_LEN; j++)
- buffs[i][j] = rand();
+ erasedIndexes[0] = 1;
+ erasedIndexes[1] = 7;
- // The matrix generated by gf_gen_cauchy1_matrix
- // is always invertable.
- h_gf_gen_cauchy_matrix(encode_matrix, m, k);
+ backupUnits[0] = allUnits[1];
+ backupUnits[1] = allUnits[7];
- // Generate g_tbls from encode matrix encode_matrix
- h_ec_init_tables(k, m - k, &encode_matrix[k * k], g_tbls);
+ allUnits[0] = NULL; // Not to read
+ allUnits[1] = NULL;
+ allUnits[7] = NULL;
- // Perform matrix dot_prod for EC encoding
- // using g_tbls from encode matrix encode_matrix
- h_ec_encode_data(TEST_LEN, k, m - k, g_tbls, buffs, &buffs[k]);
+ decodingOutput[0] = malloc(chunkSize);
+ decodingOutput[1] = malloc(chunkSize);
- // Choose random buffers to be in erasure
- memset(src_in_err, 0, TEST_SOURCES);
- gen_err_list(src_err_list, src_in_err, &nerrs, &nsrcerrs, k, m);
-
- // Generate decode matrix
- re = gf_gen_decode_matrix(encode_matrix, decode_matrix,
- invert_matrix, decode_index, src_err_list, src_in_err,
- nerrs, nsrcerrs, k, m);
- if (re != 0) {
- snprintf(err, err_len, "%s", "gf_gen_decode_matrix failed");
- return -1;
- }
- // Pack recovery array as list of valid sources
- // Its order must be the same as the order
- // to generate matrix b in gf_gen_decode_matrix
- for (i = 0; i < k; i++) {
- recov[i] = buffs[decode_index[i]];
- }
-
- // Recover data
- h_ec_init_tables(k, nerrs, decode_matrix, g_tbls);
- h_ec_encode_data(TEST_LEN, k, nerrs, g_tbls, recov, &temp_buffs[k]);
- for (i = 0; i < nerrs; i++) {
- if (0 != memcmp(temp_buffs[k + i], buffs[src_err_list[i]], TEST_LEN)) {
- snprintf(err, err_len, "%s", "Error recovery failed");
- printf("Fail error recovery (%d, %d, %d)\n", m, k, nerrs);
-
- printf(" - erase list = ");
- for (j = 0; j < nerrs; j++) {
- printf(" %d", src_err_list[j]);
- }
-
- printf(" - Index = ");
- for (p = 0; p < k; p++) {
- printf(" %d", decode_index[p]);
- }
-
- printf("\nencode_matrix:\n");
- dump_u8xu8((unsigned char *) encode_matrix, m, k);
- printf("inv b:\n");
- dump_u8xu8((unsigned char *) invert_matrix, k, k);
- printf("\ndecode_matrix:\n");
- dump_u8xu8((unsigned char *) decode_matrix, m, k);
- printf("recov %d:", src_err_list[i]);
- dump(temp_buffs[k + i], 25);
- printf("orig :");
- dump(buffs[src_err_list[i]], 25);
+ decode(pDecoder, allUnits, erasedIndexes, 2, decodingOutput, chunkSize);
+ for (i = 0; i < pDecoder->numErased; i++) {
+ if (0 != memcmp(decodingOutput[i], backupUnits[i], chunkSize)) {
+ fprintf(stderr, "Decoding failed\n\n");
+ dumpDecoder(pDecoder);
return -1;
}
}
- printf("done EC tests: Pass\n");
+ dumpDecoder(pDecoder);
+ fprintf(stdout, "Successfully done, passed!\n\n");
+
return 0;
}