/*
* Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this
* list of conditions and the following disclaimer in the documentation and/or
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* liberasurecode proprocssing helpers implementation
*
* vi: set noai tw=79 ts=4 sw=4:
*/
#include "erasurecode_preprocessing.h"
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
#include "erasurecode_helpers_ext.h"
#include "erasurecode_log.h"
#include "erasurecode_stdinc.h"
__attribute__((visibility("internal"))) int prepare_fragments_for_encode(ec_backend_t instance,
int k, int m, const char *orig_data, uint64_t orig_data_size, /* input */
char **encoded_data, char **encoded_parity, /* output */
int *blocksize)
{
int i, ret = 0;
int data_len; /* data len to write to fragment headers */
int aligned_data_len; /* EC algorithm compatible data length */
int buffer_size, payload_size = 0;
int metadata_size, data_offset = 0;
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = payload_size = (aligned_data_len / k);
metadata_size
= instance->common.ops->get_backend_metadata_size(instance->desc.backend_desc, *blocksize);
data_offset
= instance->common.ops->get_encode_offset(instance->desc.backend_desc, metadata_size);
buffer_size = payload_size + metadata_size;
for (i = 0; i < k; i++) {
int copy_size = data_len > payload_size ? payload_size : data_len;
char *fragment = (char *)alloc_fragment_buffer(buffer_size);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
}
/* Copy existing data into clean, zero'd out buffer */
encoded_data[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(encoded_data[i] + data_offset, orig_data, copy_size);
}
orig_data += copy_size;
data_len -= copy_size;
}
for (i = 0; i < m; i++) {
char *fragment = (char *)alloc_fragment_buffer(buffer_size);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
}
encoded_parity[i] = get_data_ptr_from_fragment(fragment);
}
out:
return ret;
out_error:
printf("ERROR in encode\n");
if (encoded_data) {
for (i = 0; i < k; i++) {
if (encoded_data[i])
free_fragment_buffer(encoded_data[i]);
}
check_and_free_buffer(encoded_data);
}
if (encoded_parity) {
for (i = 0; i < m; i++) {
if (encoded_parity[i])
free_fragment_buffer(encoded_parity[i]);
}
check_and_free_buffer(encoded_parity);
}
goto out;
}
/*
* Note that the caller should always check realloc_bm during success or
* failure to free buffers allocated here. We could free up in this function,
* but it is internal to this library and only used in a few places. In any
* case, the caller has to free up in the success case, so it may as well do
* so in the failure case.
*/
__attribute__((visibility("internal"))) int prepare_fragments_for_decode(int k, int m, char **data,
char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size,
struct ec_bm *realloc_bm)
{
int i; /* a counter */
struct ec_bm missing_bm = NEW_BM; /* bitmap form of missing indexes list */
int orig_data_size = -1;
int payload_size = -1;
convert_list_to_bitmap(missing_idxs, &missing_bm);
/*
* Determine if each data fragment is:
* 1.) Alloc'd: if not, alloc new buffer (for missing fragments)
* 2.) Aligned to 16-byte boundaries: if not, alloc a new buffer
* memcpy the contents and free the old buffer
*/
for (i = 0; i < k; i++) {
/*
* Allocate or replace with aligned buffer if the buffer was not
* aligned.
* DO NOT FREE: the python GC should free the original when cleaning up
* 'data_list'
*/
if (NULL == data[i]) {
data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == data[i]) {
log_error("Could not allocate data buffer!");
return -ENOMEM;
}
bm_set_value(realloc_bm, i, 1);
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -ENOMEM;
}
memcpy(tmp_buf, data[i], fragment_size);
data[i] = tmp_buf;
bm_set_value(realloc_bm, i, 1);
}
/* Need to determine the size of the original data */
if (!bm_get_value(&missing_bm, i) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(data[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
return -EBADHEADER;
}
payload_size = get_fragment_payload_size(data[i]);
if (payload_size < 0) {
log_error("Invalid fragment_size in fragment header!");
return -EBADHEADER;
}
}
}
/* Perform the same allocation, alignment checks on the parity fragments */
for (i = 0; i < m; i++) {
/*
* Allocate or replace with aligned buffer, if the buffer was not aligned.
* DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
*/
if (NULL == parity[i]) {
parity[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == parity[i]) {
log_error("Could not allocate parity buffer!");
return -ENOMEM;
}
bm_set_value(realloc_bm, k + i, 1);
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -ENOMEM;
}
memcpy(tmp_buf, parity[i], fragment_size);
parity[i] = tmp_buf;
bm_set_value(realloc_bm, k + i, 1);
}
/* Need to determine the size of the original data */
if (!bm_get_value(&missing_bm, k + i) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
return -EBADHEADER;
}
payload_size = get_fragment_payload_size(parity[i]);
if (payload_size < 0) {
log_error("Invalid fragment_size in fragment header!");
return -EBADHEADER;
}
}
}
*orig_size = orig_data_size;
*fragment_payload_size = payload_size;
return 0;
}
int get_fragment_partition(
int k, int m, char **fragments, int num_fragments, char **data, char **parity, int *missing)
{
int i = 0;
int num_missing = 0;
int index;
/*
* Set all data and parity entries to NULL
*/
for (i = 0; i < k; i++) {
data[i] = NULL;
}
for (i = 0; i < m; i++) {
parity[i] = NULL;
}
/*
* Fill in data and parity with available fragments
*/
for (i = 0; i < num_fragments; i++) {
index = get_fragment_idx(fragments[i]);
if (index < 0 || index >= (k + m)) {
return -EBADHEADER;
}
if (index < k) {
data[index] = fragments[i];
} else {
parity[index - k] = fragments[i];
}
}
/*
* Fill in missing array with missing indexes
*/
for (i = 0; i < k; i++) {
if (NULL == data[i]) {
missing[num_missing] = i;
num_missing++;
}
}
for (i = 0; i < m; i++) {
if (NULL == parity[i]) {
missing[num_missing] = i + k;
num_missing++;
}
}
return 0;
}
__attribute__((visibility("internal"))) int fragments_to_string(
int k, int m, char **fragments, int num_fragments, char **orig_payload, uint64_t *payload_len)
{
char *internal_payload = NULL;
char **data = NULL;
int orig_data_size = -1;
int i;
int index;
int data_size;
int num_data = 0;
int string_off = 0;
int ret = -1;
if (num_fragments < k) {
/*
* This is not necessarily an error condition, so *do not log here*
* We can maybe debug log, if necessary.
*/
goto out;
}
data = (char **)get_aligned_buffer16(sizeof(char *) * k);
if (NULL == data) {
log_error("Could not allocate buffer for data!!");
ret = -ENOMEM;
goto out;
}
for (i = 0; i < num_fragments; i++) {
index = get_fragment_idx(fragments[i]);
data_size = get_fragment_payload_size(fragments[i]);
if ((index < 0) || (data_size < 0)) {
log_error("Invalid fragment header information!");
ret = -EBADHEADER;
goto out;
}
/* Validate the original data size */
if (orig_data_size < 0) {
orig_data_size = get_orig_data_size(fragments[i]);
} else {
if (get_orig_data_size(fragments[i]) != orig_data_size) {
log_error("Inconsistent orig_data_size in fragment header!");
ret = -EBADHEADER;
goto out;
}
}
/* Skip parity fragments, put data fragments in index order */
if (index >= k) {
continue;
} else {
/* Make sure we account for duplicates */
if (NULL == data[index]) {
data[index] = fragments[i];
num_data++;
}
}
}
/* We do not have enough data fragments to do this! */
if (num_data != k) {
/*
* This is not necessarily an error condition, so *do not log here*
* We can maybe debug log, if necessary.
*/
goto out;
}
/* Create the string to return */
internal_payload = (char *)get_aligned_buffer16(orig_data_size);
if (NULL == internal_payload) {
log_error("Could not allocate buffer for decoded string!");
ret = -ENOMEM;
goto out;
}
/* Pass the original data length back */
*payload_len = orig_data_size;
/* Copy fragment data into cstring (fragments should be in index order) */
for (i = 0; i < num_data && orig_data_size > 0; i++) {
char *fragment_data = get_data_ptr_from_fragment(data[i]);
int fragment_size = get_fragment_payload_size(data[i]);
int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
memcpy(internal_payload + string_off, fragment_data, payload_size);
orig_data_size -= payload_size;
string_off += payload_size;
}
/* Everything worked just fine */
ret = 0;
out:
if (NULL != data) {
free(data);
}
*orig_payload = internal_payload;
return ret;
}