from os.path import join
import numpy as np
def subdivide_matrix_into_files(conf, mat, file_dir, file_prefix):
def _write_submatrix_to_file(sm_bytes, row_idx, col_idx):
file_name = conf.get_submatrix_key(
file_prefix, conf.n_splits, row_idx, col_idx
)
file_path = join(file_dir, file_name)
with open(file_path, "wb") as fh:
fh.write(sm_bytes)
do_subdivide_matrix(conf, mat, _write_submatrix_to_file)
def reconstruct_matrix_from_files(conf, file_dir, file_prefix):
def _read_submatrix_from_file(row_idx, col_idx):
file_name = conf.get_submatrix_key(
file_prefix, conf.n_splits, row_idx, col_idx
)
file_path = join(file_dir, file_name)
with open(file_path, "rb") as fh:
file_bytes = fh.read()
return file_bytes
return do_reconstruct_matrix(conf, _read_submatrix_from_file)
def do_subdivide_matrix(conf, mat, write_func):
# Step through rows and columns of original matrix, appending submatrix
# bytes to the overall byte stream
sm_per_row = conf.get_submatrices_per_row(conf.n_splits)
sm_size = conf.get_submatrix_size(conf.n_splits)
for row_idx in range(0, sm_per_row):
for col_idx in range(0, sm_per_row):
# Work out the position of the top left and bottom right corner of
# the submatrix
row_start = row_idx * sm_size
col_start = col_idx * sm_size
row_end = row_start + sm_size
col_end = col_start + sm_size
# Extract the submatrix and write to bytes
sub_mat = mat[row_start:row_end, col_start:col_end]
sm_bytes = sub_mat.tobytes()
write_func(sm_bytes, row_idx, col_idx)
def do_reconstruct_matrix(conf, read_func):
result = None
sm_per_row = conf.get_submatrices_per_row(conf.n_splits)
sm_size = conf.get_submatrix_size(conf.n_splits)
# Need to read in row by row concatenate the rows
for row_idx in range(0, sm_per_row):
submatrices = []
for col_idx in range(0, sm_per_row):
sm_data = read_func(row_idx, col_idx)
this_submat = np.frombuffer(sm_data, dtype=np.float32)
this_submat = this_submat.reshape(sm_size, sm_size)
submatrices.append(this_submat)
this_row = np.concatenate(submatrices, axis=1)
if row_idx == 0:
# Initialise the result
result = this_row
else:
# Add the row to the existing result
result = np.concatenate((result, this_row), axis=0)
return result