diff --git a/Compiler/instructions.py b/Compiler/instructions.py index bc3f1d48..e5679eea 100644 --- a/Compiler/instructions.py +++ b/Compiler/instructions.py @@ -1989,6 +1989,7 @@ class closeclientconnection(base.IOInstruction): code = base.opcodes['CLOSECLIENTCONNECTION'] arg_format = ['ci'] +@base.gf2n class writesharestofile(base.VectorInstruction, base.IOInstruction): """ Write shares to ``Persistence/Transactions-P.data`` (appending at the end). @@ -2001,12 +2002,13 @@ class writesharestofile(base.VectorInstruction, base.IOInstruction): """ __slots__ = [] code = base.opcodes['WRITEFILESHARE'] - arg_format = tools.chain(['ci'], itertools.repeat('s')) + arg_format = tools.chain(['ci'], tools.cycle(['s'])) vector_index = 1 def has_var_args(self): return True +@base.gf2n class readsharesfromfile(base.VectorInstruction, base.IOInstruction): """ Read shares from ``Persistence/Transactions-P.data``. @@ -2018,7 +2020,7 @@ class readsharesfromfile(base.VectorInstruction, base.IOInstruction): """ __slots__ = [] code = base.opcodes['READFILESHARE'] - arg_format = tools.chain(['ci', 'ciw'], itertools.repeat('sw')) + arg_format = tools.chain(['ci', 'ciw'], tools.cycle(['sw'])) vector_index = 2 def has_var_args(self): diff --git a/Compiler/instructions_base.py b/Compiler/instructions_base.py index 5abbf97b..552ef817 100644 --- a/Compiler/instructions_base.py +++ b/Compiler/instructions_base.py @@ -369,7 +369,7 @@ def gf2n(instruction): if isinstance(arg_format, list): __format = [] for __f in arg_format: - if __f in ('int', 'long', 'p', 'ci', 'str'): + if __f in ('int', 'long', 'p', 'ci', 'ciw', 'str'): __format.append(__f) else: __format.append(__f[0] + 'g' + __f[1:]) diff --git a/Compiler/types.py b/Compiler/types.py index 1c82c8a7..c97a07ee 100644 --- a/Compiler/types.py +++ b/Compiler/types.py @@ -2430,6 +2430,47 @@ class _secret(_arithmetic_register, _secret_structure): def output(self): print_reg_plains(self) + @classmethod + @set_instruction_type + def read_from_file(cls, start, n_items=1, crash_if_missing=True, size=1): + """ Read shares from + ``Persistence/Transactions-[gf2n-]P.data``. See :ref:`this + section ` for details on the data format. + + :param start: starting position in number of shares from beginning (int/regint/cint) + :param n_items: number of items (int) + :param crash_if_missing: crash if file not found (default) + :param size: vector size (int) + :returns: destination for final position, -1 for eof reached, or -2 for file not found (regint) + :returns: list of shares + """ + shares = [cls(size=size) for i in range(n_items)] + stop = regint() + readsharesfromfile(regint.conv(start), stop, *shares) + if crash_if_missing: + library.runtime_error_if(stop == -2, 'Persistence not found') + return stop, shares + + @classmethod + @set_instruction_type + def write_to_file(cls, shares, position=None): + """ Write shares to ``Persistence/Transactions-[gf2n-]P.data`` + (appending at the end). See :ref:`this section ` + for details on the data format. + + :param shares: (list or iterable of shares) + :param position: start position (int/regint/cint), + defaults to end of file + """ + if isinstance(shares, cls): + shares = [shares] + for share in shares: + assert isinstance(share, cls) + assert share.size == shares[0].size + if position is None: + position = -1 + writesharestofile(regint.conv(position), *shares) + class sint(_secret, _int): """ Secret integer in the protocol-specific domain. It supports @@ -2699,45 +2740,6 @@ class sint(_secret, _int): """ writesocketshare(client_id, message_type, values[0].size, *values) - @classmethod - def read_from_file(cls, start, n_items=1, crash_if_missing=True, size=1): - """ Read shares from - ``Persistence/Transactions-P.data``. See :ref:`this - section ` for details on the data format. - - :param start: starting position in number of shares from beginning (int/regint/cint) - :param n_items: number of items (int) - :param crash_if_missing: crash if file not found (default) - :param size: vector size (int) - :returns: destination for final position, -1 for eof reached, or -2 for file not found (regint) - :returns: list of shares - """ - shares = [cls(size=size) for i in range(n_items)] - stop = regint() - readsharesfromfile(regint.conv(start), stop, *shares) - if crash_if_missing: - library.runtime_error_if(stop == -2, 'Persistence not found') - return stop, shares - - @staticmethod - def write_to_file(shares, position=None): - """ Write shares to ``Persistence/Transactions-P.data`` - (appending at the end). See :ref:`this section ` - for details on the data format. - - :param shares: (list or iterable of sint) - :param position: start position (int/regint/cint), - defaults to end of file - """ - if isinstance(shares, sint): - shares = [shares] - for share in shares: - assert isinstance(share, sint) - assert share.size == shares[0].size - if position is None: - position = -1 - writesharestofile(regint.conv(position), *shares) - @vectorized_classmethod def load_mem(cls, address, mem_type=None): """ Load from memory by public address. """ diff --git a/Processor/Binary_File_IO.h b/Processor/Binary_File_IO.h index 5ed62ff0..27648f07 100644 --- a/Processor/Binary_File_IO.h +++ b/Processor/Binary_File_IO.h @@ -16,17 +16,18 @@ using namespace std; * Intended for MPC application specific file IO. */ +template class Binary_File_IO { public: static string filename(int my_number); + static void reset(int my_number); /* * Append the buffer values as binary to the filename. * Throws file_error. */ - template void write_to_file(const string filename, const vector& buffer, long start_pos); @@ -36,7 +37,6 @@ class Binary_File_IO * Returns the current posn in the file or -1 if at eof. * Throws file_error. */ - template void read_from_file(const string filename, vector& buffer, const long start_posn, long& end_posn); }; diff --git a/Processor/Binary_File_IO.hpp b/Processor/Binary_File_IO.hpp index 8b74d9d5..bfa9bb84 100644 --- a/Processor/Binary_File_IO.hpp +++ b/Processor/Binary_File_IO.hpp @@ -6,16 +6,35 @@ * Intended for application specific file IO. */ -inline string Binary_File_IO::filename(int my_number) +template +inline string Binary_File_IO::filename(int my_number) { string dir = "Persistence"; mkdir_p(dir.c_str()); - return dir + "/Transactions-P" + to_string(my_number) + ".data"; + string res = dir + "/Transactions"; + if (T::clear::characteristic_two) + res += "-gf2n"; + return res + "-P" + to_string(my_number) + ".data"; } -template +template +void Binary_File_IO::reset(int my_number) +{ + string filename = Binary_File_IO::filename(my_number); + ifstream pers(filename); + try + { + check_file_signature(pers, filename); + } + catch (signature_mismatch&) + { + ofstream pers(filename, ios::binary); + file_signature().output(pers); + } +} -void Binary_File_IO::write_to_file(const string filename, +template +void Binary_File_IO::write_to_file(const string filename, const vector& buffer, long start_pos) { ofstream outf; @@ -44,7 +63,7 @@ void Binary_File_IO::write_to_file(const string filename, } template -void Binary_File_IO::read_from_file(const string filename, vector& buffer, +void Binary_File_IO::read_from_file(const string filename, vector& buffer, const long start_posn, long& end_posn) { ifstream inf; diff --git a/Processor/Instruction.h b/Processor/Instruction.h index 965789e6..ba45f65c 100644 --- a/Processor/Instruction.h +++ b/Processor/Instruction.h @@ -311,6 +311,8 @@ enum GRAWOUTPUT = 0x1B7, GSTARTPRIVATEOUTPUT = 0x1B8, GSTOPPRIVATEOUTPUT = 0x1B9, + GWRITEFILESHARE = 0x1BD, + GREADFILESHARE = 0x1BE, // Commsec ops INITSECURESOCKET = 0x1BA, RESPSECURESOCKET = 0x1BB diff --git a/Processor/Instruction.hpp b/Processor/Instruction.hpp index 24a77ec9..a963acb3 100644 --- a/Processor/Instruction.hpp +++ b/Processor/Instruction.hpp @@ -328,6 +328,7 @@ void BaseInstruction::parse_operands(istream& s, int pos, int file_pos) // read from file, input is opcode num_args, // start_file_posn (read), end_file_posn(write) var1, var2, ... case READFILESHARE: + case GREADFILESHARE: case CALL_TAPE: num_var_args = get_int(s) - 2; r[0] = get_int(s); @@ -397,6 +398,7 @@ void BaseInstruction::parse_operands(istream& s, int pos, int file_pos) case EDABIT: case SEDABIT: case WRITEFILESHARE: + case GWRITEFILESHARE: case CONCATS: num_var_args = get_int(s) - 1; r[0] = get_int(s); @@ -1371,11 +1373,21 @@ inline void Instruction::execute(Processor& Proc) const break; case WRITEFILESHARE: // Write shares to file system - Proc.write_shares_to_file(Proc.read_Ci(r[0]), start, size); + Procp.write_shares_to_file(Proc.read_Ci(r[0]), start, size); return; case READFILESHARE: // Read shares from file system - Proc.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size); + Procp.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size, + Proc); + return; + case GWRITEFILESHARE: + // Write shares to file system + Proc2.write_shares_to_file(Proc.read_Ci(r[0]), start, size); + return; + case GREADFILESHARE: + // Read shares from file system + Proc2.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size, + Proc); return; case PUBINPUT: Proc.get_Cp_ref(r[0]) = Proc.template diff --git a/Processor/Machine.hpp b/Processor/Machine.hpp index 5c72ffc3..dbfb1bd6 100644 --- a/Processor/Machine.hpp +++ b/Processor/Machine.hpp @@ -165,17 +165,8 @@ void Machine::prepare(const string& progname_str) { if (prog.writes_persistence) { - string filename = Binary_File_IO::filename(my_number); - ifstream pers(filename); - try - { - check_file_signature(pers, filename); - } - catch (signature_mismatch&) - { - ofstream pers(filename, ios::binary); - file_signature().output(pers); - } + Binary_File_IO::reset(my_number); + Binary_File_IO::reset(my_number); break; } } diff --git a/Processor/Processor.h b/Processor/Processor.h index d6ab05a6..522f321e 100644 --- a/Processor/Processor.h +++ b/Processor/Processor.h @@ -38,6 +38,8 @@ class SubProcessor DataPositions bit_usage; NamedStats stats; + Binary_File_IO binary_file_io; + void resize(size_t size) { C.resize(size); S.resize(size); } void matmulsm_prep(int ii, int j, const MemoryPart& source, @@ -128,6 +130,13 @@ public: void push_stack(); void push_args(const vector& args); void pop_stack(const vector& results); + + // Read and write secret numeric data to file (name hardcoded at present) + template + void read_shares_from_file(long start_file_pos, int end_file_pos_register, + const vector& data_registers, size_t vector_size, U& Proc); + void write_shares_to_file(long start_pos, const vector& data_registers, + size_t vector_size); }; class ArithmeticProcessor : public ProcessorBase @@ -228,7 +237,6 @@ class Processor : public ArithmeticProcessor TempVars temp; ExternalClients& external_clients; - Binary_File_IO binary_file_io; CommStats client_stats; Timer& client_timer; @@ -289,12 +297,6 @@ class Processor : public ArithmeticProcessor void read_socket_private(int client_id, const vector& registers, int size, bool send_macs); - // Read and write secret numeric data to file (name hardcoded at present) - void read_shares_from_file(long start_file_pos, int end_file_pos_register, - const vector& data_registers, size_t vector_size); - void write_shares_to_file(long start_pos, const vector& data_registers, - size_t vector_size); - cint get_inverse2(unsigned m); void fixinput(const Instruction& instruction); diff --git a/Processor/Processor.hpp b/Processor/Processor.hpp index c6beb9a6..5092eebe 100644 --- a/Processor/Processor.hpp +++ b/Processor/Processor.hpp @@ -88,7 +88,7 @@ Processor::Processor(int thread_num,Player& P, Procb(machine.bit_memories), Proc2(*this,MC2,DataF.DataF2,P),Procp(*this,MCp,DataF.DataFp,P), external_clients(machine.external_clients), - binary_file_io(Binary_File_IO()), client_timer(client_stats.timer) + client_timer(client_stats.timer) { reset(program,0); @@ -388,12 +388,13 @@ void Processor::read_socket_private(int client_id, // Read share data from a file starting at file_pos until registers filled. // file_pos_register is written with new file position (-1 is eof). // Tolerent to no file if no shares yet persisted. -template -void Processor::read_shares_from_file(long start_file_posn, +template +template +void SubProcessor::read_shares_from_file(long start_file_posn, int end_file_pos_register, const vector& data_registers, - size_t vector_size) + size_t vector_size, U& Proc) { - if (not sint::real_shares(P)) + if (not T::real_shares(P)) return; string filename; @@ -401,7 +402,7 @@ void Processor::read_shares_from_file(long start_file_posn, unsigned int size = data_registers.size(); - PointerVector outbuf(size * vector_size); + PointerVector outbuf(size * vector_size); auto end_file_posn = start_file_posn; @@ -411,36 +412,36 @@ void Processor::read_shares_from_file(long start_file_posn, for (unsigned int i = 0; i < size; i++) { for (size_t j = 0; j < vector_size; j++) - get_Sp_ref(data_registers[i] + j) = outbuf.next(); + get_S_ref(data_registers[i] + j) = outbuf.next(); } - write_Ci(end_file_pos_register, (long)end_file_posn); + Proc.write_Ci(end_file_pos_register, (long)end_file_posn); } catch (file_missing& e) { if (OnlineOptions::singleton.has_option("verbose_persistence")) cerr << "Got file missing error, will return -2. " << e.what() << endl; - write_Ci(end_file_pos_register, (long)-2); + Proc.write_Ci(end_file_pos_register, (long)-2); } } // Append share data in data_registers to end of file. Expects Persistence directory to exist. -template -void Processor::write_shares_to_file(long start_pos, +template +void SubProcessor::write_shares_to_file(long start_pos, const vector& data_registers, size_t vector_size) { - if (not sint::real_shares(P)) + if (not T::real_shares(P)) return; string filename = binary_file_io.filename(P.my_num()); unsigned int size = data_registers.size(); - PointerVector inpbuf(size * vector_size); + PointerVector inpbuf(size * vector_size); for (unsigned int i = 0; i < size; i++) { for (size_t j = 0; j < vector_size; j++) - inpbuf.next() = get_Sp_ref(data_registers[i] + j); + inpbuf.next() = get_S_ref(data_registers[i] + j); } binary_file_io.write_to_file(filename, inpbuf, start_pos); diff --git a/Processor/Program.cpp b/Processor/Program.cpp index 57c8f33e..8489226e 100644 --- a/Processor/Program.cpp +++ b/Processor/Program.cpp @@ -23,7 +23,7 @@ void Program::compute_constants() max_mem[reg_type] = max(max_mem[reg_type], p[i].get_mem(RegType(reg_type))); } - writes_persistence |= p[i].opcode == WRITEFILESHARE; + writes_persistence |= (p[i].opcode & 0xFF) == WRITEFILESHARE; } }