File persistance for GF(2^n).

Compiler/instructions.py

@@ -1989,6 +1989,7 @@ class closeclientconnection(base.IOInstruction):
     code = base.opcodes['CLOSECLIENTCONNECTION']
     arg_format = ['ci']
 
+@base.gf2n
 class writesharestofile(base.VectorInstruction, base.IOInstruction):
     """ Write shares to ``Persistence/Transactions-P<playerno>.data``
     (appending at the end).
@@ -2001,12 +2002,13 @@ class writesharestofile(base.VectorInstruction, base.IOInstruction):
     """
     __slots__ = []
     code = base.opcodes['WRITEFILESHARE']
-    arg_format = tools.chain(['ci'], itertools.repeat('s'))
+    arg_format = tools.chain(['ci'], tools.cycle(['s']))
     vector_index = 1
 
     def has_var_args(self):
         return True
 
+@base.gf2n
 class readsharesfromfile(base.VectorInstruction, base.IOInstruction):
     """ Read shares from ``Persistence/Transactions-P<playerno>.data``.
 
@@ -2018,7 +2020,7 @@ class readsharesfromfile(base.VectorInstruction, base.IOInstruction):
     """
     __slots__ = []
     code = base.opcodes['READFILESHARE']
-    arg_format = tools.chain(['ci', 'ciw'], itertools.repeat('sw'))
+    arg_format = tools.chain(['ci', 'ciw'], tools.cycle(['sw']))
     vector_index = 2
 
     def has_var_args(self):

Compiler/instructions_base.py

@@ -369,7 +369,7 @@ def gf2n(instruction):
         if isinstance(arg_format, list):
             __format = []
             for __f in arg_format:
-                if __f in ('int', 'long', 'p', 'ci', 'str'):
+                if __f in ('int', 'long', 'p', 'ci', 'ciw', 'str'):
                     __format.append(__f)
                 else:
                     __format.append(__f[0] + 'g' + __f[1:])

Compiler/types.py

@@ -2430,6 +2430,47 @@ class _secret(_arithmetic_register, _secret_structure):
     def output(self):
         print_reg_plains(self)
 
+    @classmethod
+    @set_instruction_type
+    def read_from_file(cls, start, n_items=1, crash_if_missing=True, size=1):
+        """ Read shares from
+        ``Persistence/Transactions-[gf2n-]P<playerno>.data``. See :ref:`this
+        section <persistence>` for details on the data format.
+
+        :param start: starting position in number of shares from beginning (int/regint/cint)
+        :param n_items: number of items (int)
+        :param crash_if_missing: crash if file not found (default)
+        :param size: vector size (int)
+        :returns: destination for final position, -1 for eof reached, or -2 for file not found (regint)
+        :returns: list of shares
+        """
+        shares = [cls(size=size) for i in range(n_items)]
+        stop = regint()
+        readsharesfromfile(regint.conv(start), stop, *shares)
+        if crash_if_missing:
+            library.runtime_error_if(stop == -2, 'Persistence not found')
+        return stop, shares
+
+    @classmethod
+    @set_instruction_type
+    def write_to_file(cls, shares, position=None):
+        """ Write shares to ``Persistence/Transactions-[gf2n-]P<playerno>.data``
+        (appending at the end). See :ref:`this section <persistence>`
+        for details on the data format.
+
+        :param shares: (list or iterable of shares)
+        :param position: start position (int/regint/cint),
+            defaults to end of file
+        """
+        if isinstance(shares, cls):
+            shares = [shares]
+        for share in shares:
+            assert isinstance(share, cls)
+            assert share.size == shares[0].size
+        if position is None:
+            position = -1
+        writesharestofile(regint.conv(position), *shares)
+
 class sint(_secret, _int):
     """
     Secret integer in the protocol-specific domain. It supports
@@ -2699,45 +2740,6 @@ class sint(_secret, _int):
         """
         writesocketshare(client_id, message_type, values[0].size, *values)
 
-    @classmethod
-    def read_from_file(cls, start, n_items=1, crash_if_missing=True, size=1):
-        """ Read shares from
-        ``Persistence/Transactions-P<playerno>.data``. See :ref:`this
-        section <persistence>` for details on the data format.
-
-        :param start: starting position in number of shares from beginning (int/regint/cint)
-        :param n_items: number of items (int)
-        :param crash_if_missing: crash if file not found (default)
-        :param size: vector size (int)
-        :returns: destination for final position, -1 for eof reached, or -2 for file not found (regint)
-        :returns: list of shares
-        """
-        shares = [cls(size=size) for i in range(n_items)]
-        stop = regint()
-        readsharesfromfile(regint.conv(start), stop, *shares)
-        if crash_if_missing:
-            library.runtime_error_if(stop == -2, 'Persistence not found')
-        return stop, shares
-
-    @staticmethod
-    def write_to_file(shares, position=None):
-        """ Write shares to ``Persistence/Transactions-P<playerno>.data``
-        (appending at the end). See :ref:`this section <persistence>`
-        for details on the data format.
-
-        :param shares: (list or iterable of sint)
-        :param position: start position (int/regint/cint),
-            defaults to end of file
-        """
-        if isinstance(shares, sint):
-            shares = [shares]
-        for share in shares:
-            assert isinstance(share, sint)
-            assert share.size == shares[0].size
-        if position is None:
-            position = -1
-        writesharestofile(regint.conv(position), *shares)
-
     @vectorized_classmethod
     def load_mem(cls, address, mem_type=None):
         """ Load from memory by public address. """

Processor/Binary_File_IO.h

@@ -16,17 +16,18 @@ using namespace std;
  * Intended for MPC application specific file IO.
  */
 
+template<class T>
 class Binary_File_IO
 {
   public:
 
   static string filename(int my_number);
+  static void reset(int my_number);
 
   /*
    * Append the buffer values as binary to the filename.
    * Throws file_error.   
    */
-  template <class T>
   void write_to_file(const string filename, const vector<T>& buffer,
       long start_pos);
 
@@ -36,7 +37,6 @@ class Binary_File_IO
    * Returns the current posn in the file or -1 if at eof.
    * Throws file_error.
    */
-  template <class T>
   void read_from_file(const string filename, vector<T>& buffer,
       const long start_posn, long& end_posn);
 };

Processor/Binary_File_IO.hpp

@@ -6,16 +6,35 @@
  * Intended for application specific file IO.
  */
 
-inline string Binary_File_IO::filename(int my_number)
+template<class T>
+inline string Binary_File_IO<T>::filename(int my_number)
 {
   string dir = "Persistence";
   mkdir_p(dir.c_str());
-  return dir + "/Transactions-P" + to_string(my_number) + ".data";
+  string res = dir + "/Transactions";
+  if (T::clear::characteristic_two)
+    res += "-gf2n";
+  return res + "-P" + to_string(my_number) + ".data";
 }
 
-template<class T> 
+template<class T>
+void Binary_File_IO<T>::reset(int my_number)
+{
+  string filename = Binary_File_IO<T>::filename(my_number);
+  ifstream pers(filename);
+  try
+  {
+      check_file_signature<T>(pers, filename);
+  }
+  catch (signature_mismatch&)
+  {
+      ofstream pers(filename, ios::binary);
+      file_signature<T>().output(pers);
+  }
+}
 
-void Binary_File_IO::write_to_file(const string filename,
+template<class T>
+void Binary_File_IO<T>::write_to_file(const string filename,
     const vector<T>& buffer, long start_pos)
 {
   ofstream outf;
@@ -44,7 +63,7 @@ void Binary_File_IO::write_to_file(const string filename,
 }
 
 template<class T>
-void Binary_File_IO::read_from_file(const string filename, vector<T>& buffer,
+void Binary_File_IO<T>::read_from_file(const string filename, vector<T>& buffer,
     const long start_posn, long& end_posn)
 {
   ifstream inf;

Processor/Instruction.h

@@ -311,6 +311,8 @@ enum
     GRAWOUTPUT = 0x1B7,
     GSTARTPRIVATEOUTPUT = 0x1B8,
     GSTOPPRIVATEOUTPUT = 0x1B9,
+    GWRITEFILESHARE = 0x1BD,
+    GREADFILESHARE = 0x1BE,
     // Commsec ops
     INITSECURESOCKET = 0x1BA,
     RESPSECURESOCKET = 0x1BB

Processor/Instruction.hpp

@@ -328,6 +328,7 @@ void BaseInstruction::parse_operands(istream& s, int pos, int file_pos)
       // read from file, input is opcode num_args, 
       //   start_file_posn (read), end_file_posn(write) var1, var2, ...
       case READFILESHARE:
+      case GREADFILESHARE:
       case CALL_TAPE:
         num_var_args = get_int(s) - 2;
         r[0] = get_int(s);
@@ -397,6 +398,7 @@ void BaseInstruction::parse_operands(istream& s, int pos, int file_pos)
       case EDABIT:
       case SEDABIT:
       case WRITEFILESHARE:
+      case GWRITEFILESHARE:
       case CONCATS:
           num_var_args = get_int(s) - 1;
           r[0] = get_int(s);
@@ -1371,11 +1373,21 @@ inline void Instruction::execute(Processor<sint, sgf2n>& Proc) const
         break;
       case WRITEFILESHARE:
         // Write shares to file system
-        Proc.write_shares_to_file(Proc.read_Ci(r[0]), start, size);
+        Procp.write_shares_to_file(Proc.read_Ci(r[0]), start, size);
         return;
       case READFILESHARE:
         // Read shares from file system
-        Proc.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size);
+        Procp.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size,
+            Proc);
+        return;
+      case GWRITEFILESHARE:
+        // Write shares to file system
+        Proc2.write_shares_to_file(Proc.read_Ci(r[0]), start, size);
+        return;
+      case GREADFILESHARE:
+        // Read shares from file system
+        Proc2.read_shares_from_file(Proc.read_Ci(r[0]), r[1], start, size,
+            Proc);
         return;
       case PUBINPUT:
         Proc.get_Cp_ref(r[0]) = Proc.template

Processor/Machine.hpp

@@ -165,17 +165,8 @@ void Machine<sint, sgf2n>::prepare(const string& progname_str)
     {
       if (prog.writes_persistence)
         {
-          string filename = Binary_File_IO::filename(my_number);
+          Binary_File_IO<sint>::reset(my_number);
-          ifstream pers(filename);
+          Binary_File_IO<sgf2n>::reset(my_number);
-          try
-          {
-              check_file_signature<sint>(pers, filename);
-          }
-          catch (signature_mismatch&)
-          {
-              ofstream pers(filename, ios::binary);
-              file_signature<sint>().output(pers);
-          }
           break;
         }
     }

Processor/Processor.h

@@ -38,6 +38,8 @@ class SubProcessor
   DataPositions bit_usage;
   NamedStats stats;
 
+  Binary_File_IO<T> binary_file_io;
+
   void resize(size_t size)       { C.resize(size); S.resize(size); }
 
   void matmulsm_prep(int ii, int j, const MemoryPart<T>& source,
@@ -128,6 +130,13 @@ public:
   void push_stack();
   void push_args(const vector<int>& args);
   void pop_stack(const vector<int>& results);
+
+  // Read and write secret numeric data to file (name hardcoded at present)
+  template<class U>
+  void read_shares_from_file(long start_file_pos, int end_file_pos_register,
+      const vector<int>& data_registers, size_t vector_size, U& Proc);
+  void write_shares_to_file(long start_pos, const vector<int>& data_registers,
+      size_t vector_size);
 };
 
 class ArithmeticProcessor : public ProcessorBase
@@ -228,7 +237,6 @@ class Processor : public ArithmeticProcessor
   TempVars<sint, sgf2n> temp;
 
   ExternalClients& external_clients;
-  Binary_File_IO binary_file_io;
 
   CommStats client_stats;
   Timer& client_timer;
@@ -289,12 +297,6 @@ class Processor : public ArithmeticProcessor
   void read_socket_private(int client_id, const vector<int>& registers,
       int size, bool send_macs);
 
-  // Read and write secret numeric data to file (name hardcoded at present)
-  void read_shares_from_file(long start_file_pos, int end_file_pos_register,
-      const vector<int>& data_registers, size_t vector_size);
-  void write_shares_to_file(long start_pos, const vector<int>& data_registers,
-      size_t vector_size);
-  
   cint get_inverse2(unsigned m);
 
   void fixinput(const Instruction& instruction);

Processor/Processor.hpp

@@ -88,7 +88,7 @@ Processor<sint, sgf2n>::Processor(int thread_num,Player& P,
   Procb(machine.bit_memories),
   Proc2(*this,MC2,DataF.DataF2,P),Procp(*this,MCp,DataF.DataFp,P),
   external_clients(machine.external_clients),
-  binary_file_io(Binary_File_IO()), client_timer(client_stats.timer)
+  client_timer(client_stats.timer)
 {
   reset(program,0);
 
@@ -388,12 +388,13 @@ void Processor<sint, sgf2n>::read_socket_private(int client_id,
 // Read share data from a file starting at file_pos until registers filled.
 // file_pos_register is written with new file position (-1 is eof).
 // Tolerent to no file if no shares yet persisted.
-template<class sint, class sgf2n>
+template<class T>
-void Processor<sint, sgf2n>::read_shares_from_file(long start_file_posn,
+template<class U>
+void SubProcessor<T>::read_shares_from_file(long start_file_posn,
     int end_file_pos_register, const vector<int>& data_registers,
-    size_t vector_size)
+    size_t vector_size, U& Proc)
 {
-  if (not sint::real_shares(P))
+  if (not T::real_shares(P))
     return;
 
   string filename;
@@ -401,7 +402,7 @@ void Processor<sint, sgf2n>::read_shares_from_file(long start_file_posn,
 
   unsigned int size = data_registers.size();
 
-  PointerVector<sint> outbuf(size * vector_size);
+  PointerVector<T> outbuf(size * vector_size);
 
   auto end_file_posn = start_file_posn;
 
@@ -411,36 +412,36 @@ void Processor<sint, sgf2n>::read_shares_from_file(long start_file_posn,
     for (unsigned int i = 0; i < size; i++)
     {
       for (size_t j = 0; j < vector_size; j++)
-        get_Sp_ref(data_registers[i] + j) = outbuf.next();
+        get_S_ref(data_registers[i] + j) = outbuf.next();
     }
 
-    write_Ci(end_file_pos_register, (long)end_file_posn);    
+    Proc.write_Ci(end_file_pos_register, (long)end_file_posn);
   }
   catch (file_missing& e) {
     if (OnlineOptions::singleton.has_option("verbose_persistence"))
       cerr << "Got file missing error, will return -2. " << e.what() << endl;
-    write_Ci(end_file_pos_register, (long)-2);
+    Proc.write_Ci(end_file_pos_register, (long)-2);
   }
 }
 
 // Append share data in data_registers to end of file. Expects Persistence directory to exist.
-template<class sint, class sgf2n>
+template<class T>
-void Processor<sint, sgf2n>::write_shares_to_file(long start_pos,
+void SubProcessor<T>::write_shares_to_file(long start_pos,
     const vector<int>& data_registers, size_t vector_size)
 {
-  if (not sint::real_shares(P))
+  if (not T::real_shares(P))
     return;
 
   string filename = binary_file_io.filename(P.my_num());
 
   unsigned int size = data_registers.size();
 
-  PointerVector<sint> inpbuf(size * vector_size);
+  PointerVector<T> inpbuf(size * vector_size);
 
   for (unsigned int i = 0; i < size; i++)
   {
     for (size_t j = 0; j < vector_size; j++)
-      inpbuf.next() = get_Sp_ref(data_registers[i] + j);
+      inpbuf.next() = get_S_ref(data_registers[i] + j);
   }
 
   binary_file_io.write_to_file(filename, inpbuf, start_pos);

Processor/Program.cpp

@@ -23,7 +23,7 @@ void Program::compute_constants()
           max_mem[reg_type] = max(max_mem[reg_type],
               p[i].get_mem(RegType(reg_type)));
         }
-      writes_persistence |= p[i].opcode == WRITEFILESHARE;
+      writes_persistence |= (p[i].opcode & 0xFF) == WRITEFILESHARE;
     }
 }