ORAM in SPDZ-BMR.

This commit is contained in:
Marcel Keller
2018-03-07 12:25:45 +00:00
parent e43be00836
commit 2f50444b93
340 changed files with 8450 additions and 1839 deletions

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Commit.h"

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef _Commit
#define _Commit

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
// Client key file format:
// X25519 Public Key

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Tools/octetStream.h"
#include "Networking/Player.h"

109
Tools/FlexBuffer.cpp Normal file
View File

@@ -0,0 +1,109 @@
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* FlexBuffer.cpp
*
*/
#include "FlexBuffer.h"
#include <iostream>
#include <unistd.h>
#include "BMR/network/utils.h"
using namespace std;
#ifndef BUFFER_DIR
#define BUFFER_DIR "/tmp"
#endif
ReceivedMsgStore::~ReceivedMsgStore()
{
cout << "Stored " << (double)total_size / 1e9 << " GB in "
<< push_timer.elapsed() << " seconds and retrieved them in "
<< pop_timer.elapsed() << " seconds " << endl;
}
void ReceivedMsgStore::push(ReceivedMsg& msg)
{
#ifdef DEBUG_STORE
cout << "pushing msg of length " << msg.size() << endl;
//phex(msg.data(), min(100UL, msg.size()));
#endif
TimeScope ts(push_timer);
total_size += msg.size();
if (mem_size != N and files.empty())
{
mem[(start + mem_size) % N] = msg;
mem_size++;
}
else
{
char filename[1000];
sprintf(filename, "%s/%d.XXXXXX", BUFFER_DIR, getpid());
FILE* file = fdopen(mkstemp(filename), "w");
if (!file)
throw runtime_error("can't open file");
size_t len = msg.size();
size_t ptr = msg.ptr - msg.buf;
if (fwrite(&len, sizeof(len), 1, file) != 1)
throw runtime_error("can't write");
if (fwrite(&ptr, sizeof(len), 1, file) != 1)
throw runtime_error("can't write");
if (len != 0)
if (fwrite(msg.data(), msg.size(), 1, file) != 1)
throw runtime_error("can't write");
if (fclose(file) != 0)
throw runtime_error("can't close");
files.push_back(filename);
}
}
bool ReceivedMsgStore::pop(ReceivedMsg& msg)
{
TimeScope ts(pop_timer);
if (mem_size != 0)
{
msg = mem[start];
start = (start + 1) % N;
mem_size--;
#ifdef DEBUG_STORE
cout << "popping from memory msg of length " << msg.size() << endl;
//phex(msg.data(), min(100UL, msg.size()));
#endif
return true;
}
else if (!files.empty())
{
string filename = files.front();
FILE* file = fopen(filename.c_str(), "r");
files.pop_front();
fseek(file, 0, SEEK_SET);
size_t len;
if (fread(&len, sizeof(len), 1, file) != 1)
{
perror("can't read length");
throw runtime_error("can't read length");
}
msg.resize(len);
size_t ptr;
if (fread(&ptr, sizeof(len), 1, file) != 1)
{
perror("can't read length");
throw runtime_error("can't read length");
}
msg.ptr = msg.buf + ptr;
if (len != 0)
if (fread(msg.data(), len, 1, file) != 1)
{
perror("can't read data");
throw runtime_error("can't read data");
}
fclose(file);
remove(filename.c_str());
#ifdef DEBUG_FLEXBUF
cout << "popping from disk msg of length " << msg.size() << endl;
phex(msg.data(), min(100UL, msg.size()));
#endif
return true;
}
return false;
}

214
Tools/FlexBuffer.h Normal file
View File

@@ -0,0 +1,214 @@
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* FlexBuffer.h
*
*/
#ifndef TOOLS_FLEXBUFFER_H_
#define TOOLS_FLEXBUFFER_H_
#include "Tools/avx_memcpy.h"
#include "Tools/time-func.h"
#include <stdio.h>
#include <stdexcept>
#include <deque>
#include <iostream>
using namespace std;
class FlexBuffer
{
protected:
char* buf, *ptr;
size_t len, max_len;
void del();
void reset() { buf = ptr = 0; len = max_len = 0; }
public:
FlexBuffer() : buf(0), ptr(0), len(0), max_len(0) {}
FlexBuffer(const FlexBuffer&);
~FlexBuffer() { del(); }
void operator=(FlexBuffer& msg);
char* data() { return buf; }
const char* data() const { return buf; }
size_t size() const { return len; }
size_t capacity() const { return max_len; }
};
class ReceivedMsg : public virtual FlexBuffer
{
friend class ReceivedMsgStore;
public:
void reset_head() { ptr = buf; }
void resize(size_t new_len);
void unserialize(void* output, size_t size);
template <class T>
void unserialize(T& output);
template <class T>
void nonavx_unserialize(T& output);
char* consume(size_t l) { check_buffer(l); char* res = ptr; ptr += l; return res; }
char pop_front() { check_buffer(1); return *ptr++; }
size_t left() { return len - (ptr - buf); }
void check_buffer(size_t size);
};
class SendBuffer : public virtual FlexBuffer
{
public:
void resize(size_t new_len);
void resize_copy(size_t new_max_len);
void clear() { len = 0; }
const char& operator[](size_t i) { return data()[i]; }
void push_back(char c) { serialize(c); }
template <class T>
void serialize(const T& source) { serialize(&source, sizeof(T)); }
void serialize(const void* source, size_t size);
void allocate(size_t size);
template <class T>
void serialize_no_allocate(const T& source);
void serialize_no_allocate(const void* source, size_t size);
};
class LocalBuffer : public ReceivedMsg, public SendBuffer
{
};
class ReceivedMsgStore
{
static const int N = 1;
ReceivedMsg mem[N];
int start, mem_size;
deque<string> files;
size_t total_size;
Timer push_timer, pop_timer;
public:
ReceivedMsgStore() : start(0), mem_size(0), total_size(0) {}
~ReceivedMsgStore();
void push(ReceivedMsg& msg);
bool pop(ReceivedMsg& msg);
bool empty() { return mem_size == 0 and files.empty(); }
};
inline FlexBuffer::FlexBuffer(const FlexBuffer& msg)
{
if (msg.buf)
throw runtime_error("can only copy empty buffers");
reset();
}
inline void FlexBuffer::operator=(FlexBuffer& msg)
{
if (this != &msg)
{
del();
buf = msg.buf;
ptr = msg.ptr;
len = msg.len;
max_len = msg.max_len;
#ifdef DEBUG_FLEXBUF
cout << "moved " << (void*)buf << " " << (void*)msg.buf << " from " << &msg << " to " << this << endl;
#endif
msg.reset();
}
}
inline void ReceivedMsg::resize(size_t new_len)
{
if (new_len > max_len)
{
del();
max_len = new_len;
buf = new char[max_len];
#ifdef DEBUG_FLEXBUF
cout << "allocated " << (void*)buf << " for " << this << endl;
#endif
}
len = new_len;
ptr = buf;
}
inline void FlexBuffer::del()
{
#ifdef DEBUG_FLEXBUF
printf("delete 0x%x for 0x%x\n", buf, this);
#endif
if (buf)
delete[] buf;
reset();
}
inline void ReceivedMsg::unserialize(void* output, size_t size)
{
check_buffer(size);
avx_memcpy(output, ptr, size);
ptr += size;
}
template<class T>
inline void ReceivedMsg::unserialize(T& output)
{
unserialize(&output, sizeof(T));
}
template<class T>
inline void ReceivedMsg::nonavx_unserialize(T& output)
{
check_buffer(sizeof(T));
memcpy(&output, ptr, sizeof(T));
ptr += sizeof(T);
}
inline void SendBuffer::resize(size_t new_size)
{
while (size() < new_size)
push_back(0);
}
inline void SendBuffer::resize_copy(size_t new_max_len)
{
char* old = buf;
max_len = new_max_len;
buf = new char[max_len];
if (old)
{
avx_memcpy(buf, old, len);
delete[] old;
}
ptr = buf + (ptr - old);
}
inline void SendBuffer::serialize(const void* source, size_t size)
{
allocate(size);
serialize_no_allocate(source, size);
}
inline void SendBuffer::allocate(size_t size)
{
if (len + size > max_len)
resize_copy(2 * (len + size));
}
template<class T>
inline void SendBuffer::serialize_no_allocate(const T& source)
{
serialize_no_allocate(&source, sizeof(T));
}
inline void SendBuffer::serialize_no_allocate(const void* source, size_t size)
{
avx_memcpy(buf + len, source, size);
len += size;
}
inline void ReceivedMsg::check_buffer(size_t size)
{
(void)size;
#ifdef CHECK_BUFFER
if (ptr + size > buf + len)
throw overflow_error("not enough data in buffer");
#endif
}
#endif /* TOOLS_FLEXBUFFER_H_ */

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* Lock.cpp

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* Lock.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* MMO.cpp

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* MMO.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* MemoryUsage.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* OfflineMachineBase.cpp

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* OfflineMachineBase.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* Signal.cpp

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* Signal.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* WaitQueue.h
@@ -79,6 +79,21 @@ public:
return running;
}
bool pop_dont_stop(T& value)
{
lock();
if (running and queue.size() == 0)
wait();
bool something_for_you = queue.size() > 0;
if (something_for_you)
{
value = queue.front();
queue.pop_front();
}
unlock();
return something_for_you;
}
void stop()
{
lock();

63
Tools/Worker.h Normal file
View File

@@ -0,0 +1,63 @@
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* Worker.h
*
*/
#ifndef TOOLS_WORKER_H_
#define TOOLS_WORKER_H_
#include "WaitQueue.h"
template <class T>
class Worker
{
pthread_t thread;
WaitQueue<T*> input;
WaitQueue<int> output;
Timer timer;
static void* run_thread(void* worker)
{
((Worker*)worker)->run();
return 0;
}
void run()
{
T* job = 0;
while (input.pop(job))
{
TimeScope ts(timer);
output.push(job->run());
}
}
public:
Worker() : timer(CLOCK_THREAD_CPUTIME_ID)
{
pthread_create(&thread, 0, Worker::run_thread, this);
}
~Worker()
{
input.stop();
pthread_join(thread, 0);
cout << "Worker time: " << timer.elapsed() << endl;
}
void request(T& job)
{
input.push(&job);
}
int done()
{
int res = 0;
output.pop(res);
return res;
}
};
#endif /* TOOLS_WORKER_H_ */

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "aes.h"

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "aes.h"

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef __AES_H
#define __AES_H

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* memcpy.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* benchmarking.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
This file is part of ezOptionParser. See MIT-LICENSE.

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* int.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Tools/mkpath.h"
#include <string.h>

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef TOOLS_MKPATH_H_
#define TOOLS_MKPATH_H_

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Processor/Data_Files.h"

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include <fcntl.h>
@@ -11,6 +11,7 @@
#include "Exceptions/Exceptions.h"
#include "Networking/data.h"
#include "Math/bigint.h"
#include "Tools/time-func.h"
void octetStream::clear()
@@ -140,7 +141,7 @@ void octetStream::store_int(size_t l, int n_bytes)
void octetStream::store(int l)
{
resize(len+4);
INT_TO_BYTES(data+len,l);
encode_length(data+len,l,4);
len+=4;
}
@@ -154,7 +155,7 @@ size_t octetStream::get_int(int n_bytes)
void octetStream::get(int& l)
{
l=BYTES_TO_INT(data+ptr);
l=decode_length(data+ptr,4);
ptr+=4;
}
@@ -193,6 +194,69 @@ void octetStream::get(bigint& ans)
}
void octetStream::exchange(int send_socket, int receive_socket, octetStream& receive_stream)
{
send(send_socket, len, LENGTH_SIZE);
const size_t buffer_size = 100000;
size_t sent = 0, received = 0;
bool length_received = false;
size_t new_len = 0;
#ifdef TIME_ROUNDS
Timer recv_timer;
#endif
while (received < new_len or sent < len or not length_received)
{
if (sent < len)
{
size_t to_send = min(buffer_size, len - sent);
send(send_socket, data + sent, to_send);
sent += to_send;
}
// avoid extra branching, false before length received
if (received < new_len)
{
// same buffer for sending and receiving
// only receive up to already sent data
// or when all is sent
size_t to_receive = 0;
if (sent == len)
to_receive = new_len - received;
else if (sent > received)
to_receive = sent - received;
if (to_receive > 0)
{
#ifdef TIME_ROUNDS
TimeScope ts(recv_timer);
#endif
received += receive_non_blocking(receive_socket,
receive_stream.data + received, to_receive);
}
}
else if (not length_received)
{
#ifdef TIME_ROUNDS
TimeScope ts(recv_timer);
#endif
octet blen[LENGTH_SIZE];
if (receive_all_or_nothing(receive_socket,blen,LENGTH_SIZE) == LENGTH_SIZE)
{
new_len=decode_length(blen,sizeof(blen));
receive_stream.resize(max(new_len, len));
length_received = true;
}
}
}
#ifdef TIME_ROUNDS
cout << "Exchange time: " << recv_timer.elapsed() << " seconds to receive "
<< 1e-3 * new_len << " KB" << endl;
#endif
receive_stream.len = new_len;
receive_stream.reset_read_head();
}
void octetStream::store(const vector<int>& v)
{
store(v.size());
@@ -211,34 +275,6 @@ void octetStream::get(vector<int>& v)
}
void octetStream::exchange(int send_socket, int receive_socket)
{
send(send_socket, len, LENGTH_SIZE);
size_t new_len;
receive(receive_socket, new_len, LENGTH_SIZE);
if (new_len > mxlen)
resize_precise(max(new_len, len));
const size_t buffer_size = 100000;
size_t sent = 0, received = 0;
while (sent < len or received < new_len)
{
if (sent < len)
{
size_t to_send = min(buffer_size, len - sent);
send(send_socket, data + sent, to_send);
sent += to_send;
}
if (received < new_len)
{
size_t to_receive = min(buffer_size, new_len - received);
receive(receive_socket, data + received, to_receive);
received += to_receive;
}
}
len = new_len;
reset_read_head();
}
// Construct the ciphertext as `crypto_secretbox(pt, counter||random)`
void octetStream::encrypt_sequence(const octet* key, uint64_t counter)
{

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef _octetStream
#define _octetStream
@@ -21,6 +21,7 @@
#include "Networking/data.h"
#include "Networking/sockets.h"
#include "Tools/avx_memcpy.h"
#include <string.h>
#include <vector>
@@ -63,7 +64,7 @@ class octetStream
bool done() const { return ptr == len; }
bool empty() const { return len == 0; }
int left() const { return len - ptr; }
size_t left() const { return len - ptr; }
octetStream hash() const;
// output must have length at least HASH_SIZE
@@ -88,6 +89,7 @@ class octetStream
// Append with no padding for decoding
void append(const octet* x,const size_t l);
void append_no_resize(const octet* x,const size_t l);
// Read l octets, with no padding for decoding
void consume(octet* x,const size_t l);
// Return pointer to next l octets and advance pointer
@@ -144,11 +146,12 @@ class octetStream
void encrypt(const octet* key);
void decrypt(const octet* key);
void exchange(int send_socket, int receive_socket);
void input(istream& s);
void output(ostream& s);
void exchange(int send_socket, int receive_socket) { exchange(send_socket, receive_socket, *this); }
void exchange(int send_socket, int receive_socket, octetStream& receive_stream);
friend ostream& operator<<(ostream& s,const octetStream& o);
friend class PRNG;
};
@@ -182,14 +185,19 @@ inline void octetStream::append(const octet* x, const size_t l)
{
if (len+l>mxlen)
resize(len+l);
memcpy(data+len,x,l*sizeof(octet));
avx_memcpy(data+len,x,l*sizeof(octet));
len+=l;
}
inline void octetStream::append_no_resize(const octet* x, const size_t l)
{
avx_memcpy(data+len,x,l*sizeof(octet));
len+=l;
}
inline void octetStream::consume(octet* x,const size_t l)
{
memcpy(x,data+ptr,l*sizeof(octet));
avx_memcpy(x,data+ptr,l*sizeof(octet));
ptr+=l;
}

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* parse.h

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include <iostream>

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Tools/random.h"
@@ -138,26 +138,6 @@ unsigned int PRNG::get_uint()
unsigned char PRNG::get_uchar()
{
if (cnt>=RAND_SIZE) { next(); }
unsigned char ans=random[cnt];
cnt++;
// print_state(); cout << " UCHA " << (int) ans << endl;
return ans;
}
__m128i PRNG::get_doubleword()
{
if (cnt > RAND_SIZE - 16)
next();
__m128i ans = _mm_loadu_si128((__m128i*)&random[cnt]);
cnt += 16;
return ans;
}
void PRNG::get_octetStream(octetStream& ans,int len)
{
ans.resize(len);
@@ -168,22 +148,6 @@ void PRNG::get_octetStream(octetStream& ans,int len)
}
void PRNG::get_octets(octet* ans,int len)
{
int pos=0;
while (len)
{
int step=min(len,RAND_SIZE-cnt);
memcpy(ans+pos,random+cnt,step);
pos+=step;
len-=step;
cnt+=step;
if (cnt==RAND_SIZE)
next();
}
}
bigint PRNG::randomBnd(const bigint& B, bool positive)
{
bigint x;

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef _random
#define _random
@@ -87,4 +87,41 @@ class PRNG
{ return seed; }
};
inline unsigned char PRNG::get_uchar()
{
if (cnt>=RAND_SIZE) { next(); }
unsigned char ans=random[cnt];
cnt++;
// print_state(); cout << " UCHA " << (int) ans << endl;
return ans;
}
inline __m128i PRNG::get_doubleword()
{
if (cnt > RAND_SIZE - 16)
next();
__m128i ans = _mm_loadu_si128((__m128i*)&random[cnt]);
cnt += 16;
return ans;
}
inline void PRNG::get_octets(octet* ans,int len)
{
int pos=0;
while (len)
{
int step=min(len,RAND_SIZE-cnt);
avx_memcpy(ans+pos,random+cnt,step);
pos+=step;
len-=step;
cnt+=step;
if (cnt==RAND_SIZE)
next();
}
}
#endif

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
/*
* SHA1 routine optimized to do word accesses rather than byte accesses,

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef _SHA1
#define _SHA1

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#include "Tools/time-func.h"

View File

@@ -1,4 +1,4 @@
// (C) 2018 University of Bristol. See License.txt
// (C) 2018 University of Bristol, Bar-Ilan University. See License.txt
#ifndef _timer
#define _timer