Reorganize build process

Fixes #7

Author: schoppmp

.gitignore

@@ -1,7 +1,11 @@
 bin/
-obj/
 *.pb-c.h
 *.pb-c.c
 __pycache__/
 *.pyc
 .ipynb_checkpoints/
+*.d
+*.od
+*.o
+*.oo
+secure_multiplication.proto

Makefile

@@ -1,79 +1,82 @@
 OBLIVCC=$(OBLIVC_PATH)/bin/oblivcc
 
-binDir=bin
-objDir=obj
-srcDir=src
-libDir=lib
+BINDIR=bin
+SRCDIR=src
+LIBDIR=lib
 
 REMOTE_HOST=localhost
 BIT_WIDTH_32_P1=0
 BIT_WIDTH_32_P2=0
-CFLAGS=-O3 -g -Werror -I $(srcDir) -I $(OBLIVC_PATH)/src/ext/oblivc -std=c11 -D_POSIX_C_SOURCE=201605L -DBIT_WIDTH_32_P1=$(BIT_WIDTH_32_P1) -DBIT_WIDTH_32_P2=$(BIT_WIDTH_32_P2)
-LFLAGS=-L$(HOME)/lib
-OCFLAGS=$(CFLAGS) -DREMOTE_HOST=$(REMOTE_HOST)
-
-ackLibDir=$(libDir)/absentminded-crypto-kit/build/lib
-ackLib=$(ackLibDir)/liback.a
-ackIncDir=$(libDir)/absentminded-crypto-kit/src/
-OLFLAGS += -L$(ackLibDir) -lack -lm
-OCFLAGS += -g -DREMOTE_HOST=$(REMOTE_HOST) -O3 -Werror -I$(ackIncDir)
-
-mkpath=mkdir -p $(@D)
-compile=$(mkpath) && $(CC) $(CFLAGS) -c $< -o $@
-link=$(mkpath) && $(CC) $(LFLAGS) $^ -o $@
-compile_obliv=$(mkpath) && $(OBLIVCC) $(OCFLAGS) -c $< -o $@
-link_obliv=$(mkpath) && $(OBLIVCC) $(OLFLAGS) $^ -o $@
-
-native=$(objDir)/$(1)_c.o
-obliv=$(objDir)/$(1)_o.o
-both=$(call native,$(1)) $(call obliv,$(1))
-
-all: $(binDir)/test_linear_system $(binDir)/test_fixed $(binDir)/secure_multiplication $(binDir)/main
-
-$(binDir)/main: $(objDir)/main.o $(objDir)/secure_multiplication/node.o $(objDir)/secure_multiplication/config.o $(objDir)/secure_multiplication/phase1.o $(objDir)/secure_multiplication/secure_multiplication.pb-c.o $(call both,linear) $(call both,fixed) $(call native,util) $(call obliv,ldlt) $(call obliv,cholesky) $(call obliv,cgd) $(call native,input)
-	$(link_obliv) -lprotobuf-c -lm
 
-$(binDir)/secure_multiplication:$(objDir)/secure_multiplication/secure_multiplication.pb-c.o $(objDir)/secure_multiplication/secure_multiplication.o $(objDir)/secure_multiplication/config.o $(objDir)/secure_multiplication/node.o $(objDir)/linear.o $(objDir)/fixed.o $(objDir)/secure_multiplication/phase1.o $(objDir)/util.o
-	$(link_obliv) -lprotobuf-c -lm
+SOURCES := $(shell find -L $(SRCDIR) -type f -name '*.c' -not -path '*/cmd/*' )
+OBJECTS := $(patsubst %.c, %.o, $(SOURCES))
+SOURCES_OBLIVC := $(shell find -L $(SRCDIR) -type f -name '*.oc')
+OBJECTS_OBLIVC := $(patsubst %.oc, %.oo, $(SOURCES_OBLIVC))
+SOURCES_BIN := $(shell find -L $(SRCDIR)/cmd -type f -name '*.c')
+OBJECTS_BIN := $(patsubst %.c, %.o, $(SOURCES_BIN))
+BINARIES = $(patsubst $(SRCDIR)/cmd/%.c, $(BINDIR)/%, $(SOURCES_BIN))
 
-$(binDir)/test_linear_system: $(ackLib) $(call native,test/test_linear_system) $(call both,linear) $(call both,fixed) $(call native,util) $(call obliv,ldlt) $(call obliv,cholesky) $(call obliv,cgd) $(call native,input)
-	$(link_obliv)
-
-$(binDir)/test_fixed: $(call both,test/test_fixed) $(call both,fixed) $(call native,util)
-	$(link_obliv)
+CFLAGS=-O3 -g -Werror -pthread -I$(SRCDIR) -I$(OBLIVC_PATH)/src/ext/oblivc -std=c11 -D_POSIX_C_SOURCE=201605L -DBIT_WIDTH_32_P1=$(BIT_WIDTH_32_P1) -DBIT_WIDTH_32_P2=$(BIT_WIDTH_32_P2)
+LDFLAGS= -lm -lgcrypt -lprotobuf-c
+OCFLAGS=$(CFLAGS) -DREMOTE_HOST=$(REMOTE_HOST)
 
-$(binDir)/test_input: $(call native,input) $(call obliv,test/test_input) $(call native,util)
-	$(link_obliv)
+# Absentminded Crypto Kit
+ACKLIBDIR=$(LIBDIR)/absentminded-crypto-kit/build/lib
+ACKLIB=$(ACKLIBDIR)/liback.a
+LDFLAGS += -L$(ACKLIBDIR) -L$(OBLIVC_PATH)/_build -lack -lobliv
+OCFLAGS += -I$(LIBDIR)/absentminded-crypto-kit/src/ -D_Float128=double
 
-$(ackLib): $(libDir)/absentminded-crypto-kit/Makefile
-	cd $(libDir)/absentminded-crypto-kit && make
+all: $(BINARIES)
 
-$(libDir)/absentminded-crypto-kit/Makefile:
+# Libraries
+$(ACKLIB): $(LIBDIR)/absentminded-crypto-kit/Makefile
+	CFLAGS=-D_Float128=double make -C $(LIBDIR)/absentminded-crypto-kit
+$(LIBDIR)/absentminded-crypto-kit/Makefile:
 	git submodule update --init
 
-$(objDir)/%_c.o: $(srcDir)/%.c
-	$(compile_obliv)
-
-$(objDir)/%_o.o: $(srcDir)/%.oc
-	$(compile_obliv)
-
-$(objDir)/%.o: $(srcDir)/%.c
-	$(compile)
-
+# Protocol Buffers
+PROTOBUF_PROTOS = $(SRCDIR)/protobuf/secure_multiplication.proto
+PROTOBUF_HEADERS = $(PROTOBUF_PROTOS:.proto=.pb-c.h)
+PROTOBUF_OBJECTS = $(PROTOBUF_HEADERS:.h=.o)
+PROTOBUF_ALL = $(PROTOBUF_PROTOS) $(PROTOBUF_HEADERS) $(PROTOBUF_OBJECTS) $(PROTOBUF_HEADERS:.h=.{c,d})
 ifeq ($(BIT_WIDTH_32_P1), 1)
-$(srcDir)/%.pb-c.c: $(srcDir)/%.proto
-	cd $(<D) && protoc-c secure_multiplication_bitwidth_32.proto --c_out=. &&\
-	mv secure_multiplication_bitwidth_32.pb-c.c secure_multiplication.pb-c.c &&\
-	cp secure_multiplication_bitwidth_32.pb-c.h secure_multiplication.pb-c.h
+BIT_WIDTH_P1 = 32
 else
-$(srcDir)/%.pb-c.c: $(srcDir)/%.proto
-	cd $(<D) && protoc-c secure_multiplication_bitwidth_64.proto --c_out=. &&\
-	mv secure_multiplication_bitwidth_64.pb-c.c secure_multiplication.pb-c.c &&\
-	cp secure_multiplication_bitwidth_64.pb-c.h secure_multiplication.pb-c.h
+BIT_WIDTH_P1 = 64
 endif
-
+%.proto: %.bitwidth_$(BIT_WIDTH_P1).proto
+	cp $< $@
+%.pb-c.c %.pb-c.h: %.proto
+	cd $(@D) && protoc-c $(<F) --c_out=.
+
+# Dependencies
+-include $(SOURCES:.c=.d)
+-include $(SOURCES_BIN:.c=.d)
+-include $(SOURCES_OBLIVC:.oc=.od)
+
+# do not delete intermediate objects
+.SECONDARY: $(OBJECTS) $(OBJECTS_BIN) $(OBJECTS_OBLIVC) $(PROTOBUF_ALL)
+
+# Binaries
+$(BINDIR)/%: $(OBJECTS) $(OBJECTS_OBLIVC) $(SRCDIR)/cmd/%.o  $(PROTOBUF_OBJECTS) | $(ACKLIB)
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+# C sources
+%.o: %.c $(PROTOBUF_HEADERS)
+	$(CC) $(CFLAGS) -o $@ -c $<
+	$(CC) -MM $(CFLAGS) -MT "$*.o $@" $< > $*.d;
+# Obliv-C Sources
+%.oo : %.oc
+	$(OBLIVCC) $(OCFLAGS) -o $@ -c $<
+	$(OBLIVCC) -MM $(OCFLAGS) -MT "$*.oo $@" $< > $*.od;
+
+.PHONY: clean cleanall
 clean:
-	rm -rf $(binDir) $(objDir)
+	$(RM) -r $(BINDIR)
+	$(RM) $(OBJECTS) $(OBJECTS_BIN) $(OBJECTS_OBLIVC) $(OBJECTS_PROTOBUF)
+	$(RM) $(OBJECTS:.o=.d) $(OBJECTS_BIN:.o=.d) $(OBJECTS_OBLIVC:.oo=.od)
+	$(RM) $(PROTOBUF_ALL)
 
 cleanall: clean
-	cd $(libDir)/absentminded-crypto-kit && make clean
+	cd $(LIBDIR)/absentminded-crypto-kit && make clean

README.md

@@ -28,18 +28,18 @@ Currently only the following combinations of bitwidths for both phases are suppo
 | 64                | 64                |
 | 64                | 32                |
 
-Using smaller bit widths will increase computation speed at the cost of accuracy. 
+Using smaller bit widths will increase computation speed at the cost of accuracy.
 
 ## Running experiments
-The protocol consists of two phases. `bin/main` is used to run both phases at once.
+The protocol consists of two phases. `bin/linreg` is used to run both phases at once.
 ```
-Usage: bin/main [Input_file] [Precision] [Party] [Algorithm] [Num. iterations CGD] [Lambda] [Options]
+Usage: bin/linreg [Input_file] [Precision] [Party] [Algorithm] [Num. iterations CGD] [Lambda] [Options]
 Options: --use_ot: Enables the OT-based phase 1 protocol
          --prec_phase2=<Precision phase 2>: Use different precision for phase 2 of the protocol
 ```
 `[Precision]` specifies the number of bits used for the fractional part of fixed-point encoded numbers.
-The role of the process is given by `[Party]`. 
-Values of 1 and 2 denote the CSP and Evaluator, respectively. 
+The role of the process is given by `[Party]`.
+Values of 1 and 2 denote the CSP and Evaluator, respectively.
 Higher values denote data providers.
 `[Algorithm]` is the algorithm used for phase 2 of the protocol and can be either `cholesky`, `ldlt`, or `cgd`.
 In the case of CGD, `[Num. iterations CGD]` gives the number of iterations used before terminating.
@@ -56,32 +56,32 @@ localhost:1236 0
 localhost:1237 1
 localhost:1238 2
 10 5
--0.49490189906 0.204027068031 1.0 -0.0048744428595 0.101856000869 
-0.49217428514 0.233090721372 -0.0370213771185 0.657720359628 0.525310770733 
-0.39721227919 -0.470002959473 0.277398798616 0.726013362994 0.498046786861 
--0.595183792269 -0.0827269428691 -0.227611333929 0.80744916955 0.543968791627 
--0.419150270985 0.292994257369 -0.0232190555386 -0.108185527543 0.238913650953 
--0.247990663017 -1.0 -0.0241769053355 -0.842345696475 -1.0 
--1.0 0.437037219652 -0.140750839011 1.0 -0.0679355153937 
-0.777015345458 0.981488358907 0.2870338525 0.579832272908 -0.262265338898 
--0.0431656936622 0.0910166551363 -0.0724090181375 -0.144008713528 0.202706144381 
--0.470188350902 -0.261201944989 -0.0468243599479 0.944605903172 0.455480894023 
+-0.49490189906 0.204027068031 1.0 -0.0048744428595 0.101856000869
+0.49217428514 0.233090721372 -0.0370213771185 0.657720359628 0.525310770733
+0.39721227919 -0.470002959473 0.277398798616 0.726013362994 0.498046786861
+-0.595183792269 -0.0827269428691 -0.227611333929 0.80744916955 0.543968791627
+-0.419150270985 0.292994257369 -0.0232190555386 -0.108185527543 0.238913650953
+-0.247990663017 -1.0 -0.0241769053355 -0.842345696475 -1.0
+-1.0 0.437037219652 -0.140750839011 1.0 -0.0679355153937
+0.777015345458 0.981488358907 0.2870338525 0.579832272908 -0.262265338898
+-0.0431656936622 0.0910166551363 -0.0724090181375 -0.144008713528 0.202706144381
+-0.470188350902 -0.261201944989 -0.0468243599479 0.944605903172 0.455480894023
 10
-0.463624257142 1.03914869701 0.641866385451 -0.289962112725 -0.328203220996 -1.64668349374 -0.205128259075 2.13151959591 -0.129983842519 -0.0766970254399 
+0.463624257142 1.03914869701 0.641866385451 -0.289962112725 -0.328203220996 -1.64668349374 -0.205128259075 2.13151959591 -0.129983842519 -0.0766970254399
 ```
-It contains the number of samples, features and parties, followed by a network endpoint for each party. 
-The special roles CSP and Evaluator are defined by the first two lines. 
+It contains the number of samples, features and parties, followed by a network endpoint for each party.
+The special roles CSP and Evaluator are defined by the first two lines.
 Then, the data providers follow (3 in this example), each with a network endpoint and the starting index of its partition.
-Afterwards, the dimensions of `X` are specified, followed by `X` itself. 
+Afterwards, the dimensions of `X` are specified, followed by `X` itself.
 Finally, the length and values of `y` are given.
 
 Running this example locally with
 ```
-for party in {1..5}; do bin/main examples/readme_example.in 56 $party cgd 10 0.001 & done
+for party in {1..5}; do bin/linreg examples/readme_example.in 56 $party cgd 10 0.001 & done
 ```
 yields the following result, in addition to some debug outputs:
 ```
 Time elapsed: 5.673679
 Number of gates: 18569664
-Result:    0.984331027786964    0.792399824970372    0.754117840176144    0.592849130685193    0.057351715952213 
+Result:    0.984331027786964    0.792399824970372    0.754117840176144    0.592849130685193    0.057351715952213
 ```

src/main.c src/cmd/linreg.c

@@ -6,15 +6,14 @@
 #include <obliv.h>
 #include <obliv_common.h>
 
-#include "secure_multiplication/secure_multiplication.pb-c.h"
-#include "secure_multiplication/node.h"
-#include "secure_multiplication/config.h"
+#include "protobuf/secure_multiplication.pb-c.h"
+#include "node.h"
+#include "config.h"
 #include "check_error.h"
 #include "linear.h"
-#include "secure_multiplication/phase1.h"
+#include "phase1.h"
 #include "input.h"
 #include "util.h"
-#include "secure_multiplication/node.h"
 
 
 static int barrier(node *self) {
@@ -87,7 +86,7 @@ int main(int argc, char **argv) {
 	check(precision_phase2 >= -1, "Precision of phase 2 must be nonnegative");
 	check(precision < FIXED_BIT_SIZE, "Precision of phase 1 must be smaller than bit size of phase 1");
 	check(precision_phase2 < FIXED_BIT_SIZE_P2, "Precision of phase 2 must be smaller than bit size of phase 2");
-	
+
 	// read ls, we only need number of iterations
 	linear_system_t ls;
 	if(!strcmp(algorithm, "cgd")){
@@ -96,7 +95,7 @@ int main(int argc, char **argv) {
 	       ls.num_iterations = 0;
 	}
 
-	
+
 	// read config
 	status = config_new(&c, argv[1]);
 	check(!status, "Could not read config");
@@ -137,12 +136,12 @@ int main(int argc, char **argv) {
 	// if party > 2 then
 	//   - I am a data provider
   	//   - share_A and share_b are my shares of the equation
-	
+
 	// phase 2 starts here
 	if (precision_phase2 != -1) {
 	  precision = precision_phase2;
 	}
-	
+
 	if(party < 3){  // CSP and Evaluator
 		ProtocolDesc *pd;
 		if(party == 1) {

src/secure_multiplication/secure_multiplication.c src/cmd/secure_multiplication.c

@@ -4,7 +4,7 @@
 #include <errno.h>
 #include <time.h>
 
-#include "secure_multiplication.pb-c.h"
+#include "protobuf/secure_multiplication.pb-c.h"
 #include "node.h"
 #include "config.h"
 #include "check_error.h"

src/test/test_fixed.c src/cmd/test/test_fixed.c

@@ -1,6 +1,6 @@
 #include <obliv.h>
 #include <stdio.h>
-#include "test/test_fixed.h"
+#include "test_fixed.h"
 #include "util.h"
 #include "check_error.h"
 
@@ -36,22 +36,22 @@ int main(int argc, char **argv) {
 		// check(sscanf(argv[i+3], "%lf", &d) == 1, "Error scanning argument number %d.", i+3);
 		io.inputs[i] = double_to_fixed(d, precision);
 	}
-	
+
 	double time = wallClock();
 	ocTestUtilTcpOrDie(&pd, party==1, argv[1]);
 	setCurrentParty(&pd, party);
 	execYaoProtocol(&pd, test_fixed, &io);
 	cleanupProtocol(&pd);
 
-	if(party == 1) { 
+	if(party == 1) {
 		check(io.len != -1, "Input sizes do not match.");
 		printf("Time elapsed: %f\n", wallClock() - time);
 		printf("Number of gates: %d\n", io.gates);
 		printf("Result: %f\n", fixed_to_double(io.result, precision));
 	}
 	free(io.inputs);
 	ret = 0;
-	
+
 error:
 	return ret;
 }

src/test/test_fixed.h src/cmd/test/test_fixed.h

@@ -1,25 +1,25 @@
 #include <obliv.oh>
 #include "fixed.h"
 #include "fixed.oh"
-#include "test/test_fixed.h"
+#include "test_fixed.h"
 #include "util.h"
 #include <stdio.h>
 
 
 void test_fixed(void *v) {
 	protocolIO *args = v;
-	
+
 	// check for equal length inputs
 	bool equal;
-	revealOblivBool(&equal, 
-		feedOblivInt(args->len, 1) == 
+	revealOblivBool(&equal,
+		feedOblivInt(args->len, 1) ==
 		feedOblivInt(args->len, 2), 0);
 
 	if(!equal) {
 		args->len = -1;
 		return;
-	} 
-	
+	}
+
 	// compute the dot product of a and b
 	ofixed_t c, x, y, t;
 	ofixed_init(&c);
@@ -37,7 +37,7 @@ void test_fixed(void *v) {
 		ofixed_mul(&t, x, y, args->p);
 		ofixed_add(&c, c, t);
 	}
-	
+
 	obliv fixed_t out = ofixed_export(c);
 	revealOblivLLong(&(args->result), out, 0);
 	args->gates = yaoGateCount();

src/test/test_fixed.oc src/cmd/test/test_fixed.oc

@@ -1,15 +1,16 @@
-#include"input.h"
-#include "util.h"
-#include<obliv_types_internal.h>
-#include<obliv_common.h>
-#include<obliv_yao.h>
-#include<obliv.h>
-#include<assert.h>
-#include<error.h>
-#include<errno.h>
+#include <obliv_types_internal.h>
+#include <obliv_common.h>
+#include <obliv_yao.h>
+#include <obliv.h>
+#include <assert.h>
+#include <error.h>
+#include <errno.h>
 #include <unistd.h>
-#include "secure_multiplication/node.h"
+
+#include "node.h"
 #include "fixed.h"
+#include "input.h"
+#include "util.h"
 
 struct DualconS
 { node *self;