From e05353523c4840e4a5f52df7e9844eba2eb00fd7 Mon Sep 17 00:00:00 2001 From: Jordi Arranz Date: Fri, 13 Jan 2023 11:25:26 +0000 Subject: [PATCH] Gennet accepts single command line parameter and reads config from global json file --- config/config.json | 2 +- gennet-module/Readme.md | 12 ++ gennet-module/batch_gen.sh | 50 ++++++ gennet-module/build.sh | 3 + gennet-module/generate_network.py | 243 ++++++++++++++++++++++++++++++ gennet-module/requirements.txt | 24 +++ 6 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 gennet-module/Readme.md create mode 100755 gennet-module/batch_gen.sh create mode 100644 gennet-module/build.sh create mode 100755 gennet-module/generate_network.py create mode 100644 gennet-module/requirements.txt diff --git a/config/config.json b/config/config.json index 905cf55..512d23e 100644 --- a/config/config.json +++ b/config/config.json @@ -1,6 +1,6 @@ { "enclave_name" : "wakurtosis", - "topology_path" : "./config/network_topology_auto/", + "topology_path" : "../config/network_topology_auto/", "same_toml_configuration": true, "simulation_time": 60, "message_rate": 10, diff --git a/gennet-module/Readme.md b/gennet-module/Readme.md new file mode 100644 index 0000000..2a91c15 --- /dev/null +++ b/gennet-module/Readme.md @@ -0,0 +1,12 @@ +This repo contains scripts to generate network models (in JSON) and waku configuration files (in TOMLs) for wakukurtosis runs. + + +## generate_network.py +generate_network.py generates one network and per-node configuration files. The tool is configurable with specified number of nodes, topics, network types, node types and number of subnets. Use with Python3. Comment out the `#draw(fname, H)` line to visualise the generated graph. + +> usage: $./generate_network --help + +## batch_gen.sh +batch_gen.sh can generate given number of Waku networks and outputs them to a directory. Please make sure that the output directory does NOT exist; both relative and absolute paths work. The Wakunode parameters are generated at random; edit the MIN and MAX for finer control. The script requires bc & /dev/urandom.
+ +> usage: $./batch_gen.sh <#number of networks needed>
diff --git a/gennet-module/batch_gen.sh b/gennet-module/batch_gen.sh new file mode 100755 index 0000000..bc0ffdf --- /dev/null +++ b/gennet-module/batch_gen.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +#MAX and MIN for topics and num nodes +MIN=5 +MAX=100 + +#requires bc +getrand(){ + orig=$(od -An -N1 -i /dev/urandom) + range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc` + RANDOM=$range +} + +getrand1(){ + orig=$(od -An -N1 -i /dev/urandom) + range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc` + return range + #getrand1 # call the fun and use the return value + #n=$? +} + +if [ "$#" -ne 2 ] || [ $2 -le 0 ] ; then + echo "usage: $0 <#json files needed>" >&2 + exit 1 +fi + +path=$1 +nfiles=$2 +mkdir -p $path + +echo "Ok, will generate $nfiles networks & put them under '$path'." + +nwtype="newmanwattsstrogatz" +nodetype="desktop" + + +for i in $(seq $nfiles) +do + getrand + n=$((RANDOM+1)) + getrand + t=$((RANDOM+1)) + getrand + s=`expr $((RANDOM+1)) % $n` + + dirname="$path/$i/Waku" + mkdir "$path/$i" + echo "Generating ./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1 --num-subnets $s ...." + $(./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1 --num-subnets $s) +done diff --git a/gennet-module/build.sh b/gennet-module/build.sh new file mode 100644 index 0000000..99d2919 --- /dev/null +++ b/gennet-module/build.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# pip freeze > requirements.txt +docker image build --progress=plain -t gennet:0.0.1 ./ diff --git a/gennet-module/generate_network.py b/gennet-module/generate_network.py new file mode 100755 index 0000000..980f9de --- /dev/null +++ b/gennet-module/generate_network.py @@ -0,0 +1,243 @@ +#! /usr/bin/env python3 + +import matplotlib.pyplot as plt +import networkx as nx +import random, math +import json +import sys, os +import string +import typer +from enum import Enum + + +# Enums & Consts + +# To add a new node type, add appropriate entries to the nodeType and nodeTypeSwitch +class nodeType(Enum): + DESKTOP = "desktop" # waku desktop config + MOBILE = "mobile" # waku mobile config + +nodeTypeSwitch = { + nodeType.DESKTOP : "rpc-admin = true\nkeep-alive = true\n", + nodeType.MOBILE : "rpc-admin = true\nkeep-alive = true\n" + } + + +# To add a new network type, add appropriate entries to the networkType and networkTypeSwitch +# the networkTypeSwitch is placed before generate_network(): fwd declaration mismatch with typer/python :/ +class networkType(Enum): + CONFIGMODEL = "configmodel" + SCALEFREE = "scalefree" # power law + NEWMANWATTSSTROGATZ = "newmanwattsstrogatz" # mesh, smallworld + BARBELL = "barbell" # partition + BALANCEDTREE = "balancedtree" # committees? + STAR = "star" # spof + + +NW_DATA_FNAME = "network_data.json" +NODE_PREFIX = "waku" +SUBNET_PREFIX = "subnetwork" + + +### I/O related fns ############################################################## + +# Dump to a json file +def write_json(dirname, json_dump): + fname = os.path.join(dirname, NW_DATA_FNAME) + with open(fname, "w") as f: + json.dump(json_dump, f, indent=2) + + +def write_toml(dirname, node_name, toml): + fname = os.path.join(dirname, f"{node_name}.toml") + with open(fname, "w") as f: + f.write(toml) + + +# Draw the network and output the image to a file; does not account for subnets yet +def draw(dirname, H): + nx.draw(H, pos=nx.kamada_kawai_layout(H), with_labels=True) + fname = os.path.join(dirname, NW_DATA_FNAME) + plt.savefig(f"{os.path.splitext(fname)[0]}.png", format="png") + plt.show() + + +# Has trouble with non-integer/non-hashable keys +def read_json(fname): + with open(fname) as f: + jdata = json.load(f) + return nx.node_link_graph(jdata) + + +# check if the required dir can be created +def exists_or_nonempty(dirname): + if not os.path.exists(dirname): + return False + elif not os.path.isfile(dirname) and os.listdir(dirname): + print(f"{dirname}: exists and not empty") + return True + elif os.path.isfile(dirname): + print(f"{dirname}: exists but not a directory") + return True + else: + return False + + +### topics related fns ############################################################# + +# Generate a random string of upper case chars +def generate_random_string(n): + return "".join(random.choice(string.ascii_uppercase) for _ in range(n)) + + +# Generate the topics - topic followed by random UC chars - Eg, topic_XY" +def generate_topics(num_topics): + topic_len = int(math.log(num_topics)/math.log(26)) + 1 # base is 26 - upper case letters + topics = {i: f"topic_{generate_random_string(topic_len)}" for i in range(num_topics)} + return topics + + +# Get a random sub-list of topics +def get_random_sublist(topics): + n = len(topics) + lo = random.randint(0, n - 1) + hi = random.randint(lo + 1, n) + sublist = [] + for i in range(lo, hi): + sublist.append(topics[i]) + return sublist + + +### network processing related fns ################################################# + +# Network Types +def generate_config_model(n): + #degrees = nx.random_powerlaw_tree_sequence(n, tries=10000) + degrees = [random.randint(1, n) for i in range(n)] + if (sum(degrees)) % 2 != 0: # adjust the degree to be even + degrees[-1] += 1 + return nx.configuration_model(degrees) # generate the graph + + +def generate_scalefree_graph(n): + return nx.scale_free_graph(n) + + +# n must be larger than k=D=3 +def generate_newmanwattsstrogatz_graph(n): + return nx.newman_watts_strogatz_graph(n, 3, 0.5) + + +def generate_barbell_graph(n): + return nx.barbell_graph(int(n/2), 1) + + +def generate_balanced_tree(n, fanout=3): + height = int(math.log(n)/math.log(fanout)) + return nx.balanced_tree(fanout, height) + + +def generate_star_graph(n): + return nx.star_graph(n) + + +networkTypeSwitch = { + networkType.CONFIGMODEL : generate_config_model, + networkType.SCALEFREE : generate_scalefree_graph, + networkType.NEWMANWATTSSTROGATZ : generate_newmanwattsstrogatz_graph, + networkType.BARBELL : generate_barbell_graph, + networkType.BALANCEDTREE: generate_balanced_tree, + networkType.STAR : generate_star_graph + } + + +# Generate the network from nw type +def generate_network(n, network_type): + return postprocess_network(networkTypeSwitch.get(network_type)(n)) + + +# Label the generated network with prefix +def postprocess_network(G): + G = nx.Graph(G) # prune out parallel/multi edges + G.remove_edges_from(nx.selfloop_edges(G)) # remove the self-loops + mapping = {i: f"{NODE_PREFIX}_{i}" for i in range(len(G))} + return nx.relabel_nodes(G, mapping) # label the nodes + + +def generate_subnets(G, num_subnets): + n = len(G.nodes) + if num_subnets == n: # if num_subnets == size of the network + return {f"{NODE_PREFIX}_{i}": f"{SUBNET_PREFIX}_{i}" for i in range(n)} + + lst = list(range(n)) + random.shuffle(lst) + offsets = sorted(random.sample(range(0, n), num_subnets - 1)) + offsets.append(n-1) + + start = 0 + subnets = {} + subnet_id = 0 + for end in offsets: + for i in range(start, end+1): + subnets[f"{NODE_PREFIX}_{lst[i]}"] = f"{SUBNET_PREFIX}_{subnet_id}" + start = end + subnet_id += 1 + return subnets + + +### file format related fns ########################################################### +#Generate per node toml configs +def generate_toml(topics, node_type=nodeType.DESKTOP): + topic_str = " ".join(get_random_sublist(topics)) # space separated topics + return f"{nodeTypeSwitch.get(node_type)}topics = \"{topic_str}\"\n" + + +# Generates network-wide json and per-node toml and writes them +def generate_and_write_files(dirname, num_topics, num_subnets, G): + topics = generate_topics(num_topics) + subnets = generate_subnets(G, num_subnets) + json_dump = {} + for node in G.nodes: + write_toml(dirname, node, generate_toml(topics)) # per node toml + json_dump[node] = {} + json_dump[node]["static_nodes"] = [] + for edge in G.edges(node): + json_dump[node]["static_nodes"].append(edge[1]) + json_dump[node][SUBNET_PREFIX] = subnets[node] + write_json(dirname, json_dump) # network wide json + + +### the main ########################################################################## +def main(config_file: str = "../config/config.json"): + + """ Load config file """ + try: + with open(config_file, 'r') as f: + config_obj = json.load(f) + except Exception as e: + print(e) + sys.exit(1) + + # sanity checks + if config_obj['num_partitions'] > 1: + raise ValueError(f"--num-partitions {config_obj['num_partitions']}, Sorry, we do not yet support partitions") + if config_obj['num_subnets'] > config_obj['num_nodes']: + raise ValueError(f"num_subnets must be <= num_nodes: num_subnets={config_obj['num_subnets']}, num_nodes={config_obj['num_nodes']}") + if config_obj['num_subnets'] == -1: + config_obj['num_subnets'] = config_obj['num_nodes'] + + # Generate the network + G = generate_network(config_obj['num_nodes'], networkType(config_obj['network_type'])) + + # Refuse to overwrite non-empty dirs + if exists_or_nonempty(config_obj['topology_path']): + sys.exit(1) + os.makedirs(config_obj['topology_path'], exist_ok=True) + + # Generate file format specific data structs and write the files; optionally, draw the network + generate_and_write_files(config_obj['topology_path'], config_obj['num_topics'], config_obj['num_subnets'], G) + #draw(dirname, G) + + +if __name__ == "__main__": + typer.run(main) diff --git a/gennet-module/requirements.txt b/gennet-module/requirements.txt new file mode 100644 index 0000000..fc504bd --- /dev/null +++ b/gennet-module/requirements.txt @@ -0,0 +1,24 @@ +asyncio==3.4.3 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.3 +contourpy==1.0.6 +cycler==0.11.0 +fonttools==4.38.0 +idna==3.4 +jsonrpcclient==4.0.2 +kiwisolver==1.4.4 +matplotlib==3.6.2 +networkx==2.8.8 +numpy==1.24.0 +packaging==22.0 +Pillow==9.3.0 +pyparsing==3.0.9 +python-dateutil==2.8.2 +PyYAML==6.0 +requests==2.28.1 +scipy==1.10.0 +six==1.16.0 +tqdm==4.64.1 +typer==0.7.0 +urllib3==1.26.13