Gennet accepts single command line parameter and reads config from global json file

config/config.json

@@ -1,6 +1,6 @@
 {
     "enclave_name" : "wakurtosis",
-    "topology_path" : "./config/network_topology_auto/",
+    "topology_path" : "../config/network_topology_auto/",
     "same_toml_configuration": true,
     "simulation_time": 60,
     "message_rate": 10,

gennet-module/Readme.md

@@ -0,0 +1,12 @@
+This repo contains scripts to generate network models (in JSON) and waku configuration files (in TOMLs) for wakukurtosis runs. 
+
+
+## generate_network.py
+generate_network.py generates one network and per-node configuration files. The tool is configurable with specified number of nodes, topics, network types, node types and number of subnets. Use with Python3. Comment out the `#draw(fname, H)` line to visualise the generated graph.
+
+> usage: $./generate_network --help
+
+## batch_gen.sh
+batch_gen.sh can generate given number of Waku networks and outputs them to a directory. Please make sure that the output directory does NOT exist; both relative and absolute paths work. The Wakunode parameters are generated at random; edit the MIN and MAX for finer control. The script requires bc & /dev/urandom.<br>
+
+> usage: $./batch_gen.sh <output-dir> <#number of networks needed> </br>

gennet-module/batch_gen.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+#MAX and MIN for topics and num nodes
+MIN=5
+MAX=100
+
+#requires bc
+getrand(){
+  orig=$(od -An -N1 -i /dev/urandom)
+  range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc`
+  RANDOM=$range
+}
+
+getrand1(){
+  orig=$(od -An -N1 -i /dev/urandom)
+  range=`echo "$MIN + ($orig % ($MAX - $MIN + 1))" | bc`
+  return range
+  #getrand1  # call the fun and use the return value
+  #n=$? 
+}
+
+if [ "$#" -ne 2 ] || [ $2 -le 0 ] ; then
+  echo "usage: $0 <output dir> <#json files needed>" >&2
+  exit 1
+fi
+
+path=$1
+nfiles=$2
+mkdir -p $path
+
+echo "Ok, will generate $nfiles networks & put them  under '$path'."
+ 
+nwtype="newmanwattsstrogatz"
+nodetype="desktop"
+
+
+for i in $(seq $nfiles)
+do
+  getrand
+  n=$((RANDOM+1))
+  getrand 
+  t=$((RANDOM+1))
+  getrand 
+  s=`expr $((RANDOM+1)) % $n`
+
+  dirname="$path/$i/Waku"
+  mkdir "$path/$i"
+  echo "Generating ./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1 --num-subnets $s ...."
+  $(./generate_network.py --dirname $dirname --num-nodes $n --num-topics $t --nw-type $nwtype --node-type $nodetype --num-partitions 1 --num-subnets $s)
+done

gennet-module/build.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+# pip freeze > requirements.txt
+docker image build --progress=plain  -t gennet:0.0.1 ./

gennet-module/generate_network.py

@@ -0,0 +1,243 @@
+#! /usr/bin/env python3
+
+import matplotlib.pyplot as plt
+import networkx as nx
+import random, math
+import json
+import sys, os
+import string
+import typer
+from enum import Enum
+
+
+# Enums & Consts
+
+# To add a new node type, add appropriate entries to the nodeType and nodeTypeSwitch
+class nodeType(Enum):
+    DESKTOP = "desktop" # waku desktop config
+    MOBILE  = "mobile"  # waku mobile config
+
+nodeTypeSwitch = {
+        nodeType.DESKTOP : "rpc-admin = true\nkeep-alive = true\n",
+        nodeType.MOBILE  : "rpc-admin = true\nkeep-alive = true\n"
+    }
+
+
+# To add a new network type, add appropriate entries to the networkType and networkTypeSwitch
+# the networkTypeSwitch is placed before generate_network(): fwd declaration mismatch with typer/python :/
+class networkType(Enum):
+    CONFIGMODEL         = "configmodel"
+    SCALEFREE           = "scalefree"           # power law
+    NEWMANWATTSSTROGATZ = "newmanwattsstrogatz" # mesh, smallworld
+    BARBELL             = "barbell"             # partition
+    BALANCEDTREE        = "balancedtree"        # committees?
+    STAR                = "star"                # spof
+
+
+NW_DATA_FNAME = "network_data.json"
+NODE_PREFIX = "waku"
+SUBNET_PREFIX = "subnetwork"
+
+
+### I/O related fns ##############################################################
+
+# Dump to a json file
+def write_json(dirname, json_dump):
+    fname = os.path.join(dirname, NW_DATA_FNAME)
+    with open(fname, "w") as f:
+        json.dump(json_dump, f, indent=2)
+
+
+def write_toml(dirname, node_name, toml):
+    fname = os.path.join(dirname, f"{node_name}.toml")
+    with open(fname, "w") as f:
+        f.write(toml)
+
+
+# Draw the network and output the image to a file; does not account for subnets yet
+def draw(dirname, H):
+    nx.draw(H, pos=nx.kamada_kawai_layout(H), with_labels=True)
+    fname = os.path.join(dirname, NW_DATA_FNAME)
+    plt.savefig(f"{os.path.splitext(fname)[0]}.png", format="png")
+    plt.show()
+
+
+# Has trouble with non-integer/non-hashable keys 
+def read_json(fname):
+    with open(fname) as f:
+        jdata = json.load(f)
+    return nx.node_link_graph(jdata)
+
+
+# check if the required dir can be created
+def exists_or_nonempty(dirname):
+    if not os.path.exists(dirname):
+        return False
+    elif not os.path.isfile(dirname) and os.listdir(dirname):
+        print(f"{dirname}: exists and not empty")
+        return True
+    elif os.path.isfile(dirname):
+        print(f"{dirname}: exists but not a directory")
+        return True
+    else:
+        return False
+
+
+### topics related fns #############################################################
+
+# Generate a random string of upper case chars
+def generate_random_string(n):
+    return "".join(random.choice(string.ascii_uppercase) for _ in range(n))
+
+
+# Generate the topics - topic followed by random UC chars - Eg, topic_XY"
+def generate_topics(num_topics):
+    topic_len = int(math.log(num_topics)/math.log(26)) + 1  # base is 26 - upper case letters
+    topics = {i: f"topic_{generate_random_string(topic_len)}" for i in range(num_topics)}
+    return topics
+
+
+# Get a random sub-list of topics
+def get_random_sublist(topics):
+    n = len(topics)
+    lo = random.randint(0, n - 1)
+    hi = random.randint(lo + 1, n)
+    sublist = []
+    for i in range(lo, hi):
+        sublist.append(topics[i])
+    return sublist
+
+
+### network processing related fns #################################################
+
+# Network Types
+def generate_config_model(n):
+    #degrees = nx.random_powerlaw_tree_sequence(n, tries=10000)
+    degrees = [random.randint(1, n) for i in range(n)]
+    if (sum(degrees)) % 2 != 0:         # adjust the degree to be even
+        degrees[-1] += 1
+    return nx.configuration_model(degrees) # generate the graph
+
+
+def generate_scalefree_graph(n):
+    return  nx.scale_free_graph(n)
+
+
+# n must be larger than k=D=3
+def generate_newmanwattsstrogatz_graph(n):
+    return nx.newman_watts_strogatz_graph(n, 3, 0.5)
+
+
+def generate_barbell_graph(n):
+    return nx.barbell_graph(int(n/2), 1)
+
+
+def generate_balanced_tree(n, fanout=3):
+    height = int(math.log(n)/math.log(fanout))
+    return nx.balanced_tree(fanout, height)
+
+
+def generate_star_graph(n):
+    return nx.star_graph(n)
+
+
+networkTypeSwitch = {
+        networkType.CONFIGMODEL : generate_config_model,
+        networkType.SCALEFREE   : generate_scalefree_graph,
+        networkType.NEWMANWATTSSTROGATZ : generate_newmanwattsstrogatz_graph,
+        networkType.BARBELL     : generate_barbell_graph,
+        networkType.BALANCEDTREE: generate_balanced_tree,
+        networkType.STAR        : generate_star_graph
+    }
+
+
+# Generate the network from nw type
+def generate_network(n, network_type):
+    return postprocess_network(networkTypeSwitch.get(network_type)(n))
+
+
+# Label the generated network with prefix
+def postprocess_network(G):
+    G = nx.Graph(G)                             # prune out parallel/multi edges
+    G.remove_edges_from(nx.selfloop_edges(G))   # remove the self-loops
+    mapping = {i: f"{NODE_PREFIX}_{i}" for i in range(len(G))}
+    return nx.relabel_nodes(G, mapping)         # label the nodes
+
+
+def generate_subnets(G, num_subnets):
+    n = len(G.nodes)
+    if num_subnets == n:   # if num_subnets == size of the network
+        return {f"{NODE_PREFIX}_{i}": f"{SUBNET_PREFIX}_{i}" for i in range(n)}
+
+    lst = list(range(n))
+    random.shuffle(lst)
+    offsets = sorted(random.sample(range(0, n), num_subnets - 1))
+    offsets.append(n-1)
+
+    start = 0
+    subnets = {}
+    subnet_id =  0
+    for end in offsets:
+        for i in range(start, end+1):
+            subnets[f"{NODE_PREFIX}_{lst[i]}"] = f"{SUBNET_PREFIX}_{subnet_id}"
+        start = end
+        subnet_id += 1
+    return subnets 
+
+
+### file format related fns ###########################################################
+#Generate per node toml configs
+def generate_toml(topics, node_type=nodeType.DESKTOP):
+    topic_str = " ".join(get_random_sublist(topics))   # space separated topics
+    return f"{nodeTypeSwitch.get(node_type)}topics = \"{topic_str}\"\n"
+
+
+# Generates network-wide json and per-node toml and writes them 
+def generate_and_write_files(dirname, num_topics, num_subnets, G):
+    topics = generate_topics(num_topics)
+    subnets = generate_subnets(G, num_subnets)
+    json_dump = {}
+    for node in G.nodes:
+        write_toml(dirname, node, generate_toml(topics))        # per node toml
+        json_dump[node] = {}
+        json_dump[node]["static_nodes"] = []
+        for edge in G.edges(node):
+            json_dump[node]["static_nodes"].append(edge[1])
+        json_dump[node][SUBNET_PREFIX] = subnets[node]
+    write_json(dirname, json_dump)                              # network wide json
+
+
+### the main ##########################################################################
+def main(config_file: str = "../config/config.json"):
+
+    """ Load config file """
+    try:
+        with open(config_file, 'r') as f:
+            config_obj = json.load(f)
+    except Exception as e:
+        print(e)
+        sys.exit(1)
+    
+    # sanity checks
+    if config_obj['num_partitions'] > 1:
+        raise ValueError(f"--num-partitions {config_obj['num_partitions']}, Sorry, we do not yet support partitions")
+    if config_obj['num_subnets'] > config_obj['num_nodes']:
+        raise ValueError(f"num_subnets must be <= num_nodes: num_subnets={config_obj['num_subnets']}, num_nodes={config_obj['num_nodes']}")
+    if  config_obj['num_subnets'] == -1:
+        config_obj['num_subnets'] = config_obj['num_nodes']
+
+    # Generate the network
+    G = generate_network(config_obj['num_nodes'], networkType(config_obj['network_type']))
+
+    # Refuse to overwrite non-empty dirs
+    if exists_or_nonempty(config_obj['topology_path']):
+        sys.exit(1)
+    os.makedirs(config_obj['topology_path'], exist_ok=True)
+
+    # Generate file format specific data structs and write the files; optionally, draw the network
+    generate_and_write_files(config_obj['topology_path'], config_obj['num_topics'], config_obj['num_subnets'], G)
+    #draw(dirname, G)
+
+
+if __name__ == "__main__":
+    typer.run(main)

gennet-module/requirements.txt

@@ -0,0 +1,24 @@
+asyncio==3.4.3
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.3
+contourpy==1.0.6
+cycler==0.11.0
+fonttools==4.38.0
+idna==3.4
+jsonrpcclient==4.0.2
+kiwisolver==1.4.4
+matplotlib==3.6.2
+networkx==2.8.8
+numpy==1.24.0
+packaging==22.0
+Pillow==9.3.0
+pyparsing==3.0.9
+python-dateutil==2.8.2
+PyYAML==6.0
+requests==2.28.1
+scipy==1.10.0
+six==1.16.0
+tqdm==4.64.1
+typer==0.7.0
+urllib3==1.26.13