Solved conflicts with prometheus dump

analysis-module/src/plotting_configurations.py

@@ -17,6 +17,7 @@ plotting_config = {
         "xtic_labels": [
             "Received (Rx)"
         ],
+        "statistic": "max",
         "toMB": True
     },
     "container_network_transmit_bytes_total": {
@@ -28,6 +29,7 @@ plotting_config = {
         "xtic_labels": [
             "Sent (Tx)"
         ],
+        "statistic": "max",
         "toMB": True
     },
     "container_fs_reads_bytes_total": {
@@ -39,6 +41,7 @@ plotting_config = {
         "xtic_labels": [
             "Read"
         ],
+        "statistic": "max",
         "toMB": True
     },
     "container_fs_writes_bytes_total": {
@@ -50,6 +53,7 @@ plotting_config = {
         "xtic_labels": [
             "Write"
         ],
+        "statistic": "max",
         "toMB": True
     }
 }

analysis-module/src/prometheus.py

@@ -35,8 +35,8 @@ def get_hardware_metrics(metrics, topology, min_tss, max_tss, prom_port):
             analysis_logger.G_LOGGER.error('%s: %s' % (e.__doc__, e))
             continue
 
-    fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prometheus, container_ips, min_tss,
-                                                       max_tss)
+    #fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prometheus, container_ips, min_tss,
+    #                                                   max_tss)
 
 
 def fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, container_ips, start_ts,
@@ -117,5 +117,5 @@ def fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp
 function_dispatcher = {
     "max": max,
     "min": min,
-    "average": lambda x: sum(x) / len(x),
+    "average": lambda x: sum(x) / len(x)
 }

build.sh

@@ -1,4 +1,4 @@
-## Install Docker
+# Install Docker
 sudo apt-get update
 sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin python3-venv
 

config/config.json

@@ -45,13 +45,11 @@
   "plotting": {
     "by_node": [
       "container_cpu_load_average_10s",
-      "container_memory_usage_bytes"
-    ],
-    "by_simulation": [
-        "container_network_receive_bytes_total",
-        "container_network_transmit_bytes_total",
-        "container_fs_reads_bytes_total",
-        "container_fs_writes_bytes_total"
+      "container_memory_usage_bytes",
+      "container_network_receive_bytes_total",
+      "container_network_transmit_bytes_total",
+      "container_fs_reads_bytes_total",
+      "container_fs_writes_bytes_total"
     ]
   }
 }

main.star

@@ -7,13 +7,11 @@ grafana = import_module(vars.GRAFANA_MODULE)
 args_parser = import_module(vars.ARGUMENT_PARSER_MODULE)
 wls = import_module(vars.WLS_MODULE)
 nodes = import_module(vars.NODE_BUILDERS_MODULE)
-nwaku_builder = import_module(vars.NWAKU_BUILDER_MODULE)
-waku = import_module(vars.WAKU_MODULE)
 
 
 def run(plan, args):
-    plan.print(args)
-    # Load global config_file_content file
+
+    # Load global config file
     config_file = args_parser.get_configuration_file_name(plan, args)
     config_json = read_file(src=config_file)
     config = json.decode(config_json)
@@ -37,5 +35,5 @@ def run(plan, args):
         nodes.interconnect_nodes(plan, network_topology, interconnection_batch)
 
     # Setup WLS & Start the Simulation
-    wls_service = wls.init(plan, network_topology, config_file)
+    wls_service = wls.init(plan, network_topology, config_file, prometheus_service)
 

run.sh

@@ -233,6 +233,7 @@ echo "- Configuration file:  $wakurtosis_config_file" >> ./${enclave_name}_logs/
 # Copy simulation results
 docker cp "$wls_cid:/wls/network_topology/network_data.json" "./${enclave_name}_logs"
 docker cp "$wls_cid:/wls/messages.json" "./${enclave_name}_logs"
+docker cp "$wls_cid:/wls/prometheus_data.json" "./${enclave_name}_logs"
 
 # Run analysis
 if jq -e ."plotting" >/dev/null 2>&1 "./config/${wakurtosis_config_file}"; then

src/wls.star

@@ -33,7 +33,7 @@ def create_new_topology_information(plan, network_topology, network_artifact_nam
     return artifact_id
 
 
-def create_cmd(config_file):
+def create_cmd(config_file, prometheus_service):
     cmd = []
     config_file_name = config_file.split("/")[-1]
 
@@ -41,10 +41,14 @@ def create_cmd(config_file):
     cmd.append(vars.WLS_CONFIG_PATH + config_file_name)
     cmd.append(vars.WLS_TOPOLOGY_FILE_FLAG)
     cmd.append(vars.WLS_TOPOLOGY_PATH + vars.CONTAINER_TOPOLOGY_FILE_NAME_WLS)
+    cmd.append("--prometheus-ip")
+    cmd.append(prometheus_service.ip_address)
+    cmd.append("--prometheus-port")
+    cmd.append(str(prometheus_service.ports[vars.PROMETHEUS_PORT_ID].number))
 
     return cmd
 
-def init(plan, network_topology, config_file):
+def init(plan, network_topology, config_file, prometheus_service):
     
     # Generate simulation config
     config_artifact = upload_config(plan, config_file, vars.WLS_CONFIG_ARTIFACT_NAME)
@@ -58,7 +62,7 @@ def init(plan, network_topology, config_file):
     wls_topology = create_new_topology_information(plan, network_topology,
                                                    vars.WLS_TOPOLOGY_ARTIFACT_NAME)
 
-    wls_cmd = create_cmd(config_file)
+    wls_cmd = create_cmd(config_file, prometheus_service)
 
     add_service_config = ServiceConfig(
         image=vars.WLS_IMAGE,

wls-module/requirements.txt

@@ -24,3 +24,4 @@ tqdm==4.64.1
 typer==0.7.0
 urllib3==1.26.13
 aiohttp==3.8.4
+prometheus-api-client==0.5.3

wls-module/src/utils/prometheus.py

@@ -0,0 +1,34 @@
+# Python Imports
+import json
+from prometheus_api_client import PrometheusConnect
+
+
+def connect_to_prometheus(ip, port):
+    print(f"Connecting to {ip}:{port}")
+    url = f"http://{ip}:{port}"
+    try:
+        prometheus = PrometheusConnect(url, disable_ssl=True)
+    except Exception as e:
+        print("Cannot connect to Prometheus Service")
+        print(e)
+        return None
+
+    return prometheus
+
+
+def dump_prometheus(config, prometheus_ip, prometheus_port):
+    to_query = config["plotting"]["by_node"]
+
+    to_query = "|".join(to_query)
+
+    print(to_query)
+
+    prometheus_connection = connect_to_prometheus(prometheus_ip, prometheus_port)
+
+    query = f"{{__name__=~\"{to_query}\"}}"
+    print(query)
+
+    metrics = prometheus_connection.custom_query(query)
+
+    with open("/wls/prometheus_data.json", "w") as out_file:
+        json.dump(metrics, out_file)

wls-module/src/wls.py

@@ -2,7 +2,6 @@
 import argparse
 import hashlib
 import random
-import os
 import sys
 import time
 import tomllib
@@ -14,6 +13,7 @@ from src.utils import wls_logger
 from src.utils import waku_messaging
 from src.utils import payloads
 from src.utils import files
+from src.utils import prometheus
 
 """ Globals """
 G_DEFAULT_CONFIG_FILE = 'config.json'
@@ -27,6 +27,10 @@ def parse_cli(args):
                         default=G_DEFAULT_CONFIG_FILE)
     parser.add_argument("-t", "--topology_file", type=str, help="Topology file",
                         default=G_DEFAULT_TOPOLOGY_FILE)
+    parser.add_argument("-pi", "--prometheus-ip", type=str, help="Prometheus Port",
+                        default=None)
+    parser.add_argument("-pp", "--prometheus-port", type=str, help="Prometheus Port",
+                        default=None)
 
     parsed_args = parser.parse_args(args)
 
@@ -167,9 +171,11 @@ async def main():
 
     config_file = args.config_file
     topology_file = args.topology_file
+    prometheus_ip = args.prometheus_ip
+    prometheus_port = args.prometheus_port
         
     config = files.load_config_file(config_file)
-    
+
     # Set loglevel from config
     wls_config = config['wls']
 
@@ -199,6 +205,10 @@ async def main():
     if os.path.exists('/wls/start.signal'):
         os.remove('/wls/start.signal')
 
+    if prometheus_port is not None:
+        prometheus.dump_prometheus(config, prometheus_ip, prometheus_port)
+
 
 if __name__ == "__main__":
     asyncio.run(main())
+