From 4a676abdecbf19ac55ef8302149f8ff11c522cb1 Mon Sep 17 00:00:00 2001 From: Alberto Date: Wed, 10 May 2023 19:23:05 +0200 Subject: [PATCH] Added tests for new functions in prometheus.py --- analysis-module/src/prometheus.py | 25 ++-- analysis-module/tests/test_prometheus.py | 176 ++++++++++++++++++++++- 2 files changed, 182 insertions(+), 19 deletions(-) diff --git a/analysis-module/src/prometheus.py b/analysis-module/src/prometheus.py index fc437cb..ecff8fe 100644 --- a/analysis-module/src/prometheus.py +++ b/analysis-module/src/prometheus.py @@ -11,7 +11,7 @@ from src import analysis_logger def connect_to_prometheus(port): url = f"http://host.docker.internal:{port}" try: - analysis_logger.G_LOGGER.debug('Connecting to Prometheus server in %s' %url) + analysis_logger.G_LOGGER.debug('Connecting to Prometheus server in %s' % url) prometheus = PrometheusConnect(url, disable_ssl=True) except Exception as e: analysis_logger.G_LOGGER.error('%s: %s' % (e.__doc__, e)) @@ -40,19 +40,17 @@ def fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, container_ start_timestamp = datetime.utcfromtimestamp(start_ts / 1e9) end_timestamp = datetime.fromtimestamp(end_ts / 1e9) - # para cada metrica for metric in metrics["by_simulation"].values(): - # si es multiplot if type(metric["metric_name"]) is list: metric["values"] = [] - # para cada submetrica for i, submetric in enumerate(metric["metric_name"]): - # pillo los valores de todos los nodos (deberian tener las mismas timestamps) - values = fetch_accumulated_metric_for_all_nodes(prom, submetric, container_ips, start_timestamp, end_timestamp, metric["toMB"]) + values = fetch_accumulated_metric_for_all_nodes(prom, submetric, container_ips, start_timestamp, + end_timestamp, metric["toMB"]) metric["values"].append(values) else: metric.setdefault("values", []).append( - fetch_accumulated_metric_for_all_nodes(prom, metric["metric_name"], container_ips, start_timestamp, end_timestamp, metric["toMB"])) + fetch_accumulated_metric_for_all_nodes(prom, metric["metric_name"], container_ips, start_timestamp, + end_timestamp, metric["toMB"])) def fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, start_ts, end_ts): @@ -76,10 +74,11 @@ def fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, st metric.setdefault("values", []).append(stat_function(values)) -def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_timestamp, end_timestamp, to_mbytes=False): +def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_timestamp, end_timestamp, + to_mbytes=False): result = {} for ip in container_ips: - values = fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes) + values = fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp) for item in values: timestamp, value = item value = int(value) @@ -97,20 +96,20 @@ def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_ti def fetch_metric(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes=False): metric_result = prom.custom_query_range(f"{metric}{{container_label_com_kurtosistech_private_ip = '{ip}'}}", - start_time=start_timestamp, end_time=end_timestamp, step="1s") + start_time=start_timestamp, end_time=end_timestamp, step="1s") if not metric_result: analysis_logger.G_LOGGER.error(f"{metric} returns no data. Adding zero.") return [0] metric_values = [float(metric_result[0]['values'][i][1]) for i in range(len(metric_result[0]['values']))] if to_mbytes: - metric_values = [value/(1024*1024) for value in metric_values] + metric_values = [value / (1024 * 1024) for value in metric_values] return metric_values -def fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes=False): +def fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp): metric_result = prom.custom_query_range(f"{metric}{{container_label_com_kurtosistech_private_ip = '{ip}'}}", - start_time=start_timestamp, end_time=end_timestamp, step="1s") + start_time=start_timestamp, end_time=end_timestamp, step="1s") if not metric_result: analysis_logger.G_LOGGER.error(f"{metric} returns no data. Adding zero.") diff --git a/analysis-module/tests/test_prometheus.py b/analysis-module/tests/test_prometheus.py index 8192ed8..e9358d3 100644 --- a/analysis-module/tests/test_prometheus.py +++ b/analysis-module/tests/test_prometheus.py @@ -11,18 +11,18 @@ from src import prometheus class TestPrometheus(unittest.TestCase): @patch('src.prometheus.fetch_metric') - def test_fetch_cadvisor_stats_from_prometheus(self, mock_fetch_metric): + def test_fetch_cadvisor_stats_from_prometheus_by_node(self, mock_fetch_metric): mock_fetch_metric.side_effect = [ [10, 20, 30], # metric_1_name [10, 20, 30], # metric_2_name_1 - [5, 15, 25] # metric_2_name_2 + [5, 15, 25] # metric_2_name_2 ] metrics = { - "to_query": { + "by_node": { "metric_1": { "metric_name": "metric_1_name", - "statistic": "sum", + "statistic": "min", "toMB": False }, "metric_2": { @@ -39,8 +39,8 @@ class TestPrometheus(unittest.TestCase): prometheus.fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, start_ts, end_ts) - self.assertEqual(metrics["to_query"]["metric_1"]["values"], [sum([10, 20, 30])]) - self.assertEqual(metrics["to_query"]["metric_2"]["values"], [[max([10, 20, 30])], [max([5, 15, 25])]]) + self.assertEqual(metrics["by_node"]["metric_1"]["values"], [10]) + self.assertEqual(metrics["by_node"]["metric_2"]["values"], [[30], [25]]) @patch.object(PrometheusConnect, 'custom_query_range') def test_fetch_metric(self, mock_custom_query_range): @@ -80,3 +80,167 @@ class TestPrometheus(unittest.TestCase): metric_values = prometheus.fetch_metric(prom, None, None, None, None, True) self.assertEqual(metric_values, [5.1328125, 5.1328125, 5.1328125, 6.0, 9.8125]) + + @patch.object(PrometheusConnect, 'custom_query_range') + def test_fetch_metric_empty(self, mock_custom_query_range): + mock_custom_query_range.return_value = [] + prom = PrometheusConnect() + + metric_values = prometheus.fetch_metric(prom, None, None, None, None, True) + self.assertEqual(metric_values, [0]) + + @patch('src.prometheus.fetch_accumulated_metric_for_all_nodes') + def test_fetch_cadvisor_stats_from_prometheus_by_simulation(self, mock_fetch_accumulated_metric_for_all_nodes): + metrics = { + "by_simulation": { + "bandwith": { + "metric_name": [ + "container_network_receive_bytes_total", + "container_network_transmit_bytes_total" + ], + "toMB": True + } + } + } + + expected_metrics = { + "by_simulation": { + "bandwith": { + "metric_name": [ + "container_network_receive_bytes_total", + "container_network_transmit_bytes_total" + ], + "values": [[12, 17, 13, 5, 8, 4], [1, 4, 3]], + "toMB": True + } + } + } + + mock_fetch_accumulated_metric_for_all_nodes.side_effect = [[12, 17, 13, 5, 8, 4], [1, 4, 3]] + prom = PrometheusConnect() + + prometheus.fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, None, 0, 0) + + self.assertEqual(metrics, expected_metrics) + + @patch('src.prometheus.fetch_accumulated_metric_for_all_nodes') + def test_fetch_cadvisor_stats_from_prometheus_by_simulation_multiple(self, + mock_fetch_accumulated_metric_for_all_nodes): + metrics = { + "by_simulation": { + "bandwith": { + "metric_name": [ + "container_network_receive_bytes_total", + "container_network_transmit_bytes_total" + ], + "toMB": True + }, + "disk": { + "metric_name": [ + "container_fs_reads_bytes_total", + "container_fs_writes_bytes_total" + ], + "toMB": True + } + } + } + + expected_metrics = { + "by_simulation": { + "bandwith": { + "metric_name": [ + "container_network_receive_bytes_total", + "container_network_transmit_bytes_total" + ], + "values": [[5, 6, 5, 4, 8], [1, 4, 3]], + "toMB": True + }, + "disk": { + "metric_name": [ + "container_fs_reads_bytes_total", + "container_fs_writes_bytes_total" + ], + "values": [[7, 3, 1, 5, 10], [4, 7, 1]], + "toMB": True + } + } + } + + mock_fetch_accumulated_metric_for_all_nodes.side_effect = [[5, 6, 5, 4, 8], [1, 4, 3], + [7, 3, 1, 5, 10], [4, 7, 1], + ] + prom = PrometheusConnect() + + prometheus.fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, None, 0, 0) + + self.assertEqual(metrics, expected_metrics) + + @patch('src.prometheus.fetch_metric_with_timestamp') + def test_fetch_accumulated_metric_for_all_nodes_1(self, mock_fetch_metric_with_timestamp): + mock_fetch_metric_with_timestamp.side_effect = [[[1683281260, 1], + [1683281260, 4], + [1683281261, 3], + [1683281261, 3], + [1683281262, 5], + [1683281263, 4], + [1683281320, 8]]] + + prom = PrometheusConnect() + ips = ["test"] + metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None) + + self.assertEqual(metric_values, [5, 6, 5, 4, 8]) + + @patch('src.prometheus.fetch_metric_with_timestamp') + def test_fetch_accumulated_metric_for_all_nodes_2(self, mock_fetch_metric_with_timestamp): + mock_fetch_metric_with_timestamp.side_effect = [[[1683281260, 1], [1683281260, 4], [1683281261, 3], + [1683281261, 3], [1683281262, 5], [1683281263, 4], + [1683281320, 8]], + [[1683281260, 7], [1683281260, 1], [1683281261, 6], + [1683281261, 5], [1683281262, 8], [1683281263, 1], + [1683281330, 4]]] + + prom = PrometheusConnect() + ips = ["test1", "test2"] + metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None) + + self.assertEqual(metric_values, [13, 17, 13, 5, 8, 4]) + + @patch('src.prometheus.fetch_metric_with_timestamp') + def test_fetch_accumulated_metric_for_all_nodes_empty(self, mock_fetch_metric_with_timestamp): + mock_fetch_metric_with_timestamp.side_effect = [[[0, 0]], [[0, 0]]] + + prom = PrometheusConnect() + ips = ["test1", "test2"] + metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None) + + self.assertEqual(metric_values, [0]) + + @patch.object(PrometheusConnect, 'custom_query_range') + def test_fetch_metric_with_timestamp(self, mock_custom_query_range): + mock_custom_query_range.return_value = [ + { + 'metric': {'__name__': 'container_memory_usage_bytes'}, + 'values': [ + [1683281260, '5382144'], + [1683281261, '5382144'], + [1683281262, '5382144'], + [1683281263, '6291456'], + [1683281320, '10289152'] + ] + } + ] + prom = PrometheusConnect() + + metric_values = prometheus.fetch_metric_with_timestamp(prom, None, None, None, None) + self.assertEqual(metric_values, + [[1683281260, '5382144'], [1683281261, '5382144'], [1683281262, '5382144'], [1683281263, '6291456'], + [1683281320, '10289152']]) + + @patch.object(PrometheusConnect, 'custom_query_range') + def test_fetch_metric_with_timestamp_empty_data(self, mock_custom_query_range): + mock_custom_query_range.return_value = [] + prom = PrometheusConnect() + + metric_values = prometheus.fetch_metric_with_timestamp(prom, None, None, None, None) + self.assertEqual(metric_values, [[0, 0]])