AtHeartEngineer/wakurtosis
1
0
Fork 0
mirror of https://github.com/vacp2p/wakurtosis.git synced 2026-01-09 14:58:02 -05:00
Code Issues Packages Projects Releases Wiki Activity

Added tests for new functions in prometheus.py

Browse Source
This commit is contained in:
Alberto
2023-05-10 19:23:05 +02:00
parent 3d93bc371f
commit 4a676abdec
2 changed files with 182 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
analysis-module/src/prometheus.py
View File

@@ -11,7 +11,7 @@ from src import analysis_logger
def connect_to_prometheus(port):
url = f"http://host.docker.internal:{port}"
try:
analysis_logger.G_LOGGER.debug('Connecting to Prometheus server in %s' %url)
analysis_logger.G_LOGGER.debug('Connecting to Prometheus server in %s' % url)
prometheus = PrometheusConnect(url, disable_ssl=True)
except Exception as e:
analysis_logger.G_LOGGER.error('%s: %s' % (e.__doc__, e))
@@ -40,19 +40,17 @@ def fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, container_
start_timestamp = datetime.utcfromtimestamp(start_ts / 1e9)
end_timestamp = datetime.fromtimestamp(end_ts / 1e9)
# para cada metrica
for metric in metrics["by_simulation"].values():
# si es multiplot
if type(metric["metric_name"]) is list:
metric["values"] = []
# para cada submetrica
for i, submetric in enumerate(metric["metric_name"]):
# pillo los valores de todos los nodos (deberian tener las mismas timestamps)
values = fetch_accumulated_metric_for_all_nodes(prom, submetric, container_ips, start_timestamp, end_timestamp, metric["toMB"])
values = fetch_accumulated_metric_for_all_nodes(prom, submetric, container_ips, start_timestamp,
end_timestamp, metric["toMB"])
metric["values"].append(values)
else:
metric.setdefault("values", []).append(
fetch_accumulated_metric_for_all_nodes(prom, metric["metric_name"], container_ips, start_timestamp, end_timestamp, metric["toMB"]))
fetch_accumulated_metric_for_all_nodes(prom, metric["metric_name"], container_ips, start_timestamp,
end_timestamp, metric["toMB"]))
def fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, start_ts, end_ts):
@@ -76,10 +74,11 @@ def fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, st
metric.setdefault("values", []).append(stat_function(values))
def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_timestamp, end_timestamp, to_mbytes=False):
def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_timestamp, end_timestamp,
to_mbytes=False):
result = {}
for ip in container_ips:
values = fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes)
values = fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp)
for item in values:
timestamp, value = item
value = int(value)
@@ -97,20 +96,20 @@ def fetch_accumulated_metric_for_all_nodes(prom, metric, container_ips, start_ti
def fetch_metric(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes=False):
metric_result = prom.custom_query_range(f"{metric}{{container_label_com_kurtosistech_private_ip = '{ip}'}}",
start_time=start_timestamp, end_time=end_timestamp, step="1s")
start_time=start_timestamp, end_time=end_timestamp, step="1s")
if not metric_result:
analysis_logger.G_LOGGER.error(f"{metric} returns no data. Adding zero.")
return [0]
metric_values = [float(metric_result[0]['values'][i][1]) for i in range(len(metric_result[0]['values']))]
if to_mbytes:
metric_values = [value/(1024*1024) for value in metric_values]
metric_values = [value / (1024 * 1024) for value in metric_values]
return metric_values
def fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp, to_mbytes=False):
def fetch_metric_with_timestamp(prom, metric, ip, start_timestamp, end_timestamp):
metric_result = prom.custom_query_range(f"{metric}{{container_label_com_kurtosistech_private_ip = '{ip}'}}",
start_time=start_timestamp, end_time=end_timestamp, step="1s")
start_time=start_timestamp, end_time=end_timestamp, step="1s")
if not metric_result:
analysis_logger.G_LOGGER.error(f"{metric} returns no data. Adding zero.")

176
analysis-module/tests/test_prometheus.py
View File

@@ -11,18 +11,18 @@ from src import prometheus
class TestPrometheus(unittest.TestCase):
@patch('src.prometheus.fetch_metric')
def test_fetch_cadvisor_stats_from_prometheus(self, mock_fetch_metric):
def test_fetch_cadvisor_stats_from_prometheus_by_node(self, mock_fetch_metric):
mock_fetch_metric.side_effect = [
[10, 20, 30], # metric_1_name
[10, 20, 30], # metric_2_name_1
[5, 15, 25] # metric_2_name_2
[5, 15, 25] # metric_2_name_2
]
metrics = {
"to_query": {
"by_node": {
"metric_1": {
"metric_name": "metric_1_name",
"statistic": "sum",
"statistic": "min",
"toMB": False
},
"metric_2": {
@@ -39,8 +39,8 @@ class TestPrometheus(unittest.TestCase):
prometheus.fetch_cadvisor_stats_from_prometheus_by_node(metrics, prom, container_ip, start_ts, end_ts)
self.assertEqual(metrics["to_query"]["metric_1"]["values"], [sum([10, 20, 30])])
self.assertEqual(metrics["to_query"]["metric_2"]["values"], [[max([10, 20, 30])], [max([5, 15, 25])]])
self.assertEqual(metrics["by_node"]["metric_1"]["values"], [10])
self.assertEqual(metrics["by_node"]["metric_2"]["values"], [[30], [25]])
@patch.object(PrometheusConnect, 'custom_query_range')
def test_fetch_metric(self, mock_custom_query_range):
@@ -80,3 +80,167 @@ class TestPrometheus(unittest.TestCase):
metric_values = prometheus.fetch_metric(prom, None, None, None, None, True)
self.assertEqual(metric_values, [5.1328125, 5.1328125, 5.1328125, 6.0, 9.8125])
@patch.object(PrometheusConnect, 'custom_query_range')
def test_fetch_metric_empty(self, mock_custom_query_range):
mock_custom_query_range.return_value = []
prom = PrometheusConnect()
metric_values = prometheus.fetch_metric(prom, None, None, None, None, True)
self.assertEqual(metric_values, [0])
@patch('src.prometheus.fetch_accumulated_metric_for_all_nodes')
def test_fetch_cadvisor_stats_from_prometheus_by_simulation(self, mock_fetch_accumulated_metric_for_all_nodes):
metrics = {
"by_simulation": {
"bandwith": {
"metric_name": [
"container_network_receive_bytes_total",
"container_network_transmit_bytes_total"
],
"toMB": True
}
}
}
expected_metrics = {
"by_simulation": {
"bandwith": {
"metric_name": [
"container_network_receive_bytes_total",
"container_network_transmit_bytes_total"
],
"values": [[12, 17, 13, 5, 8, 4], [1, 4, 3]],
"toMB": True
}
}
}
mock_fetch_accumulated_metric_for_all_nodes.side_effect = [[12, 17, 13, 5, 8, 4], [1, 4, 3]]
prom = PrometheusConnect()
prometheus.fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, None, 0, 0)
self.assertEqual(metrics, expected_metrics)
@patch('src.prometheus.fetch_accumulated_metric_for_all_nodes')
def test_fetch_cadvisor_stats_from_prometheus_by_simulation_multiple(self,
mock_fetch_accumulated_metric_for_all_nodes):
metrics = {
"by_simulation": {
"bandwith": {
"metric_name": [
"container_network_receive_bytes_total",
"container_network_transmit_bytes_total"
],
"toMB": True
},
"disk": {
"metric_name": [
"container_fs_reads_bytes_total",
"container_fs_writes_bytes_total"
],
"toMB": True
}
}
}
expected_metrics = {
"by_simulation": {
"bandwith": {
"metric_name": [
"container_network_receive_bytes_total",
"container_network_transmit_bytes_total"
],
"values": [[5, 6, 5, 4, 8], [1, 4, 3]],
"toMB": True
},
"disk": {
"metric_name": [
"container_fs_reads_bytes_total",
"container_fs_writes_bytes_total"
],
"values": [[7, 3, 1, 5, 10], [4, 7, 1]],
"toMB": True
}
}
}
mock_fetch_accumulated_metric_for_all_nodes.side_effect = [[5, 6, 5, 4, 8], [1, 4, 3],
[7, 3, 1, 5, 10], [4, 7, 1],
]
prom = PrometheusConnect()
prometheus.fetch_cadvisor_stats_from_prometheus_by_simulation(metrics, prom, None, 0, 0)
self.assertEqual(metrics, expected_metrics)
@patch('src.prometheus.fetch_metric_with_timestamp')
def test_fetch_accumulated_metric_for_all_nodes_1(self, mock_fetch_metric_with_timestamp):
mock_fetch_metric_with_timestamp.side_effect = [[[1683281260, 1],
[1683281260, 4],
[1683281261, 3],
[1683281261, 3],
[1683281262, 5],
[1683281263, 4],
[1683281320, 8]]]
prom = PrometheusConnect()
ips = ["test"]
metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None)
self.assertEqual(metric_values, [5, 6, 5, 4, 8])
@patch('src.prometheus.fetch_metric_with_timestamp')
def test_fetch_accumulated_metric_for_all_nodes_2(self, mock_fetch_metric_with_timestamp):
mock_fetch_metric_with_timestamp.side_effect = [[[1683281260, 1], [1683281260, 4], [1683281261, 3],
[1683281261, 3], [1683281262, 5], [1683281263, 4],
[1683281320, 8]],
[[1683281260, 7], [1683281260, 1], [1683281261, 6],
[1683281261, 5], [1683281262, 8], [1683281263, 1],
[1683281330, 4]]]
prom = PrometheusConnect()
ips = ["test1", "test2"]
metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None)
self.assertEqual(metric_values, [13, 17, 13, 5, 8, 4])
@patch('src.prometheus.fetch_metric_with_timestamp')
def test_fetch_accumulated_metric_for_all_nodes_empty(self, mock_fetch_metric_with_timestamp):
mock_fetch_metric_with_timestamp.side_effect = [[[0, 0]], [[0, 0]]]
prom = PrometheusConnect()
ips = ["test1", "test2"]
metric_values = prometheus.fetch_accumulated_metric_for_all_nodes(prom, None, ips, None, None, None)
self.assertEqual(metric_values, [0])
@patch.object(PrometheusConnect, 'custom_query_range')
def test_fetch_metric_with_timestamp(self, mock_custom_query_range):
mock_custom_query_range.return_value = [
{
'metric': {'__name__': 'container_memory_usage_bytes'},
'values': [
[1683281260, '5382144'],
[1683281261, '5382144'],
[1683281262, '5382144'],
[1683281263, '6291456'],
[1683281320, '10289152']
]
}
]
prom = PrometheusConnect()
metric_values = prometheus.fetch_metric_with_timestamp(prom, None, None, None, None)
self.assertEqual(metric_values,
[[1683281260, '5382144'], [1683281261, '5382144'], [1683281262, '5382144'], [1683281263, '6291456'],
[1683281320, '10289152']])
@patch.object(PrometheusConnect, 'custom_query_range')
def test_fetch_metric_with_timestamp_empty_data(self, mock_custom_query_range):
mock_custom_query_range.return_value = []
prom = PrometheusConnect()
metric_values = prometheus.fetch_metric_with_timestamp(prom, None, None, None, None)
self.assertEqual(metric_values, [[0, 0]])