Add initial Artifacts util script (#73)

- Add `artifacts.py` to `harness_utils` to support capturing of
(settings/config) artifacts during a test run.

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 Changes are grouped by the date they are merged to the main branch of the repository and are ordered from newest to oldest. Dates use the ISO 8601 extended calendar date format, i.e. YYYY-MM-DD.
 
+## 2024-09-06
+
+- Add `artifacts.py` to `harness_utils` for capturing test artifacts.
+
 ## 2024-08-30
 
 - Add Stellaris harness based on one_year console command.

harness_utils/README.md

@@ -3,15 +3,84 @@
 Harness Utils contains scripts that are loosely connected around providing helper utilities used across
 multiple test harnesses.
 
+## Artifacts
+
+`artifacts.py`
+
+Contains class for capturing test artifacts.
+
+### Usage:
+
+Import `ArtifactManager` and `ArtifactType`
+
+```
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+```
+
+Instantiate an Artifact Manager; This should ideally be the same directory as specified for the output directory of the harness, e.g. `./run`.
+
+```python
+# Assuming /run is used, get output directory relative to script location.
+SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = SCRIPT_DIR.joinpath("run")
+
+am = ArtifactManager(LOG_DIR)
+```
+
+Capture artifacts using `ArtifactManager.copy_file` and `ArtifactManager.take_screenshot`.
+
+```python
+# src as a string, capturing a configuration text file
+am.copy_file("path/to/config.ini", ArtifactType.CONFIG_TEXT, "a config file")
+
+# src as a pathlib.Path, capturing a results text file
+am.copy_file(Path("path", "to", "results.txt"), ArtifactType.RESULTS_TEXT, "some results")
+
+am.take_screenshot("cool_picture.png", ArtifactType.CONFIG_IMAGE, "picture of settings")
+```
+
+Optionally, an override to the screenshot function can optionally be provided if the `mss` library is not sufficient.
+
+```python
+def my_screenshot_function(filename: str) -> None:
+    # Take the screenshot here using the filename
+    pass
+
+am.take_screenshot("something.png", ArtifactType.CONFIG_IMAGE, "a picture taken with my function", my_screenshot_function)
+```
+
+Once all desired artifacts have been captured, create an artifact manifest with `ArtifactManager.create_manifest`.
+
+```python
+am.create_manifest()
+```
+
+Given the configuration and artifacts captured in the above code snippets, the resulting manifest should be created at `./run/artifacts.yaml` and contain the following data:
+
+```yaml
+- filename: config.ini
+  type: config_text
+  description: a config file
+- filename: results.txt
+  type: results_text
+  description: some results
+- filename: cool_picture.png
+  type: config_image
+  description: picture of settings
+- filename: something.png
+  type: config_image
+  description: a picture taken with my function
+```
+
 ## Keras Service
 
 `keras_service.py`
 
 Contains class for instancing connection to a Keras Service and provides access to its web API.
 
-## Logging
+## Output
 
-`logging.py`
+`output.py`
 
 Functions related to logging and formatting output from test harnesses.
 

harness_utils/artifacts.py

@@ -0,0 +1,128 @@
+"""Provides ArtifactManager class for capturing artifacts from test runs."""
+import os
+import mss
+import yaml
+from dataclasses import dataclass
+from enum import Enum, unique
+from shutil import copy
+from pathlib import Path
+from collections.abc import Callable
+
+
+@unique
+class ArtifactType(Enum):
+    """
+    Describes different types of artifacts to be saved from test runs.
+    """
+
+    CONFIG_IMAGE = "config_image"
+    """
+    Meant to describe images displaying an applications settings.
+    For games this might be a screenshot of the in-game settings menu.
+    """
+
+    RESULTS_IMAGE = "results_image"
+    """
+    Meant to describe images displaying results of a benchmark.
+    For games with built in benchmarks, this might be a screenshot of an in-game benchmark results screen.
+    """
+
+    CONFIG_TEXT = "config_text"
+    """
+    Meant to describe text-based files which contain application settings.
+    For games this might be a .ini or .cfg file containing graphics settings.
+    """
+
+    RESULTS_TEXT = "results_text"
+    """
+    Meant to describe text-based files which contain results of a benchmark.
+    For games with built in benchmarks, this might be a .txt or .xml file containing results from an in-game benchmark.
+    """
+
+
+_IMAGE_ARTIFACT_TYPES = (ArtifactType.CONFIG_IMAGE, ArtifactType.RESULTS_IMAGE)
+
+
+@dataclass
+class Artifact:
+    """
+    Describes an artifact captured by the ArtifactManager.
+    """
+    filename: str
+    type: ArtifactType
+    description: str
+
+    def as_dict(self) -> dict:
+        """
+        Returns the Artifact object as a dictionary. Converts its type to a string value.
+        """
+        d = self.__dict__.copy()
+        d["type"] = d["type"].value
+        return d
+
+
+class ArtifactManager:
+    """
+    Used to manage artifacts captured during a test run, either by coping files or taking screenshots.
+    The manager maintains a list of artifacts it has captured and can produce a manifest file listing them.
+    """
+    def __init__(self, output_path: str | os.PathLike) -> None:
+        self.output_path = Path(output_path)
+        self.artifacts: list[Artifact] = []
+
+        self.output_path.mkdir(parents=True, exist_ok=True)
+
+    def copy_file(self, src: str | os.PathLike, artifact_type: ArtifactType, description=""):
+        """
+        Copies a file from `src` to the manager's `output_path` and adds a new Artifact to the manager's artifacts list.
+
+        The newly created artifact's `type` and `description` fields are set to the given
+        `artifact_type` and `description` arguments respectively while the artifact's `filename`
+        is set to the basename of `src`.
+        
+        Raises a `ValueError` if `src` points to a directory instead of a file.
+        """
+        src_path = Path(src)
+        if src_path.is_dir():
+            raise ValueError("src should point to a file, not a directory")
+        filename = src_path.name
+        try:
+            copy(src, self.output_path / filename)
+            artifact = Artifact(filename, artifact_type, description)
+            self.artifacts.append(artifact)
+        except OSError as e:
+            raise e
+
+    def take_screenshot(
+            self,
+            filename: str,
+            artifact_type: ArtifactType,
+            description="",
+            screenshot_override: Callable[[str | os.PathLike], None] | None = None):
+        """
+        Takes a screenshot and saves it to the manager's `output_path` with the given `filename`
+        and adds a new Artifact to the manager's artifact list.
+
+        The newly created artifact's `filename`, `type` and `description` fields are set to the
+        given `filename`, `artifact_type` and `description` arguments respectively.
+
+        Raises a `ValueError` if `artifact_type` is not one of the `ArtifactType` values which represents an image. 
+        """
+        if artifact_type not in _IMAGE_ARTIFACT_TYPES:
+            raise ValueError("artifact_type should be a type that represents an image artifact")
+
+        if screenshot_override is None:
+            with mss.mss() as sct:
+                sct.shot(output=str(self.output_path / filename))
+        else:
+            screenshot_override(self.output_path / filename)
+        artifact = Artifact(filename, artifact_type, description)
+        self.artifacts.append(artifact)
+
+    def create_manifest(self):
+        """
+        Creates a file `artifacts.yaml` which lists the artifacts in the manager's `artifacts` list.
+        The file is created at the location specified by the manager's `output_path`.
+        """
+        with open(self.output_path / "artifacts.yaml", encoding="utf-8", mode="w") as f:
+            yaml.safe_dump([a.as_dict() for a in self.artifacts], f, sort_keys=False)