Add back shadow of the tomb raider, but with Artifact Manager support (#75)

CHANGELOG.md

@@ -7,6 +7,7 @@ Changes are grouped by the date they are merged to the main branch of the reposi
 ## 2024-09-13
 
 - Update godot compile harness with some path fixes.
+- Add back Shadow of The Tomb Raider harness that uses Keras.
 
 ## 2024-09-06
 

harness_utils/keras_service.py

@@ -1,6 +1,7 @@
 """Allows accessing Keras Service if available."""
 import json
 import logging
+import os
 import time
 import mss
 import cv2
@@ -18,7 +19,7 @@ class KerasService():
             self,
             ip_addr: str,
             port: int | str,
-            screenshot_path: str,
+            screenshot_path: str | os.PathLike,
             timeout: float = DEFAULT_TIMEOUT) -> None:
         self.ip_addr = ip_addr
         self.port = str(port)
@@ -31,7 +32,7 @@ class KerasService():
             monitor_1 = sct.monitors[1]  # Identify the display to capture
             screen = np.array(sct.grab(monitor_1))
             screen = cv2.cvtColor(screen, cv2.COLOR_RGB2GRAY)
-            cv2.imwrite(self.screenshot_path, screen)
+            cv2.imwrite(str(self.screenshot_path), screen)
 
     def _query_service(self, word: str, report_file: any) -> any:
         try:

shadowofthetombraider/README.md

@@ -0,0 +1,27 @@
+# Shadow of the Tomb Raider
+
+This script navigates through the game menus to the built in benchmark and runs it with the current settings. It then waits for a results screen.
+
+## Prerequisites
+
+- Python 3.10+
+- Shadow of the Tomb Raider installed via Steam
+- Keras OCR service
+
+## Options
+
+- `kerasHost`: string representing the IP address of the Keras service. e.x. `0.0.0.0` 
+- `kerasPort`: string representing the port of the Keras service. e.x. `8080`
+
+## Output
+
+report.json
+- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080`
+- `start_time`: number representing a timestamp of the test's start time in milliseconds
+- `end_time`: number representing a timestamp of the test's end time in milliseconds
+
+## Common Issues
+1. "Steam cannot sync with cloud" 
+    - A steam modal between test runs (when repeated) will come up.
+    - If you are monitoring the test, you can simply manually close the modal and the test should continue normally.
+    - The best solution is to disable cloud syncing for all steam games on the test bench.

shadowofthetombraider/manifest.yaml

@@ -0,0 +1,9 @@
+friendly_name: "Shadow of the Tomb Raider"
+executable: "shadowofthetombraider.py"
+process_name: "SOTTR.exe"
+output_dir: "run"
+options:
+  - name: kerasHost
+    type: input
+  - name: kerasPort
+    type: input

shadowofthetombraider/shadow_of_the_tomb_raider_utils.py

@@ -0,0 +1,54 @@
+"""Utility functions for Shadow of the Tomb Raider test script"""
+from argparse import ArgumentParser
+import os
+from pathlib import Path
+import winreg
+
+
+def get_reg(name) -> any:
+    """Get registry key value"""
+    reg_path = r'SOFTWARE\Eidos Montreal\Shadow of the Tomb Raider\Graphics'
+    try:
+        registry_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, reg_path, 0,
+                                      winreg.KEY_READ)
+        value, _ = winreg.QueryValueEx(registry_key, name)
+        winreg.CloseKey(registry_key)
+        return value
+    except Exception:
+        return None
+
+
+def get_resolution() -> tuple[int]:
+    """Get resolution from registry"""
+    width = get_reg("FullscreenWidth")
+    height = get_reg("FullscreenHeight")
+    return (height, width)
+
+
+def get_args() -> any:
+    """Returns command line arg values"""
+    parser = ArgumentParser()
+    parser.add_argument("--kerasHost", dest="keras_host",
+                        help="Host for Keras OCR service", required=True)
+    parser.add_argument("--kerasPort", dest="keras_port",
+                        help="Port for Keras OCR service", required=True)
+    return parser.parse_args()
+
+
+def get_latest_file_report(directory: Path):
+    """
+    get latest benchmark report from SOTTR documents directory
+    """
+    # Get list of all items in the directory with full paths
+    entries = (os.path.join(directory, fn) for fn in os.listdir(directory))
+    # Filter out directories, keep only files
+    files = [
+        file for file in entries
+        if os.path.isfile(file) and not file.endswith('.log') and "frametimes" not in file
+    ]
+    if not files:
+        return None  # No files found
+    # Get the file with the latest modification time
+    latest_file = max(files, key=os.path.getmtime)
+    return latest_file
+    

shadowofthetombraider/shadowofthetombraider.py

@@ -0,0 +1,154 @@
+"""Shadow of the Tomb Raider test script"""
+import logging
+import os
+from pathlib import Path
+import time
+import pydirectinput as user
+import sys
+from shadow_of_the_tomb_raider_utils import get_latest_file_report, get_resolution, get_args
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+#pylint: disable=wrong-import-position
+from harness_utils.output import (
+    setup_log_directory,
+    write_report_json,
+    format_resolution,
+    seconds_to_milliseconds,
+    DEFAULT_LOGGING_FORMAT,
+    DEFAULT_DATE_FORMAT)
+from harness_utils.process import terminate_processes
+from harness_utils.keras_service import KerasService
+from harness_utils.steam import exec_steam_game
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+
+STEAM_GAME_ID = 750920
+PROCESS_NAME = "SOTTR.exe"
+SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = SCRIPT_DIR.joinpath("run")
+
+
+def setup_logging():
+    """default logging config"""
+    setup_log_directory(LOG_DIR)
+    logging.basicConfig(filename=f'{LOG_DIR}/harness.log',
+                        format=DEFAULT_LOGGING_FORMAT,
+                        datefmt=DEFAULT_DATE_FORMAT,
+                        level=logging.DEBUG)
+    console = logging.StreamHandler()
+    formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+    console.setFormatter(formatter)
+    logging.getLogger('').addHandler(console)
+
+
+def start_game():
+    """Launch the game with console enabled and FPS unlocked"""
+    return exec_steam_game(STEAM_GAME_ID, game_params=["-nolauncher"])
+
+
+def run_benchmark():
+    """Start game via Steam and enter fullscreen mode"""
+    setup_start_time = time.time()
+    start_game()
+
+    args = get_args()
+    keras_service = KerasService(args.keras_host, args.keras_port, LOG_DIR.joinpath("screenshot.jpg"))
+    am = ArtifactManager(LOG_DIR)
+
+    if keras_service.wait_for_word(word="options", timeout=30, interval=1) is None:
+        logging.info("Did not find the options menu. Did the game launch correctly?")
+        sys.exit(1)
+
+    user.press("up")
+    time.sleep(0.2)
+    user.press("up")
+    time.sleep(0.2)
+    user.press("up")
+    time.sleep(0.2)
+    user.press("enter")
+    time.sleep(1)
+
+    if keras_service.wait_for_word(word="graphics", timeout=30, interval=1) is None:
+        logging.info("Did not find the graphics menu. Did the menu get stuck?")
+        sys.exit(1)
+
+    user.press("down")
+    time.sleep(0.2)
+    user.press("down")
+    time.sleep(0.2)
+    user.press("down")
+    time.sleep(0.2)
+    user.press("enter")
+    time.sleep(1)
+
+    if keras_service.wait_for_word(word="benchmark", timeout=30, interval=1) is None:
+        logging.info("Did not find the benchmark option on the screen. Did the menu get stuck?")
+        sys.exit(1)
+
+    am.take_screenshot("display.png", ArtifactType.CONFIG_IMAGE, "picture of display settings")
+
+    # press up until we have DISPLAY hilighted so we can flip to the graphics tab
+    for _ in range(21):
+        user.press("up")
+        time.sleep(0.2)
+
+    user.press("right")
+    am.take_screenshot("graphics.png", ArtifactType.CONFIG_IMAGE, "picture of graphics settings")
+
+    user.press("r")
+    elapsed_setup_time = round(time.time() - setup_start_time, 2)
+    logging.info("Setup took %f seconds", elapsed_setup_time)
+
+    time.sleep(2)
+    test_start_time = time.time()
+
+    # Wait for benchmark to complete
+    time.sleep(180)
+
+    test_end_time = time.time()
+
+    result = keras_service.wait_for_word(word="tomb", timeout=10, interval=0.1)
+    if result is None:
+        logging.error("Unable to find the loading screen. Using default end time value.")
+    else:
+        test_end_time = time.time()
+
+    if keras_service.wait_for_word(word="results", timeout=20, interval=1) is None:
+        logging.error("Results screen after running benchmark not found, exiting.")
+        sys.exit(1)
+
+    logging.info("Run completed. Closing game.")
+
+    time.sleep(2)
+
+    elapsed_test_time = round((test_end_time - test_start_time), 2)
+    logging.info("Benchmark took %f seconds", elapsed_test_time)
+    am.take_screenshot("results.png", ArtifactType.RESULTS_IMAGE, "benchmark results")
+
+    username = os.getlogin()
+    game_document_dir = Path(f"C:\\Users\\{username}\\Documents\\Shadow of the Tomb Raider")
+    game_log = game_document_dir.joinpath("Shadow of the Tomb Raider.log")
+    am.copy_file(Path(game_log), ArtifactType.RESULTS_TEXT, "game log")
+    am.copy_file(get_latest_file_report(game_document_dir), ArtifactType.RESULTS_TEXT, "benchmark result")
+
+    terminate_processes(PROCESS_NAME)
+    height, width = get_resolution()
+    report = {
+        "resolution": format_resolution(width, height),
+        "start_time": seconds_to_milliseconds(test_start_time),
+        "end_time": seconds_to_milliseconds(test_end_time)
+    }
+
+    am.create_manifest()
+    write_report_json(LOG_DIR, "report.json", report)
+
+
+if __name__ == "__main__":
+    try:
+        setup_logging()
+        run_benchmark()
+    except Exception as ex:
+        logging.error("Something went wrong running the benchmark!")
+        logging.exception(ex)
+        terminate_processes(PROCESS_NAME)
+        sys.exit(1)