From fc0ab8831c8f37d3f4e7d160ebcb1e0ae1dd5fce Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 11:56:19 -0800
Subject: [PATCH 01/25] Horizon Zero Dawn Harness pre-alpha

---
 horizonzdr/README.md     |  21 ++++
 horizonzdr/hzdr.py       | 207 +++++++++++++++++++++++++++++++++++++++
 horizonzdr/hzdr_utils.py |  93 ++++++++++++++++++
 horizonzdr/manifest.yaml |   9 ++
 4 files changed, 330 insertions(+)
 create mode 100644 horizonzdr/README.md
 create mode 100644 horizonzdr/hzdr.py
 create mode 100644 horizonzdr/hzdr_utils.py
 create mode 100644 horizonzdr/manifest.yaml

diff --git a/horizonzdr/README.md b/horizonzdr/README.md
new file mode 100644
index 0000000..226213f
--- /dev/null
+++ b/horizonzdr/README.md
@@ -0,0 +1,21 @@
+# Horizon Zero Dawn Remastered
+
+Navigates menus to the in-game benchmark then runs it.
+
+## Prerequisites
+
+- Python 3.10+
+- Horizon Zero Dawn Remastered installed
+- Keras OCR service
+
+## Options
+
+- `kerasHost`: string representing the IP address of the Keras service. e.x. `0.0.0.0` 
+- `kerasPort`: string representing the port of the Keras service. e.x. `8080`
+
+## Output
+
+report.json
+- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080`
+- `start_time`: number representing a timestamp of the test's start time in milliseconds
+- `end_time`: number representing a timestamp of the test's end time in milliseconds
\ No newline at end of file
diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
new file mode 100644
index 0000000..e4a1516
--- /dev/null
+++ b/horizonzdr/hzdr.py
@@ -0,0 +1,207 @@
+"""Returnal test script"""
+import os
+import logging
+import sys
+import time
+import pydirectinput as user
+
+from hzdr_utils import get_resolution, get_args
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+from harness_utils.keras_service import KerasService
+from harness_utils.output import (
+    format_resolution,
+    seconds_to_milliseconds,
+    setup_log_directory,
+    write_report_json,
+    DEFAULT_LOGGING_FORMAT,
+    DEFAULT_DATE_FORMAT,
+)
+from harness_utils.misc import remove_files, press_n_times
+from harness_utils.process import terminate_processes
+from harness_utils.steam import (
+  exec_steam_run_command,
+  get_steamapps_common_path,
+  get_build_id
+)
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+
+STEAM_GAME_ID = 2561580
+SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
+PROCESS_NAME = "HorizonZeroDawnRemastered"
+LOCAL_USER_SETTINGS = os.path.join(
+  os.getenv('LOCALAPPDATA'), "Returnal", "Steam",
+  "Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
+  )
+VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
+
+user.FAILSAFE = False
+
+intro_videos = [
+    os.path.join(VIDEO_PATH, "weaseltron_logo.bk2"),
+    os.path.join(VIDEO_PATH, "sony_studios_reel.bk2"),
+    os.path.join(VIDEO_PATH, "nixxes_logo.bk2"),
+    os.path.join(VIDEO_PATH, "Logo.bk2"),
+    os.path.join(VIDEO_PATH, "guerilla_logo.bk2")
+]
+
+def navigate_options_menu() -> None:
+    """Simulate inputs to navigate to options menu"""
+    logging.info("Navigating to options menu")
+    user.press("esc")
+    time.sleep(0.2)
+    user.press("enter")
+    time.sleep(0.2)
+    user.press("q")
+    time.sleep(0.2)
+    user.keyDown("tab")
+    time.sleep(5)
+    user.keyUp("tab")
+
+
+def run_benchmark() -> tuple[float]:
+    """Run the benchmark"""
+    logging.info("Removing intro videos")
+    remove_files(intro_videos)
+
+    logging.info("Starting game")
+    exec_steam_run_command(STEAM_GAME_ID)
+    setup_start_time = time.time()
+    am = ArtifactManager(LOG_DIRECTORY)
+
+    time.sleep(10)
+
+    # Make sure the game started correctly
+    result = kerasService.look_for_word("locate", 10, 5)
+    if not result:
+        logging.info("Could not find prompt to open menu!")
+        sys.exit(1)
+
+    # Navigate to display menu
+    user.press("esc")
+    time.sleep(1)
+    user.press("enter")
+    time.sleep(1)
+    user.press("q")
+    time.sleep(1)
+    user.press("q")
+    time.sleep(1)
+
+    # Verify that we have navigated to the video settings menu and take a screenshot
+    if kerasService.wait_for_word(word="aspect", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings")
+
+    # Navigate to graphics menu
+    user.press("e")
+    time.sleep(1)
+
+    if kerasService.wait_for_word(word="vsync", timeout=30, interval=1) is None:
+        logging.info("Did not find the graphics settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "first picture of graphics settings")
+
+    # We check for a keyword that indicates DLSS is active because this changes how we navigate the menu
+    if kerasService.wait_for_word(word="sharpness", timeout=10, interval=1) is None:
+        logging.info("No DLSS Settings Detected")
+        # Scroll down graphics menu
+        press_n_times("down", 15, 0.2)
+    else:
+        logging.info("DLSS Settings Detected")
+        # Scroll down graphics menu
+        press_n_times("down", 17, 0.2)
+
+    if kerasService.wait_for_word(word="volumetric", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'volumetric'. Did the the menu scroll correctly?")
+        sys.exit(1)
+    am.take_screenshot("graphics_2.png", ArtifactType.CONFIG_IMAGE, "second picture of graphics settings")
+
+    # Scroll down graphics menu
+    press_n_times("down", 15, 0.2)
+
+    if kerasService.wait_for_word(word="hdr", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'hdr'. Did the the menu scroll correctly?")
+        sys.exit(1)
+    am.take_screenshot("graphics_3.png", ArtifactType.CONFIG_IMAGE, "third picture of graphics settings")
+
+    # Launch the benchmark
+    user.keyDown("tab")
+    time.sleep(5)
+    user.keyUp("tab")
+
+    setup_end_time = time.time()
+    elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
+    logging.info("Setup took %s seconds", elapsed_setup_time)
+
+    result = kerasService.wait_for_word("performance", interval=0.2, timeout=30)
+    if not result:
+        logging.info(
+            "Performance graph was not found! Could not mark the start time.")
+        sys.exit(1)
+
+    test_start_time = time.time()
+
+    # Wait for benchmark to complete
+    time.sleep(112)
+
+    # Wait for results screen to display info
+    result = kerasService.wait_for_word("lost", interval=0.1, timeout=11)
+    if not result:
+        logging.info(
+            "Didn't see signal lost. Could not mark the proper end time!")
+
+    test_end_time = round(time.time() - 2)
+
+    result = kerasService.wait_for_word("benchmark", interval=0.5, timeout=15)
+    if not result:
+        logging.info(
+            "Results screen was not found! Did harness not wait long enough? Or test was too long?")
+        sys.exit(1)
+
+    # Give results screen time to fill out, then save screenshot and config file
+    time.sleep(2)
+    am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of benchmark result")
+    am.copy_file(LOCAL_USER_SETTINGS, ArtifactType.CONFIG_TEXT, "config file")
+
+    elapsed_test_time = round((test_end_time - test_start_time), 2)
+    logging.info("Benchmark took %s seconds", elapsed_test_time)
+
+    terminate_processes(PROCESS_NAME)
+    am.create_manifest()
+
+    return test_start_time, test_end_time
+
+
+setup_log_directory(LOG_DIRECTORY)
+
+logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log',
+                    format=DEFAULT_LOGGING_FORMAT,
+                    datefmt=DEFAULT_DATE_FORMAT,
+                    level=logging.DEBUG)
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger('').addHandler(console)
+
+args = get_args()
+kerasService = KerasService(args.keras_host, args.keras_port)
+
+try:
+    start_time, end_time = run_benchmark()
+    height, width = get_resolution(LOCAL_USER_SETTINGS)
+    report = {
+        "resolution": format_resolution(width, height),
+        "start_time": seconds_to_milliseconds(start_time),
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
+    }
+
+    write_report_json(LOG_DIRECTORY, "report.json", report)
+except Exception as e:
+    logging.error("Something went wrong running the benchmark!")
+    logging.exception(e)
+    terminate_processes(PROCESS_NAME)
+    sys.exit(1)
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
new file mode 100644
index 0000000..942958b
--- /dev/null
+++ b/horizonzdr/hzdr_utils.py
@@ -0,0 +1,93 @@
+"""Utility functions supporting Returnal test script."""
+from argparse import ArgumentParser
+import re
+import winreg
+
+
+input_file = 'input.reg'
+config_file = 'config_registry.txt'
+hive = winreg.KEY_CURRENT_USER
+subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
+
+def export_registry_key(hive, subkey, input_file):
+    try:
+        with winreg.OpenKey(hive,subkey) as reg_key:
+            with open(input_file, 'w') as reg_file:
+                reg_file.write(f"Windows Registry Editor Version 5.00\n\n")
+                reg_file.write(f"[{subkey}]\n")
+                try:
+                    index = 0
+                    while True:
+                        value_name, value_data, value_type = winreg.EnumValue(reg_key, index)
+                        if value_type == winreg.REG_DWORD:
+                            value_data = f"dword:{value_data:08x}"
+                        elif value_type == winreg.REG_SZ:
+                            value_data = f'"{value_data}"'
+                        elif value_type == winreg.REG_QWORD:
+                            value_data = f"qword:{value_data:0x16x}"
+                        else:
+                            value_data = f'"{value_data}"'
+                        reg_file.write(f'"{value_name}"={value_data}\n')
+                        index += 1
+                except OSError:
+                    pass
+    except WindowsError as e:
+        print(f"Failed to open the registry key: {e}")
+
+def convert_dword_to_decimal(dword_hex):
+    return int(dword_hex, 16)
+
+def process_registry_file(input_file, config_file):
+    export_registry_key()
+    with open(input_file, 'r') as file:
+        lines = file.readlines()
+    
+    modified_lines = []
+
+    dword_pattern = re.compile(r'^(\"[^\"]+\")=dword:([0-9a-fA-F]+)', re.IGNORECASE)
+
+    for line in lines:
+        match = dword_pattern.search(line)
+        if match:
+            key = match.group(1)
+            hex_value = match.group(2)
+            decimal_value = convert_dword_to_decimal(hex_value)
+            modified_line = f'{key}={decimal_value}\n'
+            modified_lines.append(modified_line)
+        else:
+            modified_lines.append(line)
+    with open(config_file, 'w') as file:
+        file.writelines(modified_lines)
+
+
+
+def get_resolution(config_file: str) -> tuple[int]:
+    """Retrieve the resolution from local configuration files."""
+    width_pattern = re.compile(r"ResolutionSizeX=(\d+)")
+    height_pattern = re.compile(r"ResolutionSizeY=(\d+)")
+    width = 0
+    height = 0
+
+    with open(config_file, encoding="utf-8") as file:
+        lines = file.readlines()
+        for line in lines:
+            width_match = width_pattern.match(line)
+            height_match = height_pattern.match(line)
+
+            if width_match:
+                width = width_match.group(1)
+            if height_match:
+                height = height_match.group(1)
+
+    return (height, width)
+
+def get_args() -> any:
+    """Get command line arguments"""
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--kerasHost", dest="keras_host", help="Host for Keras OCR service", required=True)
+    parser.add_argument(
+        "--kerasPort", dest="keras_port", help="Port for Keras OCR service", required=True)
+    return parser.parse_args()
+
+
diff --git a/horizonzdr/manifest.yaml b/horizonzdr/manifest.yaml
new file mode 100644
index 0000000..c54a27c
--- /dev/null
+++ b/horizonzdr/manifest.yaml
@@ -0,0 +1,9 @@
+friendly_name: "Horizon Zero Dawn Remastered"
+executable: "hzdr.py"
+process_name: "HorizonZeroDawnRemastered.exe"
+output_dir: "run"
+options:
+  - name: kerasHost
+    type: input
+  - name: kerasPort
+    type: input
\ No newline at end of file

From 197e429cdc57c6237d2b5667ed53bf5f2c297d08 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:03:34 -0800
Subject: [PATCH 02/25] Building functionality of processing a registry file

---
 horizonzdr/hzdr.py       | 9 ++++++++-
 horizonzdr/hzdr_utils.py | 4 ++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index e4a1516..bc29514 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -4,8 +4,9 @@ import logging
 import sys
 import time
 import pydirectinput as user
+import winreg
 
-from hzdr_utils import get_resolution, get_args
+from hzdr_utils import get_resolution, get_args, process_registry_file
 
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
 
@@ -36,6 +37,10 @@ LOCAL_USER_SETTINGS = os.path.join(
   "Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
   )
 VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
+input_file = 'input.reg'
+config_file = 'config_registry.txt'
+hive = winreg.KEY_CURRENT_USER
+subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
 
 user.FAILSAFE = False
 
@@ -73,6 +78,8 @@ def run_benchmark() -> tuple[float]:
 
     time.sleep(10)
 
+    process_registry_file()
+    sys.exit(1)
     # Make sure the game started correctly
     result = kerasService.look_for_word("locate", 10, 5)
     if not result:
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index 942958b..70dbe53 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -63,8 +63,8 @@ def process_registry_file(input_file, config_file):
 
 def get_resolution(config_file: str) -> tuple[int]:
     """Retrieve the resolution from local configuration files."""
-    width_pattern = re.compile(r"ResolutionSizeX=(\d+)")
-    height_pattern = re.compile(r"ResolutionSizeY=(\d+)")
+    width_pattern = re.compile(r"\"FullscreenWidth\"=(\d+)")
+    height_pattern = re.compile(r"\"FullscreenHeight\"=(\d+)")
     width = 0
     height = 0
 

From 376803e9e4546d59764843d8ee2e996609c7ee1a Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:13:54 -0800
Subject: [PATCH 03/25] More logic to the harness

---
 horizonzdr/hzdr.py | 102 ++++++++++++++++++---------------------------
 1 file changed, 41 insertions(+), 61 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index bc29514..16ffd3f 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -52,19 +52,6 @@ intro_videos = [
     os.path.join(VIDEO_PATH, "guerilla_logo.bk2")
 ]
 
-def navigate_options_menu() -> None:
-    """Simulate inputs to navigate to options menu"""
-    logging.info("Navigating to options menu")
-    user.press("esc")
-    time.sleep(0.2)
-    user.press("enter")
-    time.sleep(0.2)
-    user.press("q")
-    time.sleep(0.2)
-    user.keyDown("tab")
-    time.sleep(5)
-    user.keyUp("tab")
-
 
 def run_benchmark() -> tuple[float]:
     """Run the benchmark"""
@@ -81,97 +68,90 @@ def run_benchmark() -> tuple[float]:
     process_registry_file()
     sys.exit(1)
     # Make sure the game started correctly
-    result = kerasService.look_for_word("locate", 10, 5)
+    result = kerasService.look_for_word("remastered", 10, 5)
     if not result:
-        logging.info("Could not find prompt to open menu!")
+        logging.info("Could not find the main menu. Did the game load?")
         sys.exit(1)
 
     # Navigate to display menu
-    user.press("esc")
+    user.press("down")
+    time.sleep(1)
+    user.press("down")
     time.sleep(1)
     user.press("enter")
     time.sleep(1)
-    user.press("q")
-    time.sleep(1)
-    user.press("q")
-    time.sleep(1)
 
     # Verify that we have navigated to the video settings menu and take a screenshot
-    if kerasService.wait_for_word(word="aspect", timeout=30, interval=1) is None:
+    if kerasService.wait_for_word(word="language", timeout=30, interval=1) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
         sys.exit(1)
-    am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings")
+
+    user.press("e")
+    time.sleep(1)
+
+    if kerasService.wait_for_word(word="monitor", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "1st picture of display settings")
+
+    user.press("up")
+    time.sleep(1)
+
+    if kerasService.wait_for_word(word="upscale", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("display2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of display settings")
 
     # Navigate to graphics menu
     user.press("e")
     time.sleep(1)
 
-    if kerasService.wait_for_word(word="vsync", timeout=30, interval=1) is None:
-        logging.info("Did not find the graphics settings menu. Did the menu get stuck?")
+    if kerasService.wait_for_word(word="preset", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
         sys.exit(1)
-    am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "first picture of graphics settings")
+    am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "1st picture of graphics settings")
 
-    # We check for a keyword that indicates DLSS is active because this changes how we navigate the menu
-    if kerasService.wait_for_word(word="sharpness", timeout=10, interval=1) is None:
-        logging.info("No DLSS Settings Detected")
-        # Scroll down graphics menu
-        press_n_times("down", 15, 0.2)
-    else:
-        logging.info("DLSS Settings Detected")
-        # Scroll down graphics menu
-        press_n_times("down", 17, 0.2)
+    user.press("up")
+    time.sleep(1)
 
-    if kerasService.wait_for_word(word="volumetric", timeout=30, interval=1) is None:
-        logging.info("Did not find the keyword 'volumetric'. Did the the menu scroll correctly?")
+    if kerasService.wait_for_word(word="sharpness", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
         sys.exit(1)
-    am.take_screenshot("graphics_2.png", ArtifactType.CONFIG_IMAGE, "second picture of graphics settings")
-
-    # Scroll down graphics menu
-    press_n_times("down", 15, 0.2)
-
-    if kerasService.wait_for_word(word="hdr", timeout=30, interval=1) is None:
-        logging.info("Did not find the keyword 'hdr'. Did the the menu scroll correctly?")
-        sys.exit(1)
-    am.take_screenshot("graphics_3.png", ArtifactType.CONFIG_IMAGE, "third picture of graphics settings")
-
+    am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of graphics settings")
+    
     # Launch the benchmark
-    user.keyDown("tab")
-    time.sleep(5)
-    user.keyUp("tab")
+    user.press("tab")
+    time.sleep(0.5)
+    user.press("enter")
 
     setup_end_time = time.time()
     elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
     logging.info("Setup took %s seconds", elapsed_setup_time)
 
-    result = kerasService.wait_for_word("performance", interval=0.2, timeout=30)
+    result = kerasService.wait_for_word("continue", interval=0.2, timeout=30)
     if not result:
         logging.info(
             "Performance graph was not found! Could not mark the start time.")
         sys.exit(1)
 
+    user.press("enter")
+
     test_start_time = time.time()
 
     # Wait for benchmark to complete
-    time.sleep(112)
+    time.sleep(180)
 
     # Wait for results screen to display info
-    result = kerasService.wait_for_word("lost", interval=0.1, timeout=11)
+    result = kerasService.wait_for_word("results", interval=0.1, timeout=11)
     if not result:
         logging.info(
             "Didn't see signal lost. Could not mark the proper end time!")
 
-    test_end_time = round(time.time() - 2)
-
-    result = kerasService.wait_for_word("benchmark", interval=0.5, timeout=15)
-    if not result:
-        logging.info(
-            "Results screen was not found! Did harness not wait long enough? Or test was too long?")
-        sys.exit(1)
-
+    test_end_time = round(time.time())
     # Give results screen time to fill out, then save screenshot and config file
     time.sleep(2)
     am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of benchmark result")
-    am.copy_file(LOCAL_USER_SETTINGS, ArtifactType.CONFIG_TEXT, "config file")
+    am.copy_file(config_file, ArtifactType.CONFIG_TEXT, "config file")
 
     elapsed_test_time = round((test_end_time - test_start_time), 2)
     logging.info("Benchmark took %s seconds", elapsed_test_time)

From d01841db120a89e39e30c4a29e190f2692b0d16b Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:21:08 -0800
Subject: [PATCH 04/25] Update hzdr_utils.py

---
 horizonzdr/hzdr_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index 70dbe53..1b38b4e 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -6,7 +6,7 @@ import winreg
 
 input_file = 'input.reg'
 config_file = 'config_registry.txt'
-hive = winreg.KEY_CURRENT_USER
+hive = winreg.HKEY_CURRENT_USER
 subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
 
 def export_registry_key(hive, subkey, input_file):

From b6fb8b2828658375d97238e23cd08a1a28d379f7 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:23:11 -0800
Subject: [PATCH 05/25] Update hzdr.py

---
 horizonzdr/hzdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 16ffd3f..2821b46 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -39,7 +39,7 @@ LOCAL_USER_SETTINGS = os.path.join(
 VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
 input_file = 'input.reg'
 config_file = 'config_registry.txt'
-hive = winreg.KEY_CURRENT_USER
+hive = winreg.HKEY_CURRENT_USER
 subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
 
 user.FAILSAFE = False

From 538adc967442f02500ac0eb23a8b71749f773d61 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:26:31 -0800
Subject: [PATCH 06/25] added location for files to open

---
 horizonzdr/hzdr.py       | 4 ++--
 horizonzdr/hzdr_utils.py | 5 -----
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 2821b46..4e3f5ab 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -37,8 +37,8 @@ LOCAL_USER_SETTINGS = os.path.join(
   "Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
   )
 VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
-input_file = 'input.reg'
-config_file = 'config_registry.txt'
+input_file = os.path.join(SCRIPT_DIRECTORY, 'input.reg')
+config_file = os.path.join(SCRIPT_DIRECTORY, 'config_registry.txt')
 hive = winreg.HKEY_CURRENT_USER
 subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
 
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index 1b38b4e..71975cf 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -4,11 +4,6 @@ import re
 import winreg
 
 
-input_file = 'input.reg'
-config_file = 'config_registry.txt'
-hive = winreg.HKEY_CURRENT_USER
-subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
-
 def export_registry_key(hive, subkey, input_file):
     try:
         with winreg.OpenKey(hive,subkey) as reg_key:

From 0cff005928b01c7555c8c8c928b776c9ac9e8501 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:27:14 -0800
Subject: [PATCH 07/25] Update hzdr.py

---
 horizonzdr/hzdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 4e3f5ab..7c40be8 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -65,7 +65,7 @@ def run_benchmark() -> tuple[float]:
 
     time.sleep(10)
 
-    process_registry_file()
+    process_registry_file(input_file, config_file)
     sys.exit(1)
     # Make sure the game started correctly
     result = kerasService.look_for_word("remastered", 10, 5)

From 5163722d85e6f038fdd2fd8f843a5f0b3db34430 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:43:15 -0800
Subject: [PATCH 08/25] Reg file fix and naming changes

Edited the utils so that it creates a blank reg file if needed and then properly exports the data.
---
 horizonzdr/hzdr.py       | 6 +++---
 horizonzdr/hzdr_utils.py | 6 +++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 7c40be8..be312ed 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -37,10 +37,10 @@ LOCAL_USER_SETTINGS = os.path.join(
   "Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
   )
 VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
-input_file = os.path.join(SCRIPT_DIRECTORY, 'input.reg')
-config_file = os.path.join(SCRIPT_DIRECTORY, 'config_registry.txt')
+input_file = os.path.join(SCRIPT_DIRECTORY, 'graphics.reg')
+config_file = os.path.join(SCRIPT_DIRECTORY, 'graphics_config.txt')
 hive = winreg.HKEY_CURRENT_USER
-subkey = r"SOFTWARE\Guerilla Games\Horizon Zero Dawn Remastered\Graphics"
+subkey = r"SOFTWARE\\Guerilla Games\\Horizon Zero Dawn Remastered\\Graphics"
 
 user.FAILSAFE = False
 
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index 71975cf..eca9620 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -2,10 +2,14 @@
 from argparse import ArgumentParser
 import re
 import winreg
+import os
 
 
 def export_registry_key(hive, subkey, input_file):
     try:
+        if not os.path.exists(input_file):
+            with open(input_file, 'w') as file:
+                file.write("")
         with winreg.OpenKey(hive,subkey) as reg_key:
             with open(input_file, 'w') as reg_file:
                 reg_file.write(f"Windows Registry Editor Version 5.00\n\n")
@@ -33,7 +37,7 @@ def convert_dword_to_decimal(dword_hex):
     return int(dword_hex, 16)
 
 def process_registry_file(input_file, config_file):
-    export_registry_key()
+    export_registry_key(hive, subkey, input_file)
     with open(input_file, 'r') as file:
         lines = file.readlines()
     

From 21cf3be4b0a8cc99460932b3754f67d7ed1ea1d4 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 12:45:12 -0800
Subject: [PATCH 09/25] Fixed calls for hive and subkey

---
 horizonzdr/hzdr.py       | 2 +-
 horizonzdr/hzdr_utils.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index be312ed..ae3de4d 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -65,7 +65,7 @@ def run_benchmark() -> tuple[float]:
 
     time.sleep(10)
 
-    process_registry_file(input_file, config_file)
+    process_registry_file(hive, subkey, input_file, config_file)
     sys.exit(1)
     # Make sure the game started correctly
     result = kerasService.look_for_word("remastered", 10, 5)
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index eca9620..6241041 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -36,7 +36,7 @@ def export_registry_key(hive, subkey, input_file):
 def convert_dword_to_decimal(dword_hex):
     return int(dword_hex, 16)
 
-def process_registry_file(input_file, config_file):
+def process_registry_file(hive, subkey, input_file, config_file):
     export_registry_key(hive, subkey, input_file)
     with open(input_file, 'r') as file:
         lines = file.readlines()

From 5c7d2c7b82889d63ed40731c36178e8a1d392e46 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 13:26:13 -0800
Subject: [PATCH 10/25] Update hzdr.py

---
 horizonzdr/hzdr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index ae3de4d..ed50639 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -40,7 +40,7 @@ VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remast
 input_file = os.path.join(SCRIPT_DIRECTORY, 'graphics.reg')
 config_file = os.path.join(SCRIPT_DIRECTORY, 'graphics_config.txt')
 hive = winreg.HKEY_CURRENT_USER
-subkey = r"SOFTWARE\\Guerilla Games\\Horizon Zero Dawn Remastered\\Graphics"
+subkey = r"SOFTWARE\\Guerrilla Games\\Horizon Zero Dawn Remastered\\Graphics"
 
 user.FAILSAFE = False
 

From d2d309229689ba73ed5d7d734e230f9462334d04 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 13:29:35 -0800
Subject: [PATCH 11/25] Trying a full send now

---
 horizonzdr/hzdr.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index ed50639..ae82e60 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -32,10 +32,6 @@ STEAM_GAME_ID = 2561580
 SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
 PROCESS_NAME = "HorizonZeroDawnRemastered"
-LOCAL_USER_SETTINGS = os.path.join(
-  os.getenv('LOCALAPPDATA'), "Returnal", "Steam",
-  "Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
-  )
 VIDEO_PATH = os.path.join(get_steamapps_common_path(), "Horizon Zero Dawn Remastered", "Movies", "Mono")
 input_file = os.path.join(SCRIPT_DIRECTORY, 'graphics.reg')
 config_file = os.path.join(SCRIPT_DIRECTORY, 'graphics_config.txt')
@@ -65,7 +61,7 @@ def run_benchmark() -> tuple[float]:
 
     time.sleep(10)
 
-    process_registry_file(hive, subkey, input_file, config_file)
+    
     sys.exit(1)
     # Make sure the game started correctly
     result = kerasService.look_for_word("remastered", 10, 5)
@@ -151,6 +147,7 @@ def run_benchmark() -> tuple[float]:
     # Give results screen time to fill out, then save screenshot and config file
     time.sleep(2)
     am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of benchmark result")
+    process_registry_file(hive, subkey, input_file, config_file)
     am.copy_file(config_file, ArtifactType.CONFIG_TEXT, "config file")
 
     elapsed_test_time = round((test_end_time - test_start_time), 2)
@@ -178,7 +175,7 @@ kerasService = KerasService(args.keras_host, args.keras_port)
 
 try:
     start_time, end_time = run_benchmark()
-    height, width = get_resolution(LOCAL_USER_SETTINGS)
+    height, width = get_resolution(config_file)
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),

From b052849229c6c18ce04e6edbdc71d5b36f88606a Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 13:47:38 -0800
Subject: [PATCH 12/25] Update hzdr.py

---
 horizonzdr/hzdr.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index ae82e60..d543c87 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -60,9 +60,7 @@ def run_benchmark() -> tuple[float]:
     am = ArtifactManager(LOG_DIRECTORY)
 
     time.sleep(10)
-
     
-    sys.exit(1)
     # Make sure the game started correctly
     result = kerasService.look_for_word("remastered", 10, 5)
     if not result:

From 8e85020433166b42c99a93ff18ba818e107855d1 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 14:12:13 -0800
Subject: [PATCH 13/25] Speeding up the harness in general

---
 horizonzdr/hzdr.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index d543c87..4cf5315 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -60,7 +60,7 @@ def run_benchmark() -> tuple[float]:
     am = ArtifactManager(LOG_DIRECTORY)
 
     time.sleep(10)
-    
+
     # Make sure the game started correctly
     result = kerasService.look_for_word("remastered", 10, 5)
     if not result:
@@ -69,11 +69,11 @@ def run_benchmark() -> tuple[float]:
 
     # Navigate to display menu
     user.press("down")
-    time.sleep(1)
+    time.sleep(0.5)
     user.press("down")
-    time.sleep(1)
+    time.sleep(0.5)
     user.press("enter")
-    time.sleep(1)
+    time.sleep(0.5)
 
     # Verify that we have navigated to the video settings menu and take a screenshot
     if kerasService.wait_for_word(word="language", timeout=30, interval=1) is None:
@@ -81,7 +81,7 @@ def run_benchmark() -> tuple[float]:
         sys.exit(1)
 
     user.press("e")
-    time.sleep(1)
+    time.sleep(0.5)
 
     if kerasService.wait_for_word(word="monitor", timeout=30, interval=1) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
@@ -89,7 +89,7 @@ def run_benchmark() -> tuple[float]:
     am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "1st picture of display settings")
 
     user.press("up")
-    time.sleep(1)
+    time.sleep(0.5)
 
     if kerasService.wait_for_word(word="upscale", timeout=30, interval=1) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
@@ -98,7 +98,7 @@ def run_benchmark() -> tuple[float]:
 
     # Navigate to graphics menu
     user.press("e")
-    time.sleep(1)
+    time.sleep(0.5)
 
     if kerasService.wait_for_word(word="preset", timeout=30, interval=1) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
@@ -106,7 +106,7 @@ def run_benchmark() -> tuple[float]:
     am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "1st picture of graphics settings")
 
     user.press("up")
-    time.sleep(1)
+    time.sleep(0.5)
 
     if kerasService.wait_for_word(word="sharpness", timeout=30, interval=1) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
@@ -122,7 +122,7 @@ def run_benchmark() -> tuple[float]:
     elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
     logging.info("Setup took %s seconds", elapsed_setup_time)
 
-    result = kerasService.wait_for_word("continue", interval=0.2, timeout=30)
+    result = kerasService.wait_for_word("continue", interval=1, timeout=50)
     if not result:
         logging.info(
             "Performance graph was not found! Could not mark the start time.")

From 5c802a65a8d302967e5eb7e9fe4546e7a417f41f Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 15:55:42 -0800
Subject: [PATCH 14/25] Added Forza Motorsport

Added a forza motorsport harness & some minor text changes on horizon
---
 forzamotorsport/README.md        |  21 ++++
 forzamotorsport/forzams.py       | 174 +++++++++++++++++++++++++++++++
 forzamotorsport/forzams_utils.py |  35 +++++++
 forzamotorsport/manifest.yaml    |   9 ++
 horizonzdr/hzdr.py               |   2 +-
 horizonzdr/hzdr_utils.py         |   2 +-
 6 files changed, 241 insertions(+), 2 deletions(-)
 create mode 100644 forzamotorsport/README.md
 create mode 100644 forzamotorsport/forzams.py
 create mode 100644 forzamotorsport/forzams_utils.py
 create mode 100644 forzamotorsport/manifest.yaml

diff --git a/forzamotorsport/README.md b/forzamotorsport/README.md
new file mode 100644
index 0000000..5b640d1
--- /dev/null
+++ b/forzamotorsport/README.md
@@ -0,0 +1,21 @@
+# Forza Horizon 5
+
+This script runs RTSS with the provided profile config and navigates through in-game menus to the built in benchmark and runs it with the current settings.
+
+## Prerequisites
+
+- Python 3.10+
+- Forza Horizon 5 installed via Steam
+- Keras OCR service
+- RTSS installed - https://rivatuner.net/
+
+## Output
+
+report.json
+- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080`
+- `start_time`: number representing a timestamp of the test's start time in milliseconds
+- `end_time`: number representing a timestamp of the test's end time in milliseconds
+
+## Common Issues
+1. "Login to Microsoft" modal pops up
+    - This game will not let you pass into the menu if you are not signed into Xbox. If you run this game at least once before running you can login then, or pre-login before running the harness.
diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
new file mode 100644
index 0000000..bb3424c
--- /dev/null
+++ b/forzamotorsport/forzams.py
@@ -0,0 +1,174 @@
+"""Forza Motorsport test script"""
+import os
+import logging
+import sys
+import time
+import pydirectinput as user
+import winreg
+
+from hzdr_utils import get_resolution, get_args, process_registry_file
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+from harness_utils.keras_service import KerasService
+from harness_utils.output import (
+    format_resolution,
+    seconds_to_milliseconds,
+    setup_log_directory,
+    write_report_json,
+    DEFAULT_LOGGING_FORMAT,
+    DEFAULT_DATE_FORMAT,
+)
+from harness_utils.misc import press_n_times
+from harness_utils.process import terminate_processes
+from harness_utils.steam import (
+  exec_steam_run_command,
+  get_build_id
+)
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+
+STEAM_GAME_ID = 2440510
+SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
+PROCESS_NAME = "forza_steamworks_release_final"
+LOCAL_USER_SETTINGS = os.path.join(
+  os.getenv('LOCALAPPDATA'), "Microsoft.ForzaMotorsport", "User_SteamLocalStorageDirectory",
+  "ConnectedStorage", "ForzaUserConfigSelections", "UserConfigSelections"
+  )
+
+user.FAILSAFE = False
+
+
+def run_benchmark() -> tuple[float]:
+    """Run the benchmark"""
+    logging.info("Starting game")
+    exec_steam_run_command(STEAM_GAME_ID)
+    setup_start_time = time.time()
+    am = ArtifactManager(LOG_DIRECTORY)
+
+    time.sleep(20)
+
+    # Make sure the game started correctly
+    result = kerasService.look_for_word("settings", 10, 5)
+    if not result:
+        logging.info("Could not find the main menu. Did the game load?")
+        sys.exit(1)
+
+    # Navigate to display menu
+    user.press("f")
+    time.sleep(1)
+
+    if kerasService.wait_for_word(word="contrast", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+
+    user.press("]")
+    time.sleep(0.5)
+    user.press("]")
+    time.sleep(0.5)
+    user.press("]")
+    time.sleep(0.5)
+
+    # Verify that we have navigated to the video settings menu and take a screenshot
+    if kerasService.wait_for_word(word="resolution", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+
+    am.take_screenshot("display.png", ArtifactType.CONFIG_IMAGE, "picture of display settings")
+    user.press("]")
+    time.sleep(0.5)
+
+    if kerasService.wait_for_word(word="filtering", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "1st picture of graphics settings")
+
+    press_n_times("down",15,0.5)
+
+    if kerasService.wait_for_word(word="particle", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of graphics settings")
+
+    press_n_times("down",3,0.5)
+    user.press("up")
+    time.sleep(0.5)
+    user.press("down")
+    time.sleep(0.5)
+
+    if kerasService.wait_for_word(word="flare", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("graphics3.png", ArtifactType.CONFIG_IMAGE, "3rd picture of graphics settings")
+
+    # Navigate to graphics menu
+    user.press("[")
+    time.sleep(0.5)
+    user.press("enter")
+
+    setup_end_time = time.time()
+    elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
+    logging.info("Setup took %s seconds", elapsed_setup_time)
+
+    time.sleep(15)
+
+    if kerasService.wait_for_word(word="results", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("results.png", ArtifactType.CONFIG_IMAGE, "picture of results screen")
+
+    test_start_time = time.time()
+
+    # Wait for benchmark to complete
+    time.sleep(180)
+
+    # Wait for results screen to display info
+    result = kerasService.wait_for_word("results", interval=0.1, timeout=11)
+    if not result:
+        logging.info(
+            "Didn't see signal lost. Could not mark the proper end time!")
+
+    test_end_time = round(time.time())
+    # Give results screen time to fill out, then save screenshot and config file
+    time.sleep(2)
+    am.copy_file(LOCAL_USER_SETTINGS, ArtifactType.CONFIG_TEXT, "config file")
+
+    elapsed_test_time = round((test_end_time - test_start_time), 2)
+    logging.info("Benchmark took %s seconds", elapsed_test_time)
+
+    terminate_processes(PROCESS_NAME)
+    am.create_manifest()
+
+    return test_start_time, test_end_time
+
+
+setup_log_directory(LOG_DIRECTORY)
+
+logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log',
+                    format=DEFAULT_LOGGING_FORMAT,
+                    datefmt=DEFAULT_DATE_FORMAT,
+                    level=logging.DEBUG)
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger('').addHandler(console)
+
+args = get_args()
+kerasService = KerasService(args.keras_host, args.keras_port)
+
+try:
+    start_time, end_time = run_benchmark()
+    height, width = get_resolution(LOCAL_USER_SETTINGS)
+    report = {
+        "resolution": format_resolution(width, height),
+        "start_time": seconds_to_milliseconds(start_time),
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
+    }
+
+    write_report_json(LOG_DIRECTORY, "report.json", report)
+except Exception as e:
+    logging.error("Something went wrong running the benchmark!")
+    logging.exception(e)
+    terminate_processes(PROCESS_NAME)
+    sys.exit(1)
diff --git a/forzamotorsport/forzams_utils.py b/forzamotorsport/forzams_utils.py
new file mode 100644
index 0000000..2b6a8b7
--- /dev/null
+++ b/forzamotorsport/forzams_utils.py
@@ -0,0 +1,35 @@
+"""Utility functions supporting Forza Motorsport test script."""
+from argparse import ArgumentParser
+import re
+
+
+def get_resolution(config_file: str) -> tuple[int]:
+    """Retrieve the resolution from local configuration files."""
+    width_pattern = re.compile(r"\"FullscreenWidth\"=(\d+)")
+    height_pattern = re.compile(r"\"FullscreenHeight\"=(\d+)")
+    width = 0
+    height = 0
+
+    with open(config_file, encoding="utf-8") as file:
+        lines = file.readlines()
+        for line in lines:
+            width_match = width_pattern.match(line)
+            height_match = height_pattern.match(line)
+
+            if width_match:
+                width = width_match.group(1)
+            if height_match:
+                height = height_match.group(1)
+
+    return (height, width)
+
+def get_args() -> any:
+    """Get command line arguments"""
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--kerasHost", dest="keras_host", help="Host for Keras OCR service", required=True)
+    parser.add_argument(
+        "--kerasPort", dest="keras_port", help="Port for Keras OCR service", required=True)
+    return parser.parse_args()
+
+
diff --git a/forzamotorsport/manifest.yaml b/forzamotorsport/manifest.yaml
new file mode 100644
index 0000000..071889a
--- /dev/null
+++ b/forzamotorsport/manifest.yaml
@@ -0,0 +1,9 @@
+friendly_name: "Forza Motorsport"
+executable: "forzams.py"
+process_name: "forza_steamworks_release_final.exe"
+output_dir: "run"
+options:
+  - name: kerasHost
+    type: input
+  - name: kerasPort
+    type: input
diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 4cf5315..6ebca7a 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -1,4 +1,4 @@
-"""Returnal test script"""
+"""Horizon Zero Dawn Remastered test script"""
 import os
 import logging
 import sys
diff --git a/horizonzdr/hzdr_utils.py b/horizonzdr/hzdr_utils.py
index 6241041..f148b00 100644
--- a/horizonzdr/hzdr_utils.py
+++ b/horizonzdr/hzdr_utils.py
@@ -1,4 +1,4 @@
-"""Utility functions supporting Returnal test script."""
+"""Utility functions supporting Horizon Zero Dawn Remastered test script."""
 from argparse import ArgumentParser
 import re
 import winreg

From 0536bf9987cf86b3955f34fbbe1e24ed3ca402aa Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 15:58:37 -0800
Subject: [PATCH 15/25] Update forzams.py

---
 forzamotorsport/forzams.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index bb3424c..e01c028 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -6,7 +6,7 @@ import time
 import pydirectinput as user
 import winreg
 
-from hzdr_utils import get_resolution, get_args, process_registry_file
+from forzams_utils import get_resolution, get_args, process_registry_file
 
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
 

From 312ea3c2008b076dfbd2781f6502ebbbbf4c5ad0 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 16:01:15 -0800
Subject: [PATCH 16/25] Update forzams.py

---
 forzamotorsport/forzams.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index e01c028..470611b 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -6,7 +6,7 @@ import time
 import pydirectinput as user
 import winreg
 
-from forzams_utils import get_resolution, get_args, process_registry_file
+from forzams_utils import get_resolution, get_args
 
 sys.path.insert(1, os.path.join(sys.path[0], '..'))
 

From 0a2d3d217a5aafb1fc8ea749ddbc9a2f63bbbed6 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 16:32:28 -0800
Subject: [PATCH 17/25] Added resolution data and fixed some timing

---
 forzamotorsport/forzams.py       | 12 +++++-------
 forzamotorsport/forzams_utils.py | 21 +++++++--------------
 2 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index 470611b..3f28ac6 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -4,7 +4,6 @@ import logging
 import sys
 import time
 import pydirectinput as user
-import winreg
 
 from forzams_utils import get_resolution, get_args
 
@@ -12,7 +11,6 @@ sys.path.insert(1, os.path.join(sys.path[0], '..'))
 
 from harness_utils.keras_service import KerasService
 from harness_utils.output import (
-    format_resolution,
     seconds_to_milliseconds,
     setup_log_directory,
     write_report_json,
@@ -46,10 +44,10 @@ def run_benchmark() -> tuple[float]:
     setup_start_time = time.time()
     am = ArtifactManager(LOG_DIRECTORY)
 
-    time.sleep(20)
+    time.sleep(40)
 
     # Make sure the game started correctly
-    result = kerasService.look_for_word("settings", 10, 5)
+    result = kerasService.look_for_word("play", 10, 5)
     if not result:
         logging.info("Could not find the main menu. Did the game load?")
         sys.exit(1)
@@ -112,7 +110,7 @@ def run_benchmark() -> tuple[float]:
 
     time.sleep(15)
 
-    if kerasService.wait_for_word(word="results", timeout=30, interval=1) is None:
+    if kerasService.wait_for_word(word="results", timeout=60, interval=0.5) is None:
         logging.info("Did not find the video settings menu. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("results.png", ArtifactType.CONFIG_IMAGE, "picture of results screen")
@@ -158,9 +156,9 @@ kerasService = KerasService(args.keras_host, args.keras_port)
 
 try:
     start_time, end_time = run_benchmark()
-    height, width = get_resolution(LOCAL_USER_SETTINGS)
+    resolution = get_resolution(LOCAL_USER_SETTINGS)
     report = {
-        "resolution": format_resolution(width, height),
+        "resolution": f"{resolution}",
         "start_time": seconds_to_milliseconds(start_time),
         "end_time": seconds_to_milliseconds(end_time),
         "version": get_build_id(STEAM_GAME_ID)
diff --git a/forzamotorsport/forzams_utils.py b/forzamotorsport/forzams_utils.py
index 2b6a8b7..61dd071 100644
--- a/forzamotorsport/forzams_utils.py
+++ b/forzamotorsport/forzams_utils.py
@@ -4,24 +4,17 @@ import re
 
 
 def get_resolution(config_file: str) -> tuple[int]:
-    """Retrieve the resolution from local configuration files."""
-    width_pattern = re.compile(r"\"FullscreenWidth\"=(\d+)")
-    height_pattern = re.compile(r"\"FullscreenHeight\"=(\d+)")
-    width = 0
-    height = 0
-
+    """Get resolution from local game file"""
+    resolution_pattern = re.compile(r"<option id=\"IDS_ResolutionLabel\" value=\"(\d+x\d+)\"/>")
+    resolution = 0
     with open(config_file, encoding="utf-8") as file:
         lines = file.readlines()
         for line in lines:
-            width_match = width_pattern.match(line)
-            height_match = height_pattern.match(line)
+            resolution_match = resolution_pattern.search(line)
+            if resolution_match is not None:
+                resolution = resolution_match.group(1)
+    return resolution
 
-            if width_match:
-                width = width_match.group(1)
-            if height_match:
-                height = height_match.group(1)
-
-    return (height, width)
 
 def get_args() -> any:
     """Get command line arguments"""

From aeb3fbb2761c55a80a7dce02690fc03b6bc4f31a Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 16:49:42 -0800
Subject: [PATCH 18/25] Updated logging

---
 forzamotorsport/forzams.py       | 22 ++++++++++------------
 forzamotorsport/forzams_utils.py |  2 +-
 horizonzdr/hzdr.py               | 26 +++++++++++---------------
 3 files changed, 22 insertions(+), 28 deletions(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index 3f28ac6..a755755 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -47,8 +47,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(40)
 
     # Make sure the game started correctly
-    result = kerasService.look_for_word("play", 10, 5)
-    if not result:
+    if kerasService.wait_for_word(word="play", timeout=30, interval=1) is None:
         logging.info("Could not find the main menu. Did the game load?")
         sys.exit(1)
 
@@ -57,7 +56,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(1)
 
     if kerasService.wait_for_word(word="contrast", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the accessibility settings menu. Did the menu get stuck?")
         sys.exit(1)
 
     user.press("]")
@@ -69,7 +68,7 @@ def run_benchmark() -> tuple[float]:
 
     # Verify that we have navigated to the video settings menu and take a screenshot
     if kerasService.wait_for_word(word="resolution", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the display settings menu. Did the menu get stuck?")
         sys.exit(1)
 
     am.take_screenshot("display.png", ArtifactType.CONFIG_IMAGE, "picture of display settings")
@@ -77,14 +76,14 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="filtering", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the graphics settings menu. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "1st picture of graphics settings")
 
     press_n_times("down",15,0.5)
 
     if kerasService.wait_for_word(word="particle", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the particle effect settings. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of graphics settings")
 
@@ -95,7 +94,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="flare", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the lens flare settings. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("graphics3.png", ArtifactType.CONFIG_IMAGE, "3rd picture of graphics settings")
 
@@ -111,7 +110,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(15)
 
     if kerasService.wait_for_word(word="results", timeout=60, interval=0.5) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the results screen. Did the game load?")
         sys.exit(1)
     am.take_screenshot("results.png", ArtifactType.CONFIG_IMAGE, "picture of results screen")
 
@@ -121,10 +120,9 @@ def run_benchmark() -> tuple[float]:
     time.sleep(180)
 
     # Wait for results screen to display info
-    result = kerasService.wait_for_word("results", interval=0.1, timeout=11)
-    if not result:
-        logging.info(
-            "Didn't see signal lost. Could not mark the proper end time!")
+    if kerasService.wait_for_word(word="results", timeout=15, interval=0.5) is None:
+        logging.info("Did not find the results screen. Did the game crash during the run?")
+        sys.exit(1)
 
     test_end_time = round(time.time())
     # Give results screen time to fill out, then save screenshot and config file
diff --git a/forzamotorsport/forzams_utils.py b/forzamotorsport/forzams_utils.py
index 61dd071..27b8ea7 100644
--- a/forzamotorsport/forzams_utils.py
+++ b/forzamotorsport/forzams_utils.py
@@ -5,7 +5,7 @@ import re
 
 def get_resolution(config_file: str) -> tuple[int]:
     """Get resolution from local game file"""
-    resolution_pattern = re.compile(r"<option id=\"IDS_ResolutionLabel\" value=\"(\d+x\d+)\"/>")
+    resolution_pattern = re.compile(r"<option id=\"IDS_Resolution_Label\" value=\"(\d+x\d+)\"/>")
     resolution = 0
     with open(config_file, encoding="utf-8") as file:
         lines = file.readlines()
diff --git a/horizonzdr/hzdr.py b/horizonzdr/hzdr.py
index 6ebca7a..4c1a2bd 100644
--- a/horizonzdr/hzdr.py
+++ b/horizonzdr/hzdr.py
@@ -19,7 +19,7 @@ from harness_utils.output import (
     DEFAULT_LOGGING_FORMAT,
     DEFAULT_DATE_FORMAT,
 )
-from harness_utils.misc import remove_files, press_n_times
+from harness_utils.misc import remove_files
 from harness_utils.process import terminate_processes
 from harness_utils.steam import (
   exec_steam_run_command,
@@ -62,8 +62,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(10)
 
     # Make sure the game started correctly
-    result = kerasService.look_for_word("remastered", 10, 5)
-    if not result:
+    if kerasService.wait_for_word(word="remastered", timeout=30, interval=1) is None:
         logging.info("Could not find the main menu. Did the game load?")
         sys.exit(1)
 
@@ -84,7 +83,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="monitor", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the display settings menu. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "1st picture of display settings")
 
@@ -92,7 +91,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="upscale", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the upscale settings. Did the menu not scroll?")
         sys.exit(1)
     am.take_screenshot("display2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of display settings")
 
@@ -101,7 +100,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="preset", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the graphics settings menu. Did the menu get stuck?")
         sys.exit(1)
     am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "1st picture of graphics settings")
 
@@ -109,7 +108,7 @@ def run_benchmark() -> tuple[float]:
     time.sleep(0.5)
 
     if kerasService.wait_for_word(word="sharpness", timeout=30, interval=1) is None:
-        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        logging.info("Did not find the sharpness settings. Did the menu not scroll?")
         sys.exit(1)
     am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of graphics settings")
     
@@ -122,10 +121,8 @@ def run_benchmark() -> tuple[float]:
     elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
     logging.info("Setup took %s seconds", elapsed_setup_time)
 
-    result = kerasService.wait_for_word("continue", interval=1, timeout=50)
-    if not result:
-        logging.info(
-            "Performance graph was not found! Could not mark the start time.")
+    if kerasService.wait_for_word(word="continue", timeout=30, interval=1) is None:
+        logging.info("Did not find the continue button. Did the game not finish loading?")
         sys.exit(1)
 
     user.press("enter")
@@ -136,10 +133,9 @@ def run_benchmark() -> tuple[float]:
     time.sleep(180)
 
     # Wait for results screen to display info
-    result = kerasService.wait_for_word("results", interval=0.1, timeout=11)
-    if not result:
-        logging.info(
-            "Didn't see signal lost. Could not mark the proper end time!")
+    if kerasService.wait_for_word(word="results", timeout=20, interval=0.1) is None:
+        logging.info("Did not find the results screen. Did the game not finish the benchmark?")
+        sys.exit(1)
 
     test_end_time = round(time.time())
     # Give results screen time to fill out, then save screenshot and config file

From e5275aa00e8b88a56ab53a0ea08724fa361278ea Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 23 Dec 2024 17:25:20 -0800
Subject: [PATCH 19/25] Updated steam getbuildid to be more flexible

---
 forzamotorsport/forzams.py | 2 +-
 harness_utils/steam.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index a755755..a9a58e2 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -44,7 +44,7 @@ def run_benchmark() -> tuple[float]:
     setup_start_time = time.time()
     am = ArtifactManager(LOG_DIRECTORY)
 
-    time.sleep(40)
+    time.sleep(50)
 
     # Make sure the game started correctly
     if kerasService.wait_for_word(word="play", timeout=30, interval=1) is None:
diff --git a/harness_utils/steam.py b/harness_utils/steam.py
index ece087d..592abca 100644
--- a/harness_utils/steam.py
+++ b/harness_utils/steam.py
@@ -83,7 +83,7 @@ def exec_steam_game(game_id: int, steam_path=None, game_params=None) -> Popen:
 
 def get_build_id(game_id: int) -> str:
     """Gets the build ID of a game from the Steam installation directory"""
-    game_folder = Path(get_steamapps_common_path()) / "../" / f"appmanifest_{game_id}.acf"
+    game_folder = Path(get_app_install_location(game_id)) / "../" / "../" / f"appmanifest_{game_id}.acf"
     if not game_folder.exists():
         logging.warning("Game folder not found when looking for game version")
         return None

From 0da6620b7bd7d70f664160ca3dda0af90168257a Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Fri, 3 Jan 2025 12:01:37 -0800
Subject: [PATCH 20/25] Update README.md

---
 forzamotorsport/README.md | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/forzamotorsport/README.md b/forzamotorsport/README.md
index 5b640d1..014e615 100644
--- a/forzamotorsport/README.md
+++ b/forzamotorsport/README.md
@@ -1,13 +1,12 @@
-# Forza Horizon 5
+# Forza Motorsport
 
-This script runs RTSS with the provided profile config and navigates through in-game menus to the built in benchmark and runs it with the current settings.
+This script navigates through in-game menus to the built in benchmark and runs it with the current settings. The script runs for 3 minutes after the benchmark results come up to get a longer average as the game keeps running after the benchmark.
 
 ## Prerequisites
 
 - Python 3.10+
-- Forza Horizon 5 installed via Steam
+- Forza Motorsport via Steam
 - Keras OCR service
-- RTSS installed - https://rivatuner.net/
 
 ## Output
 

From c02367511542c8a343dabd0452e8b37dd56f5b44 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 6 Jan 2025 12:47:49 -0800
Subject: [PATCH 21/25] Added another harness of ashes of the singularity

---
 aots-e/README.md      |  23 +++++++
 aots-e/aotse.py       | 147 ++++++++++++++++++++++++++++++++++++++++++
 aots-e/aotse_utils.py | 110 +++++++++++++++++++++++++++++++
 aots-e/manifest.yaml  |  13 ++++
 4 files changed, 293 insertions(+)
 create mode 100644 aots-e/README.md
 create mode 100644 aots-e/aotse.py
 create mode 100644 aots-e/aotse_utils.py
 create mode 100644 aots-e/manifest.yaml

diff --git a/aots-e/README.md b/aots-e/README.md
new file mode 100644
index 0000000..83c2b5f
--- /dev/null
+++ b/aots-e/README.md
@@ -0,0 +1,23 @@
+# Ashes of the Singularity - Escalation
+
+This benchmark runs one of the built in benchmarks for the game.
+
+## Prerequisites
+
+- Python 3.10+
+- Stellaris installed
+- Keras OCR service
+
+## Options
+
+- `benchmark`: Specifies which benchmark to run
+
+## Output
+
+report.json
+- `test`: The name of the selected benchmark
+- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080`
+- `start_time`: number representing a timestamp of the test's start time in milliseconds
+- `end_time`: number representing a timestamp of the test's end time in milliseconds
+- `score` : average fps for the benchmark
+- `version` : The build number of the game according to steam
\ No newline at end of file
diff --git a/aots-e/aotse.py b/aots-e/aotse.py
new file mode 100644
index 0000000..324a3d2
--- /dev/null
+++ b/aots-e/aotse.py
@@ -0,0 +1,147 @@
+"""Ashes of the Singularity: Escalation test script"""
+from argparse import ArgumentParser
+import logging
+from pathlib import Path
+import subprocess
+import sys
+import time
+import getpass
+import glob
+import os
+from aotse_utils import read_current_resolution, run_game_benchmark, find_score_in_log, wait_for_benchmark_process, delete_old_scores
+
+PARENT_DIR = str(Path(sys.path[0], ".."))
+sys.path.append(PARENT_DIR)
+
+from harness_utils.steam import get_build_id
+from harness_utils.output import (
+    DEFAULT_DATE_FORMAT,
+    DEFAULT_LOGGING_FORMAT,
+    seconds_to_milliseconds,
+    setup_log_directory,
+    format_resolution,
+    write_report_json
+)
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+
+#####
+### Globals
+#####
+USERNAME = getpass.getuser()
+CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Ashes of the Singularity - Escalation")
+CONFIG_FILENAME = "settings.ini"
+STEAM_GAME_ID = 507490
+SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = SCRIPT_DIR / "run"
+EXECUTABLE = "AshesEscalation_DX12.exe"
+CONFIG_DIR = SCRIPT_DIR / "config"
+BENCHMARK_CONFIG = {
+    "GPU_Benchmark": {
+        "hardware": "GPU",
+        "config": "benchfinal",
+        "process_name":  "AshesEscalation_DX12.exe",
+        "score_name": "Avg Framerate:",
+        "test_name": "Ashes of the Singularity: Escalation GPU Benchmark"
+    },
+    "CPU_Benchmark": {
+        "hardware": "CPU",
+        "config": "CPUbench",
+        "process_name":  "AshesEscalation_DX12.exe",
+        "score_name": "CPU frame rate \(estimated if not GPU bound\):",
+        "test_name": "Ashes of the Singularity: Escalation CPU Benchmark"
+    }
+}
+cfg = f"{CONFIG_PATH}\\{CONFIG_FILENAME}"
+
+
+def setup_logging():
+    """setup logging"""
+    setup_log_directory(LOG_DIR)
+    logging.basicConfig(filename=LOG_DIR / "harness.log",
+                        format=DEFAULT_LOGGING_FORMAT,
+                        datefmt=DEFAULT_DATE_FORMAT,
+                        level=logging.DEBUG)
+    console = logging.StreamHandler()
+    formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+    console.setFormatter(formatter)
+    logging.getLogger('').addHandler(console)
+
+def get_arguments():
+    """get arguments"""
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--benchmark", dest="benchmark", help="Benchmark test type", required=True, choices=BENCHMARK_CONFIG.keys())
+    argies = parser.parse_args()
+    return argies
+
+def start_game():
+    """Launch the game with no launcher or start screen"""
+    test_option = BENCHMARK_CONFIG[args.benchmark]["config"]
+    return run_game_benchmark(STEAM_GAME_ID, game_params=["-benchmark", f"{test_option}"])
+
+def run_benchmark(process_name, command_to_run):
+    """Run the benchmark and wait for the benchmark process to finish."""
+    # Start Steam via subprocess.Popen
+    with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
+        
+        # Wait for the actual benchmark process to start and finish
+        test_name=BENCHMARK_CONFIG[args.benchmark]["test_name"]
+        wait_for_benchmark_process(test_name, process_name)
+        
+        # Now that the process has finished, collect the results
+        stdout, stderr = proc.communicate()  # This waits for the Steam subprocess to exit (if any)
+
+        # Log any output and error from the Steam process
+        if stdout:
+            logging.info(f"Steam Output: {stdout}")
+        if stderr:
+            logging.error(f"Steam Error: {stderr}")
+        
+        # Now, return the process object
+        return proc
+
+am = ArtifactManager(LOG_DIR)
+
+try:
+    setup_logging()
+    args = get_arguments()
+    cmd = start_game()
+    logging.info('Starting benchmark!')
+    logging.info(cmd)
+    start_time = time.time()
+    result="Output_*_*_*_*.txt"
+    delete_old_scores(result)
+    run_benchmark(BENCHMARK_CONFIG[args.benchmark]["process_name"], cmd)
+    score = find_score_in_log(BENCHMARK_CONFIG[args.benchmark]["score_name"], result)
+    
+    if score is None:
+        logging.error("Could not find average FPS output!")
+        sys.exit(1)
+
+    end_time = time.time()
+    elapsed_test_time = round(end_time - start_time, 2)
+    logging.info("Benchmark took %.2f seconds", elapsed_test_time)
+    logging.info("Score was %s", score)
+
+    am.copy_file(cfg, ArtifactType.CONFIG_TEXT, "Settings file")
+    result_file = sorted(glob.glob(os.path.join(CONFIG_PATH, result)), key=os.path.getmtime, reverse=True)
+    output_file = result_file[0]
+    am.copy_file(output_file, ArtifactType.CONFIG_TEXT, "Results file")
+    hardware = BENCHMARK_CONFIG[args.benchmark]["hardware"]
+    width, height = read_current_resolution()
+    report = {
+        "test": BENCHMARK_CONFIG[args.benchmark]["test_name"],
+        "unit": f"Average {hardware} FPS",
+        "score": score,
+        "resolution": format_resolution(width, height),
+        "start_time": seconds_to_milliseconds(start_time),
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
+    }
+
+    am.create_manifest()
+    write_report_json(LOG_DIR, "report.json", report)
+except Exception as e:
+    logging.error("Something went wrong running the benchmark!")
+    logging.exception(e)
+    sys.exit(1)
diff --git a/aots-e/aotse_utils.py b/aots-e/aotse_utils.py
new file mode 100644
index 0000000..deedfb4
--- /dev/null
+++ b/aots-e/aotse_utils.py
@@ -0,0 +1,110 @@
+"""Utility functions for Total War: Warhammer III test script"""
+import os
+import re
+import sys
+import logging
+import getpass
+from pathlib import Path
+import psutil
+import glob
+import time
+
+PARENT_DIR = str(Path(sys.path[0], ".."))
+sys.path.append(PARENT_DIR)
+
+from harness_utils.steam import get_steam_exe_path
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+USERNAME = getpass.getuser()
+SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = SCRIPT_DIR.joinpath("run")
+PROCESS_NAME = "stellaris.exe"
+STEAM_GAME_ID = 281990
+CONFIG_FILENAME = "settings.ini"
+USERNAME = getpass.getuser()
+CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Ashes of the Singularity - Escalation")
+
+def read_current_resolution():
+    """Get resolution from local game file"""
+    resolution_pattern = re.compile(r"Resolution=(\d+),(\d+)")
+    cfg = f"{CONFIG_PATH}\\{CONFIG_FILENAME}"
+    width = 0
+    height = 0
+    with open(cfg, encoding="utf-8") as file:
+        lines = file.readlines()
+        for line in lines:
+            resolution_match = resolution_pattern.search(line)
+            if resolution_match is not None:
+                width, height = resolution_match.groups()
+    return width, height
+
+def delete_old_scores(file):
+    """Deletes old score files based on a given pattern"""
+    files = glob.glob(os.path.join(CONFIG_PATH, file))
+    for file in files:
+        try:
+            os.remove(file)
+            logging.info(f"Deleted old score file: {file}")
+        except Exception as e:
+            print(f"Error deleting file {file}: {e}")
+
+def find_score_in_log(score_name, file):
+    """Reads score from local game log"""
+    files = sorted(glob.glob(os.path.join(CONFIG_PATH, file)), key=os.path.getmtime, reverse=True)
+    if not files:
+        return None
+    
+    score_pattern = re.compile(rf"^{score_name}\s*(\d+\.\d+) FPS")
+    score_value = 0
+    with open(files[0], encoding="ANSI") as file:
+        lines = file.readlines()
+        for line in lines:
+            score_match = score_pattern.search(line)
+            if score_match is not None:
+                score_value = score_match.group(1)
+                break
+    return score_value
+
+def is_process_running(process_name):
+    """check if given process is running"""
+    for process in psutil.process_iter(['pid', 'name']):
+        if process.info['name'] == process_name:
+            return process
+    return None
+
+def wait_for_benchmark_process(test_name, process_name, timeout=60):
+    """Wait for the benchmark game process to start and then finish."""
+    logging.info(f"Waiting for benchmark process '{process_name}' to start...")
+
+    start_time = time.time()
+
+    while True:
+        # Check if the benchmark process is running
+        process = is_process_running(process_name)
+        if process:
+            logging.info(f"{test_name} has started. Waiting for it to finish...")
+            process.wait()  # This will block until the process finishes
+            logging.info(f"Benchmark has finished.")
+            break
+
+        # If we exceed the timeout, break out of the loop and log an error
+        if time.time() - start_time > timeout:
+            logging.error(f"Timeout reached while waiting for process '{process_name}'.")
+            raise TimeoutError(f"Process '{process_name}' did not start within the expected time. Is the game configured for DX12?")
+
+        # Wait for 1 second before checking again
+        time.sleep(1)
+
+def run_game_benchmark(game_id: int, steam_path=None, game_params=None):
+    """Runs a game by providing steam executable with an array of parameters.
+    The `steam_path` argument can be used to specify a specifc path to the Steam executable
+    instead of relying on finding the current installation in the Window's registry.
+    """
+    if steam_path is None:
+        steam_path = get_steam_exe_path()
+    if game_params is None:
+        game_params = []
+    command = [steam_path, "-applaunch", str(game_id)] + game_params
+    logging.info(", ".join(command))
+    return command
\ No newline at end of file
diff --git a/aots-e/manifest.yaml b/aots-e/manifest.yaml
new file mode 100644
index 0000000..4b15659
--- /dev/null
+++ b/aots-e/manifest.yaml
@@ -0,0 +1,13 @@
+friendly_name: "Ashes of the Singularity: Escalation"
+executable: "aotse.py"
+process_name: "AshesEscalation_DX12.exe"
+disable_presentmon: true
+output_dir: "run"
+options:
+  - name: benchmark
+    type: select
+    values:
+      - "GPU_Benchmark"
+      - "CPU_Benchmark"
+    tooltip: Select which benchmark scene to run
+  
\ No newline at end of file

From 8d0fcf70874c2c533121622434baa39b6229195b Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:17:05 -0800
Subject: [PATCH 22/25] Added RTSS support to fix high FPS menu bug

---
 .../forza_steamworks_release_final.exe.cfg    | 74 +++++++++++++++++++
 forzamotorsport/forzams.py                    |  9 +++
 2 files changed, 83 insertions(+)
 create mode 100644 forzamotorsport/forza_steamworks_release_final.exe.cfg

diff --git a/forzamotorsport/forza_steamworks_release_final.exe.cfg b/forzamotorsport/forza_steamworks_release_final.exe.cfg
new file mode 100644
index 0000000..9c558f4
--- /dev/null
+++ b/forzamotorsport/forza_steamworks_release_final.exe.cfg
@@ -0,0 +1,74 @@
+[OSD]
+EnableOSD=1
+EnableBgnd=1
+EnableFill=0
+EnableStat=0
+BaseColor=00FF8000
+BgndColor=00000000
+FillColor=80000000
+PositionX=1
+PositionY=1
+ZoomRatio=2
+CoordinateSpace=0
+EnableFrameColorBar=0
+FrameColorBarMode=0
+RefreshPeriod=500
+IntegerFramerate=1
+MaximumFrametime=0
+EnableFrametimeHistory=0
+FrametimeHistoryWidth=-32
+FrametimeHistoryHeight=-4
+FrametimeHistoryStyle=0
+ScaleToFit=0
+[Statistics]
+FramerateAveragingInterval=1000
+PeakFramerateCalc=0
+PercentileCalc=0
+FrametimeCalc=0
+PercentileBuffer=0
+[Framerate]
+Limit=900
+LimitDenominator=1
+LimitTime=0
+LimitTimeDenominator=1
+SyncScanline0=0
+SyncScanline1=0
+SyncPeriods=0
+SyncLimiter=0
+PassiveWait=1
+[Hooking]
+EnableHooking=1
+EnableFloatingInjectionAddress=0
+EnableDynamicOffsetDetection=0
+HookLoadLibrary=0
+HookDirectDraw=0
+HookDirect3D8=1
+HookDirect3D9=1
+HookDirect3DSwapChain9Present=1
+HookDXGI=1
+HookDirect3D12=1
+HookOpenGL=1
+HookVulkan=1
+InjectionDelay=15000
+UseDetours=0
+[Font]
+Height=-9
+Weight=400
+Face=Unispace
+Load=
+[RendererDirect3D8]
+Implementation=2
+[RendererDirect3D9]
+Implementation=2
+[RendererDirect3D10]
+Implementation=2
+[RendererDirect3D11]
+Implementation=2
+[RendererDirect3D12]
+Implementation=2
+[RendererOpenGL]
+Implementation=2
+[RendererVulkan]
+Implementation=2
+[Info]
+Timestamp=19-04-2023, 16:58:10
diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index a9a58e2..8ea983c 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -23,6 +23,7 @@ from harness_utils.steam import (
   exec_steam_run_command,
   get_build_id
 )
+from harness_utils.rtss import  start_rtss_process, copy_rtss_profile
 from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 STEAM_GAME_ID = 2440510
@@ -36,9 +37,17 @@ LOCAL_USER_SETTINGS = os.path.join(
 
 user.FAILSAFE = False
 
+def start_rtss():
+    """Sets up the RTSS process"""
+    profile_path = os.path.join(SCRIPT_DIRECTORY, "forza_steamworks_release_final.exe.cfg")
+    copy_rtss_profile(profile_path)
+    return start_rtss_process()
 
 def run_benchmark() -> tuple[float]:
     """Run the benchmark"""
+    start_rtss()
+    # Give RTSS time to start
+    time.sleep(10)
     logging.info("Starting game")
     exec_steam_run_command(STEAM_GAME_ID)
     setup_start_time = time.time()

From 5be351cd36763500c5c996ba8d0ff588ae5d1c22 Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:25:07 -0800
Subject: [PATCH 23/25] Mirrored RTSS functionality of FH5 harness

---
 forzamotorsport/forzams.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/forzamotorsport/forzams.py b/forzamotorsport/forzams.py
index 8ea983c..7f99a98 100644
--- a/forzamotorsport/forzams.py
+++ b/forzamotorsport/forzams.py
@@ -29,11 +29,11 @@ from harness_utils.artifacts import ArtifactManager, ArtifactType
 STEAM_GAME_ID = 2440510
 SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
-PROCESS_NAME = "forza_steamworks_release_final"
 LOCAL_USER_SETTINGS = os.path.join(
   os.getenv('LOCALAPPDATA'), "Microsoft.ForzaMotorsport", "User_SteamLocalStorageDirectory",
   "ConnectedStorage", "ForzaUserConfigSelections", "UserConfigSelections"
   )
+PROCESSES = ["forza_steamworks_release_final.exe", "RTSS.exe"]
 
 user.FAILSAFE = False
 
@@ -141,7 +141,7 @@ def run_benchmark() -> tuple[float]:
     elapsed_test_time = round((test_end_time - test_start_time), 2)
     logging.info("Benchmark took %s seconds", elapsed_test_time)
 
-    terminate_processes(PROCESS_NAME)
+    terminate_processes(*PROCESSES)
     am.create_manifest()
 
     return test_start_time, test_end_time
@@ -175,5 +175,5 @@ try:
 except Exception as e:
     logging.error("Something went wrong running the benchmark!")
     logging.exception(e)
-    terminate_processes(PROCESS_NAME)
+    terminate_processes(*PROCESSES)
     sys.exit(1)

From 829fe64ccd5dfbac25ecf867e81e926b86ee2ceb Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Tue, 7 Jan 2025 16:11:23 -0800
Subject: [PATCH 24/25] trying launching exe directly instead

---
 aots-e/aotse.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/aots-e/aotse.py b/aots-e/aotse.py
index 324a3d2..bc42636 100644
--- a/aots-e/aotse.py
+++ b/aots-e/aotse.py
@@ -13,7 +13,7 @@ from aotse_utils import read_current_resolution, run_game_benchmark, find_score_
 PARENT_DIR = str(Path(sys.path[0], ".."))
 sys.path.append(PARENT_DIR)
 
-from harness_utils.steam import get_build_id
+from harness_utils.steam import get_app_install_location, get_build_id
 from harness_utils.output import (
     DEFAULT_DATE_FORMAT,
     DEFAULT_LOGGING_FORMAT,
@@ -74,10 +74,17 @@ def get_arguments():
     argies = parser.parse_args()
     return argies
 
+# def start_game():
+#     """Launch the game with no launcher or start screen"""
+#     test_option = BENCHMARK_CONFIG[args.benchmark]["config"]
+#     return run_game_benchmark(STEAM_GAME_ID, game_params=["-benchmark", f"{test_option}"])
+
 def start_game():
-    """Launch the game with no launcher or start screen"""
+    """Starts the game process"""
     test_option = BENCHMARK_CONFIG[args.benchmark]["config"]
-    return run_game_benchmark(STEAM_GAME_ID, game_params=["-benchmark", f"{test_option}"])
+    cmd_string = f"start /D \"{get_app_install_location(STEAM_GAME_ID)}\" {EXECUTABLE} -benchmark {test_option}"
+    logging.info(cmd_string)
+    return os.system(cmd_string)
 
 def run_benchmark(process_name, command_to_run):
     """Run the benchmark and wait for the benchmark process to finish."""
@@ -111,7 +118,7 @@ try:
     start_time = time.time()
     result="Output_*_*_*_*.txt"
     delete_old_scores(result)
-    run_benchmark(BENCHMARK_CONFIG[args.benchmark]["process_name"], cmd)
+    run_benchmark(EXECUTABLE, cmd)
     score = find_score_in_log(BENCHMARK_CONFIG[args.benchmark]["score_name"], result)
     
     if score is None:

From 6ff463c1de490d63a73c65e4c9f988567e32e30d Mon Sep 17 00:00:00 2001
From: J-Doiron <139803019+J-Doiron@users.noreply.github.com>
Date: Tue, 7 Jan 2025 18:21:26 -0800
Subject: [PATCH 25/25] Cleaned up code for executing exe directly

---
 aots-e/aotse.py       | 16 +++++-----------
 aots-e/aotse_utils.py | 15 +--------------
 2 files changed, 6 insertions(+), 25 deletions(-)

diff --git a/aots-e/aotse.py b/aots-e/aotse.py
index bc42636..76debb1 100644
--- a/aots-e/aotse.py
+++ b/aots-e/aotse.py
@@ -8,7 +8,7 @@ import time
 import getpass
 import glob
 import os
-from aotse_utils import read_current_resolution, run_game_benchmark, find_score_in_log, wait_for_benchmark_process, delete_old_scores
+from aotse_utils import read_current_resolution, find_score_in_log, wait_for_benchmark_process, delete_old_scores
 
 PARENT_DIR = str(Path(sys.path[0], ".."))
 sys.path.append(PARENT_DIR)
@@ -39,19 +39,18 @@ BENCHMARK_CONFIG = {
     "GPU_Benchmark": {
         "hardware": "GPU",
         "config": "benchfinal",
-        "process_name":  "AshesEscalation_DX12.exe",
         "score_name": "Avg Framerate:",
         "test_name": "Ashes of the Singularity: Escalation GPU Benchmark"
     },
     "CPU_Benchmark": {
         "hardware": "CPU",
         "config": "CPUbench",
-        "process_name":  "AshesEscalation_DX12.exe",
         "score_name": "CPU frame rate \(estimated if not GPU bound\):",
         "test_name": "Ashes of the Singularity: Escalation CPU Benchmark"
     }
 }
 cfg = f"{CONFIG_PATH}\\{CONFIG_FILENAME}"
+GAME_DIR = get_app_install_location(STEAM_GAME_ID)
 
 
 def setup_logging():
@@ -74,22 +73,17 @@ def get_arguments():
     argies = parser.parse_args()
     return argies
 
-# def start_game():
-#     """Launch the game with no launcher or start screen"""
-#     test_option = BENCHMARK_CONFIG[args.benchmark]["config"]
-#     return run_game_benchmark(STEAM_GAME_ID, game_params=["-benchmark", f"{test_option}"])
-
 def start_game():
     """Starts the game process"""
     test_option = BENCHMARK_CONFIG[args.benchmark]["config"]
-    cmd_string = f"start /D \"{get_app_install_location(STEAM_GAME_ID)}\" {EXECUTABLE} -benchmark {test_option}"
+    cmd_string = f"\"{GAME_DIR}\\{EXECUTABLE}\" -benchmark {test_option}"
     logging.info(cmd_string)
-    return os.system(cmd_string)
+    return cmd_string
 
 def run_benchmark(process_name, command_to_run):
     """Run the benchmark and wait for the benchmark process to finish."""
     # Start Steam via subprocess.Popen
-    with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
+    with subprocess.Popen(command_to_run, cwd=GAME_DIR, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
         
         # Wait for the actual benchmark process to start and finish
         test_name=BENCHMARK_CONFIG[args.benchmark]["test_name"]
diff --git a/aots-e/aotse_utils.py b/aots-e/aotse_utils.py
index deedfb4..7e597ab 100644
--- a/aots-e/aotse_utils.py
+++ b/aots-e/aotse_utils.py
@@ -94,17 +94,4 @@ def wait_for_benchmark_process(test_name, process_name, timeout=60):
             raise TimeoutError(f"Process '{process_name}' did not start within the expected time. Is the game configured for DX12?")
 
         # Wait for 1 second before checking again
-        time.sleep(1)
-
-def run_game_benchmark(game_id: int, steam_path=None, game_params=None):
-    """Runs a game by providing steam executable with an array of parameters.
-    The `steam_path` argument can be used to specify a specifc path to the Steam executable
-    instead of relying on finding the current installation in the Window's registry.
-    """
-    if steam_path is None:
-        steam_path = get_steam_exe_path()
-    if game_params is None:
-        game_params = []
-    command = [steam_path, "-applaunch", str(game_id)] + game_params
-    logging.info(", ".join(command))
-    return command
\ No newline at end of file
+        time.sleep(1)
\ No newline at end of file