Merge branch 'main' of github.com:LTTLabsOSS/markbench-tests into niharris/add-screenshot-splitting

F1_23/f1_23.py

@@ -2,6 +2,7 @@
 import logging
 from argparse import ArgumentParser
 import os.path
+import re
 import time
 import sys
 import pydirectinput as user
@@ -9,9 +10,9 @@ from f1_23_utils import get_resolution
 
 sys.path.insert(1, os.path.join(sys.path[0], ".."))
 
-from harness_utils.steam import exec_steam_run_command, get_app_install_location
+from harness_utils.steam import exec_steam_run_command, get_app_install_location, get_build_id
 from harness_utils.keras_service import KerasService
-from harness_utils.misc import remove_files
+from harness_utils.misc import remove_files, press_n_times
 from harness_utils.process import terminate_processes
 from harness_utils.output import (
     format_resolution,
@@ -21,6 +22,7 @@ from harness_utils.output import (
     DEFAULT_LOGGING_FORMAT,
     DEFAULT_DATE_FORMAT,
 )
+from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
@@ -28,12 +30,31 @@ PROCESS_NAME = "F1_23"
 STEAM_GAME_ID = 2108330
 VIDEO_PATH = os.path.join(get_app_install_location(STEAM_GAME_ID), "videos")
 
+username = os.getlogin()
+CONFIG_PATH = f"C:\\Users\\{username}\\Documents\\My Games\\F1 23\\hardwaresettings"
+CONFIG_FILENAME = "hardware_settings_config.xml"
+CONFIG = f"{CONFIG_PATH}\\{CONFIG_FILENAME}"
+BENCHMARK_RESULTS_PATH = f"C:\\Users\\{username}\\Documents\\My Games\\F1 23\\benchmark"
+
 intro_videos = [
     os.path.join(VIDEO_PATH, "attract.bk2"),
     os.path.join(VIDEO_PATH, "cm_f1_sting.bk2")
 ]
 
 
+def find_latest_result_file(base_path):
+    """Look for files in the benchmark results path that match the pattern in the regular expression"""
+    pattern = r"benchmark_.*\.xml"
+    list_of_files = []
+
+    for filename in os.listdir(base_path):
+        if re.search(pattern, filename, re.IGNORECASE):
+            list_of_files.append(base_path + '\\' +filename)
+
+    latest_file = max(list_of_files, key=os.path.getmtime)
+
+    return latest_file
+
 def find_settings() -> any:
     """Look for and enter settings"""
     if not kerasService.look_for_word("settings", attempts=5, interval=3):
@@ -49,49 +70,11 @@ def find_graphics() -> any:
         logging.info("Didn't find graphics!")
         sys.exit(1)
     user.press("right")
-    time.sleep(0.5)
+    time.sleep(0.2)
     user.press("enter")
     time.sleep(1.5)
 
 
-def find_benchmark() -> any:
-    """Look for and enter benchmark options"""
-    if not kerasService.look_for_word("benchmark", attempts=5, interval=3):
-        logging.info("Didn't find benchmark!")
-        sys.exit(1)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("enter")
-    time.sleep(1.5)
-
-
-def find_weather() -> any:
-    """Navigate to start benchmark"""
-    if not kerasService.look_for_word("weather", attempts=5, interval=3):
-        logging.info("Didn't find weather!")
-        sys.exit(1)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("enter")
-    time.sleep(2)
-
-
 def navigate_startup():
     """press space through the warnings and navigate startup menus"""
     result = kerasService.wait_for_word("product", timeout=40)
@@ -124,8 +107,17 @@ def navigate_startup():
         time.sleep(2)
 
 
-def navigate_menu():
-    """Simulate inputs to navigate to benchmark option."""
+def run_benchmark():
+    """Runs the actual benchmark."""
+    remove_files(intro_videos)
+    exec_steam_run_command(STEAM_GAME_ID)
+    am = ArtifactManager(LOG_DIRECTORY)
+
+    setup_start_time = time.time()
+    time.sleep(2)
+    navigate_startup()
+
+    # Navigate menus and take screenshots using the artifact manager
     result = kerasService.wait_for_word("theatre", interval=3, timeout=60)
     if not result:
         logging.info("Didn't land on the main menu!")
@@ -133,36 +125,55 @@ def navigate_menu():
 
     logging.info("Saw the options! we are good to go!")
     time.sleep(1)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
-    user.press("down")
-    time.sleep(0.5)
+
+    press_n_times("down", 7, 0.2)
     user.press("enter")
     time.sleep(2)
-
     find_settings()
     find_graphics()
-    find_benchmark()
-    find_weather() # Run benchmark!
 
-def run_benchmark():
-    """Runs the actual benchmark."""
-    remove_files(intro_videos)
-    exec_steam_run_command(STEAM_GAME_ID)
-    setup_start_time = time.time()
+    # Navigate to video settings
+    press_n_times("down", 3, 0.2)
+    user.press("enter")
+    time.sleep(0.2)
+
+    result = kerasService.wait_for_word("vsync", interval=3, timeout=60)
+    if not result:
+        logging.info("Didn't find the keyword 'vsync'. Did the program navigate to the video mode menu correctly?")
+        sys.exit(1)
+    am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings menu")
+    user.press("esc")
+    time.sleep(0.2)
+
+    result = kerasService.wait_for_word("steering", interval=3, timeout=60)
+    if not result:
+        logging.info("Didn't find the keyword 'steering'. Did the program exit the video mode menu correctly?")
+        sys.exit(1)
+
+    # Navigate through graphics settings and take screenshots of all settings contained within
+    am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "first screenshot of graphics settings")
+    press_n_times("down", 30, 0.2)
+
+    result = kerasService.wait_for_word("chromatic", interval=3, timeout=60)
+    if not result:
+        logging.info("Didn't find the keyword 'chromatic'. Did we navigate the menu correctly?")
+        sys.exit(1)
+
+    am.take_screenshot("graphics_2.png", ArtifactType.CONFIG_IMAGE, "second screenshot of graphics settings")
+    press_n_times("up", 29, 0.2)
+    user.press("enter")
+    time.sleep(0.2)
+
+    # Navigate benchmark menu
+    if not kerasService.look_for_word("weather", attempts=5, interval=3):
+        logging.info("Didn't find weather!")
+        sys.exit(1)
+
+    am.take_screenshot("benchmark.png", ArtifactType.CONFIG_IMAGE, "screenshot of benchmark settings")
+
+    press_n_times("down", 6, 0.2)
+    user.press("enter")
     time.sleep(2)
-    navigate_startup()
-    navigate_menu()
 
     elapsed_setup_time = round(time.time() - setup_start_time, 2)
     logging.info("Setup took %f seconds", elapsed_setup_time)
@@ -193,8 +204,11 @@ def run_benchmark():
         logging.info("Results screen was not found!" +
             "Did harness not wait long enough? Or test was too long?")
         sys.exit(1)
-
     logging.info("Results screen was found! Finishing benchmark.")
+    results_file = find_latest_result_file(BENCHMARK_RESULTS_PATH)
+    am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of results")
+    am.copy_file(CONFIG, ArtifactType.CONFIG_TEXT, "config file")
+    am.copy_file(results_file, ArtifactType.RESULTS_TEXT, "benchmark results xml file")
 
     if test_end_time is None:
         logging.info("Loading screen end time not found. Using results screen fallback time.")
@@ -237,6 +251,7 @@ try:
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
         "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIRECTORY, "report.json", report)

README.md

@@ -201,6 +201,8 @@ For detailed instructions on setting up this Keras Service locally, please refer
 
 > Please note that although a CUDA-capable GPU is not mandatory, it's worth mentioning that certain games may not function correctly due to slower response times when this hardware is absent.
 
+If Keras is taking images on the wrong monitor, the primary display can be modified by changing `monitor_1 = sct.monitors[2]  # Identify the display to capture` in `keras_service.py`
+
 ### Keyboard and Mouse Input
 
 For keyboard and mouse input, we employ two distinct methods. The first method involves using Virtual Key Codes (VKs) with the deprecated Win32 functions mouse_event() and keybd_event(). The second method utilizes Send Input. Specifically, [PyAutoGui](https://pyautogui.readthedocs.io/en/latest/) implements the first approach, while [PyDirectInput](https://pypi.org/project/PyDirectInput/) implements the second.

counterstrike2/cs2.py

@@ -19,7 +19,8 @@ from harness_utils.output import (
     DEFAULT_DATE_FORMAT)
 from harness_utils.process import terminate_processes
 from harness_utils.keras_service import KerasService
-from harness_utils.steam import exec_steam_game
+from harness_utils.steam import exec_steam_game, get_registry_active_user, get_steam_folder_path, get_build_id
+from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 
 SCRIPT_DIR = Path(__file__).resolve().parent
@@ -27,6 +28,9 @@ LOG_DIR = SCRIPT_DIR.joinpath("run")
 PROCESS_NAME = "cs2.exe"
 STEAM_GAME_ID = 730
 
+STEAM_USER_ID = get_registry_active_user()
+cfg = Path(get_steam_folder_path(), "userdata", str(STEAM_USER_ID), str(STEAM_GAME_ID), "local", "cfg", "cs2_video.txt")
+
 def setup_logging():
     """default logging config"""
     LOG_DIR.mkdir(exist_ok=True)
@@ -56,6 +60,7 @@ def run_benchmark(keras_service):
     copy_config()
     setup_start_time = time.time()
     start_game()
+    am = ArtifactManager(LOG_DIR)
     time.sleep(20)  # wait for game to load into main menu
 
     result = keras_service.wait_for_word("play", timeout=30, interval=0.1)
@@ -63,6 +68,62 @@ def run_benchmark(keras_service):
         logging.info("Did not find the play menu. Did the game load?")
         sys.exit(1)
 
+    height, width = get_resolution()
+    location = None
+
+    # We check the resolution so we know which screenshot to use for the locate on screen function
+    match width:
+        case "1920":
+            location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1080.png")
+        case "2560":
+            location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1440.png")
+        case "3840":
+            location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_2160.png")
+        case _:
+            logging.error("Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width)
+            sys.exit(1)
+
+    click_me = gui.center(location)
+    gui.moveTo(click_me.x, click_me.y)
+    gui.mouseDown()
+    time.sleep(0.2)
+    gui.mouseUp()
+    time.sleep(0.2)
+
+    if keras_service.wait_for_word(word="brightness", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+
+    am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings")
+
+    result = keras_service.look_for_word(word="advanced", attempts=10, interval=1)
+    if not result:
+        logging.info("Did not find the advanced video menu. Did Keras click correctly?")
+        sys.exit(1)
+
+    gui.moveTo(result["x"], result["y"])
+    gui.mouseDown()
+    time.sleep(0.2)
+    gui.mouseUp()
+    time.sleep(0.2)
+
+    am.take_screenshot("advanced_video_1.png", ArtifactType.CONFIG_IMAGE, "first picture of advanced video settings")
+
+    result = keras_service.look_for_word(word="boost", attempts=10, interval=1)
+    if not result:
+        logging.info("Did not find the keyword 'Boost' in the advanced video menu. Did Keras click correctly?")
+        sys.exit(1)
+
+    gui.moveTo(result["x"], result["y"])
+    time.sleep(1)
+    gui.scroll(-6000000)
+    time.sleep(1)
+
+    if keras_service.wait_for_word(word="particle", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'Particle' in advanced video menu. Did Keras scroll correctly?")
+        sys.exit(1)
+    am.take_screenshot("advanced_video_2.png", ArtifactType.CONFIG_IMAGE, "second picture of advanced video settings")
+
     logging.info('Starting benchmark')
     user.press("`")
     time.sleep(0.5)
@@ -107,8 +168,13 @@ def run_benchmark(keras_service):
         test_end_time = time.time()
         logging.info("The console opened. Marking end time.")
 
-    time.sleep(10)
+    # allow time for result screen to populate
+    time.sleep(8)
+
+    am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "benchmark results")
+    am.copy_file(Path(cfg), ArtifactType.CONFIG_TEXT, "cs2 video config")
     logging.info("Run completed. Closing game.")
+    time.sleep(2)
 
     elapsed_test_time = round((test_end_time - test_start_time), 2)
     logging.info("Benchmark took %f seconds", elapsed_test_time)
@@ -133,7 +199,8 @@ def main():
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
-        "end_time": seconds_to_milliseconds(end_time)
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIR, "report.json", report)

grid_legends/grid_legends.py

@@ -20,24 +20,25 @@ from harness_utils.output import (
     DEFAULT_DATE_FORMAT)
 from harness_utils.process import terminate_processes
 from harness_utils.keras_service import KerasService
-from harness_utils.steam import exec_steam_game
+from harness_utils.steam import exec_steam_game, get_build_id
+from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 LOG_DIR = SCRIPT_DIR.joinpath("run")
 PROCESS_NAME = "gridlegends.exe"
 STEAM_GAME_ID = 1307710
 
+username = os.getlogin()
+CONFIG_PATH = f"C:\\Users\\{username}\\Documents\\My Games\\GRID Legends\\hardwaresettings"
+CONFIG_FILENAME = "hardware_settings_config.xml"
+CONFIG_FULL_PATH = f"{CONFIG_PATH}\\{CONFIG_FILENAME}"
 
 def get_resolution() -> tuple[int]:
     """Gets resolution width and height from local xml file created by game."""
-    username = os.getlogin()
-    config_path = f"C:\\Users\\{username}\\Documents\\My Games\\GRID Legends\\hardwaresettings"
-    config_filename = "hardware_settings_config.xml"
     resolution = re.compile(r"<resolution width=\"(\d+)\" height=\"(\d+)\"")
-    cfg = f"{config_path}\\{config_filename}"
     height = 0
     width = 0
-    with open(cfg, encoding="utf-8") as file:
+    with open(CONFIG_FULL_PATH, encoding="utf-8") as file:
         lines = file.readlines()
         for line in lines:
             height_match = resolution.search(line)
@@ -81,6 +82,8 @@ def run_benchmark(keras_service):
     """Run Grid Legends benchmark"""
     setup_start_time = time.time()
     start_game()
+    am = ArtifactManager(LOG_DIR)
+
     time.sleep(20)  # wait for game to load to the start screen
 
     if keras_service.wait_for_word(word="press", timeout=80, interval=1) is None:
@@ -110,9 +113,26 @@ def run_benchmark(keras_service):
     if keras_service.wait_for_word(word="basic", timeout=30, interval=0.1) is None:
         logging.error("Didn't basic video options. Did the menu navigate correctly?")
         sys.exit(1)
+    am.take_screenshot("basic.png", ArtifactType.CONFIG_IMAGE, "picture of basic settings")
 
     user.press("f3")
     time.sleep(0.2)
+
+    if keras_service.wait_for_word(word="benchmark", timeout=30, interval=0.1) is None:
+        logging.error("Didn't reach advanced video options. Did the menu navigate correctly?")
+        sys.exit(1)
+    am.take_screenshot("advanced_1.png", ArtifactType.CONFIG_IMAGE, "first picture of advanced settings")
+
+    user.press("up")
+    time.sleep(0.2)
+
+    if keras_service.wait_for_word(word="shading", timeout=30, interval=0.1) is None:
+        logging.error("Didn't reach bottom of advanced video settings. Did the menu navigate correctly?")
+        sys.exit(1)
+    am.take_screenshot("advanced_2.png", ArtifactType.CONFIG_IMAGE, "second picture of advanced settings")
+
+    user.press("down")
+    time.sleep(0.2)
     user.press("enter")
     time.sleep(0.2)
 
@@ -134,6 +154,9 @@ def run_benchmark(keras_service):
     test_end_time = time.time() - 2
     time.sleep(2)
 
+    am.take_screenshot("results.png", ArtifactType.RESULTS_IMAGE, "benchmark results")
+    am.copy_file(Path(CONFIG_FULL_PATH), ArtifactType.CONFIG_TEXT, "game config")
+
     logging.info("Run completed. Closing game.")
     time.sleep(2)
     return test_start_time, test_end_time
@@ -151,7 +174,8 @@ def main():
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
-        "end_time": seconds_to_milliseconds(end_time)
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIR, "report.json", report)

harness_utils/misc.py

@@ -3,9 +3,29 @@ import logging
 import os
 from pathlib import Path
 from zipfile import ZipFile
-
+import time
+import pydirectinput as user
+import pyautogui as gui
 import requests
 
+def mouse_scroll_n_times(n: int, scroll_amount: int, pause: float):
+    """
+    Pyautogui's mouse scroll function often fails to actually scroll in game menus, this functions solves that problem
+    
+    n --> the number of times you want to scroll, should be a positive integer
+    scroll_amount --> positive is scroll up, negative is scroll down
+    pause --> the amount of time to pause betwee subsequent scrolls
+    """
+    for _ in range(n):
+        gui.vscroll(scroll_amount)
+        time.sleep(pause)
+
+def press_n_times(key: str, n: int, pause: float):
+    """A helper function press the same button multiple times"""
+    for _ in range(n):
+        user.press(key)
+        time.sleep(pause)
+
 def remove_files(paths: list[str]) -> None:
     """Removes files specified by provided list of file paths.
     Does nothing for a path that does not exist.

harness_utils/steam.py

@@ -3,6 +3,7 @@ import logging
 import winreg
 from subprocess import Popen
 from pathlib import Path
+import re
 
 
 def get_run_game_id_command(game_id: int) -> str:
@@ -79,3 +80,17 @@ def exec_steam_game(game_id: int, steam_path=None, game_params=None) -> Popen:
     command = [steam_path, "-applaunch", str(game_id)] + game_params
     logging.info(", ".join(command))
     return Popen(command)
+
+def get_build_id(game_id: int) -> str:
+    """Gets the build ID of a game from the Steam installation directory"""
+    game_folder = Path(get_steamapps_common_path()) / "../" / f"appmanifest_{game_id}.acf"
+    if not game_folder.exists():
+        logging.error("Game folder not found")
+        return None
+    with open(game_folder, 'r', encoding='utf-8') as file:
+        data = file.read()
+    buildid_match = re.search(r'"buildid"\s*"(\d+)"', data)
+    if buildid_match is not None:
+        return buildid_match.group(1)
+    logging.error("No 'buildid' found in the file")
+    return None

returnal/returnal.py

@@ -18,12 +18,14 @@ from harness_utils.output import (
     DEFAULT_LOGGING_FORMAT,
     DEFAULT_DATE_FORMAT,
 )
-from harness_utils.misc import remove_files
+from harness_utils.misc import remove_files, press_n_times
 from harness_utils.process import terminate_processes
 from harness_utils.steam import (
   exec_steam_run_command,
   get_steamapps_common_path,
+  get_build_id
 )
+from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 STEAM_GAME_ID = 1649240
 SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
@@ -90,6 +92,7 @@ def run_benchmark() -> tuple[float]:
     logging.info("Starting game")
     exec_steam_run_command(STEAM_GAME_ID)
     setup_start_time = time.time()
+    am = ArtifactManager(LOG_DIRECTORY)
 
     time.sleep(10)
 
@@ -105,8 +108,58 @@ def run_benchmark() -> tuple[float]:
         logging.info("Could not find prompt to open menu!")
         sys.exit(1)
 
-    # Navigate to in-game benchmark and start it
-    navigate_options_menu()
+    # Navigate to display menu
+    user.press("esc")
+    time.sleep(1)
+    user.press("enter")
+    time.sleep(1)
+    user.press("q")
+    time.sleep(1)
+    user.press("q")
+    time.sleep(1)
+
+    # Verify that we have navigated to the video settings menu and take a screenshot
+    if kerasService.wait_for_word(word="aspect", timeout=30, interval=1) is None:
+        logging.info("Did not find the video settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings")
+
+    # Navigate to graphics menu
+    user.press("e")
+    time.sleep(1)
+
+    if kerasService.wait_for_word(word="vsync", timeout=30, interval=1) is None:
+        logging.info("Did not find the graphics settings menu. Did the menu get stuck?")
+        sys.exit(1)
+    am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "first picture of graphics settings")
+
+    # We check for a keyword that indicates DLSS is active because this changes how we navigate the menu
+    if kerasService.wait_for_word(word="sharpness", timeout=10, interval=1) is None:
+        logging.info("No DLSS Settings Detected")
+        # Scroll down graphics menu
+        press_n_times("down", 15, 0.2)
+    else:
+        logging.info("DLSS Settings Detected")
+        # Scroll down graphics menu
+        press_n_times("down", 17, 0.2)
+
+    if kerasService.wait_for_word(word="volumetric", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'volumetric'. Did the the menu scroll correctly?")
+        sys.exit(1)
+    am.take_screenshot("graphics_2.png", ArtifactType.CONFIG_IMAGE, "second picture of graphics settings")
+
+    # Scroll down graphics menu
+    press_n_times("down", 15, 0.2)
+
+    if kerasService.wait_for_word(word="hdr", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'hdr'. Did the the menu scroll correctly?")
+        sys.exit(1)
+    am.take_screenshot("graphics_3.png", ArtifactType.CONFIG_IMAGE, "third picture of graphics settings")
+
+    # Launch the benchmark
+    user.keyDown("tab")
+    time.sleep(5)
+    user.keyUp("tab")
 
     setup_end_time = time.time()
     elapsed_setup_time = round((setup_end_time - setup_start_time), 2)
@@ -137,6 +190,11 @@ def run_benchmark() -> tuple[float]:
             "Results screen was not found! Did harness not wait long enough? Or test was too long?")
         sys.exit(1)
 
+    # Give results screen time to fill out, then save screenshot and config file
+    time.sleep(2)
+    am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of benchmark result")
+    am.copy_file(LOCAL_USER_SETTINGS, ArtifactType.CONFIG_TEXT, "config file")
+
     elapsed_test_time = round((test_end_time - test_start_time), 2)
     logging.info("Benchmark took %s seconds", elapsed_test_time)
 
@@ -164,7 +222,8 @@ try:
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
-        "end_time": seconds_to_milliseconds(end_time)
+        "end_time": seconds_to_milliseconds(end_time),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIRECTORY, "report.json", report)

total_war_pharaoh/twp.py

@@ -21,8 +21,10 @@ from harness_utils.output import (
     DEFAULT_LOGGING_FORMAT,
     DEFAULT_DATE_FORMAT
 )
-from harness_utils.steam import get_app_install_location
+from harness_utils.steam import get_app_install_location, get_build_id
 from harness_utils.keras_service import KerasService
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+from harness_utils.misc import mouse_scroll_n_times
 
 SCRIPT_DIR = Path(__file__).resolve().parent
 LOG_DIR = SCRIPT_DIR.joinpath("run")
@@ -78,8 +80,10 @@ def skip_logo_screens() -> None:
 
 def run_benchmark(keras_service):
     """Starts the benchmark"""
+    cfg = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}"
     start_game()
     setup_start_time = time.time()
+    am = ArtifactManager(LOG_DIR)
     time.sleep(5)
 
     result = keras_service.look_for_word("warning", attempts=10, interval=5)
@@ -101,6 +105,54 @@ def run_benchmark(keras_service):
     time.sleep(0.2)
     gui.mouseUp()
 
+    if keras_service.wait_for_word(word="brightness", timeout=30, interval=1) is None:
+        logging.info("Did not find the main menu. Did Keras click correctly?")
+        sys.exit(1)
+
+    am.take_screenshot("main.png", ArtifactType.CONFIG_IMAGE, "screenshot of main settings menu")
+    time.sleep(0.5)
+
+    result = keras_service.look_for_word("advanced", attempts=10, interval=1)
+    if not result:
+        logging.info("Did not find the advanced options menu. Did the game navigate to options correctly?")
+        sys.exit(1)
+
+    gui.moveTo(result["x"], result["y"])
+    time.sleep(0.2)
+    gui.mouseDown()
+    time.sleep(0.2)
+    gui.mouseUp()
+
+    if keras_service.wait_for_word(word="water", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'water' in the menu. Did Keras navigate to the advanced menu correctly?")
+        sys.exit(1)
+
+    am.take_screenshot("advanced_1.png", ArtifactType.CONFIG_IMAGE, "first screenshot of advanced settings menu")
+    time.sleep(0.5)
+
+    result = keras_service.look_for_word("water", attempts=10, interval=1)
+    if not result:
+        logging.info("Did not find the keyword 'water' in the menu. Did Keras navigate to the advanced menu correctly?")
+        sys.exit(1)
+    gui.moveTo(result["x"], result["y"])
+    time.sleep(1)
+
+    # Scroll to the middle of the advanced menu
+    mouse_scroll_n_times(15, -1, 0.1)
+    if keras_service.wait_for_word(word="heat", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'heat' in the menu. Did Keras scroll down the advanced menu far enough?")
+        sys.exit(1)
+    am.take_screenshot("advanced_2.png", ArtifactType.CONFIG_IMAGE, "second screenshot of advanced settings menu")
+    time.sleep(0.5)
+
+    # Scroll to the bottom of the advanced menu
+    mouse_scroll_n_times(15, -1, 0.1)
+    if keras_service.wait_for_word(word="bodies", timeout=30, interval=1) is None:
+        logging.info("Did not find the keyword 'bodies' in the menu. Did Keras scroll down the advanced menu far enough?")
+        sys.exit(1)
+    am.take_screenshot("advanced_3.png", ArtifactType.CONFIG_IMAGE, "third screenshot of advanced settings menu")
+    time.sleep(0.5)
+
     result = keras_service.look_for_word("bench", attempts=10, interval=1)
     if not result:
         logging.info("Did not find the benchmark menu. Did the game skip the intros?")
@@ -132,10 +184,12 @@ def run_benchmark(keras_service):
             "Results screen was not found! Did harness not wait long enough? Or test was too long?")
         sys.exit(1)
 
-    test_end_time = time.time() - 1
-
     # Wait 5 seconds for benchmark info
+    test_end_time = time.time() - 1
     time.sleep(5)
+    am.take_screenshot("results.png", ArtifactType.RESULTS_IMAGE, "benchmark results")
+    time.sleep(0.5)
+    am.copy_file(Path(cfg), ArtifactType.CONFIG_TEXT, "preferences.script.txt")
 
     # End the run
     elapsed_test_time = round(test_end_time - test_start_time, 2)
@@ -173,7 +227,8 @@ def main():
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
-        "end_time": seconds_to_milliseconds(endtime)
+        "end_time": seconds_to_milliseconds(endtime),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIR, "report.json", report)

total_war_warhammer_iii/twwh3.py

@@ -2,6 +2,7 @@
 from argparse import ArgumentParser
 import logging
 import os
+from pathlib import Path
 import time
 import sys
 import pyautogui as gui
@@ -19,14 +20,20 @@ from harness_utils.output import (
     DEFAULT_LOGGING_FORMAT,
     DEFAULT_DATE_FORMAT
 )
-from harness_utils.steam import get_app_install_location
+from harness_utils.steam import get_app_install_location, get_build_id
 from harness_utils.keras_service import KerasService
+from harness_utils.artifacts import ArtifactManager, ArtifactType
 
 SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
 LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
 PROCESS_NAME = "Warhammer3.exe"
 STEAM_GAME_ID = 1142710
 
+APPDATA = os.getenv("APPDATA")
+CONFIG_LOCATION = f"{APPDATA}\\The Creative Assembly\\Warhammer3\\scripts"
+CONFIG_FILENAME = "preferences.script.txt"
+CONFIG_FULL_PATH = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}"
+
 user.FAILSAFE = False
 
 
@@ -63,6 +70,7 @@ def run_benchmark():
     start_game()
     setup_start_time = time.time()
     time.sleep(5)
+    am = ArtifactManager(LOG_DIRECTORY)
 
     result = kerasService.look_for_word("warning", attempts=10, interval=5)
     if not result:
@@ -84,6 +92,8 @@ def run_benchmark():
     gui.mouseUp()
     time.sleep(2)
 
+    am.take_screenshot("main.png", ArtifactType.CONFIG_IMAGE, "picture of basic settings")
+
     result = kerasService.look_for_word("ad", attempts=10, interval=1)
     if not result:
         logging.info("Did not find the advanced menu. Did the game skip the intros?")
@@ -96,6 +106,8 @@ def run_benchmark():
     gui.mouseUp()
     time.sleep(0.5)
 
+    am.take_screenshot("advanced.png", ArtifactType.CONFIG_IMAGE, "picture of advanced settings")
+
     result = kerasService.look_for_word("bench", attempts=10, interval=1)
     if not result:
         logging.info("Did not find the benchmark menu. Did the game skip the intros?")
@@ -144,6 +156,9 @@ def run_benchmark():
     # Wait 5 seconds for benchmark info
     time.sleep(5)
 
+    am.take_screenshot("result.png", ArtifactType.RESULTS_IMAGE, "benchmark results")
+    am.copy_file(Path(CONFIG_FULL_PATH), ArtifactType.RESULTS_TEXT, "preferences.script.txt")
+
     # End the run
     elapsed_test_time = round(test_end_time - test_start_time, 2)
     logging.info("Benchmark took %f seconds", elapsed_test_time)
@@ -180,7 +195,8 @@ try:
     report = {
         "resolution": format_resolution(width, height),
         "start_time": seconds_to_milliseconds(start_time),
-        "end_time": seconds_to_milliseconds(endtime)
+        "end_time": seconds_to_milliseconds(endtime),
+        "version": get_build_id(STEAM_GAME_ID)
     }
 
     write_report_json(LOG_DIRECTORY, "report.json", report)