diff --git a/.gitignore b/.gitignore index 386cc3d..d91f24a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ flac-1.4.3-win.zip primesieve-12.3* y-cruncher v0.8.2.9522/ y-cruncher v0.8.2.9522.zip +y-cruncher v0.8.6.9545/ +y-cruncher.v0.8.6.9545b.zip basegame_no_intro_videos.archive *.blend 0001.png @@ -31,6 +33,8 @@ godot-4.2.1-stable/ godot-4.2.1-stable.zip godot-4.3-stable/ godot-4.3-stable.zip +godot-4.4.1-stable/ +godot-4.4.1-stable.zip mingw64/ # python diff --git a/3dmark/ul3dmark.py b/3dmark/ul3dmark.py index 21f8c85..e5eb82b 100644 --- a/3dmark/ul3dmark.py +++ b/3dmark/ul3dmark.py @@ -20,7 +20,7 @@ from harness_utils.output import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -31,25 +31,25 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "TimeSpy": { "config": CONFIG_DIR / "timespy.3dmdef", - "process_name": "3DMarkTimeSpy.exe", + "process_name": "3DMarkTimeSpy.exe", "score_name": "TimeSpyPerformanceGraphicsScore", "test_name": "3DMark Time Spy" }, "FireStrike": { "config": CONFIG_DIR / "firestrike.3dmdef", - "process_name": "3DMarkICFWorkload.exe", + "process_name": "3DMarkICFWorkload.exe", "score_name": "firestrikegraphicsscorep", "test_name": "3DMark Fire Strike" }, "PortRoyal": { "config": CONFIG_DIR / "portroyal.3dmdef", - "process_name": "3DMarkPortRoyal.exe", + "process_name": "3DMarkPortRoyal.exe", "score_name": "PortRoyalPerformanceGraphicsScore", "test_name": "3DMark Port Royal" }, "SolarBay": { "config": CONFIG_DIR / "solarbay.3dmdef", - "process_name": "3DMarkSolarBay.exe", + "process_name": "3DMarkSolarBay.exe", "score_name": "SolarBayPerformanceGraphicsScore", "test_name": "3DMark Solar Bay" } @@ -57,9 +57,10 @@ BENCHMARK_CONFIG = { RESULTS_FILENAME = "myresults.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -73,8 +74,9 @@ def setup_logging(): def get_arguments(): """get arguments""" parser = ArgumentParser() - parser.add_argument( - "--benchmark", dest="benchmark", help="Benchmark test type", required=True, choices=BENCHMARK_CONFIG.keys()) + parser.add_argument("--benchmark", dest="benchmark", + help="Benchmark test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -94,16 +96,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 30: #seconds + if elapsed >= 30: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() args = get_arguments() @@ -118,7 +121,9 @@ try: logging.error("3DMark exited with return code %d", pr.returncode) sys.exit(pr.returncode) - score = get_score(BENCHMARK_CONFIG[args.benchmark]["score_name"], REPORT_PATH) + score = get_score( + BENCHMARK_CONFIG[args.benchmark]["score_name"], + REPORT_PATH) if score is None: logging.error("Could not find average FPS output!") sys.exit(1) @@ -129,7 +134,8 @@ try: logging.info("Score was %s", score) report = { - "test": BENCHMARK_CONFIG[args.benchmark]["test_name"], + "test": "3DMark", + "test_parameter": args.benchmark, "unit": "score", "score": score, "start_time": seconds_to_milliseconds(strt), diff --git a/7z/sevenzip.py b/7z/sevenzip.py index 85dcdf7..a25b89a 100644 --- a/7z/sevenzip.py +++ b/7z/sevenzip.py @@ -26,7 +26,7 @@ formatter = logging.Formatter(LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger('').addHandler(console) -EXECUTABLE = "7zr_24.07.exe" +EXECUTABLE = "7zr_25.00.exe" ABS_EXECUTABLE_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), EXECUTABLE) @@ -35,18 +35,18 @@ if os.path.isfile(ABS_EXECUTABLE_PATH) is False: "7-Zip executable not found, downloading from network drive") copy_from_network_drive() -command = f'{ABS_EXECUTABLE_PATH}' -command = command.rstrip() +COMMAND = f'{ABS_EXECUTABLE_PATH}' +COMMAND = COMMAND.rstrip() t1 = time.time() logging.info("Starting 7-Zip benchmark! This may take a minute or so...") -with Popen([command, "b", "3"], cwd=os.path.dirname( +with Popen([COMMAND, "b", "3"], cwd=os.path.dirname( os.path.realpath(__file__)), stdout=subprocess.PIPE) as process: stdout_data, stderr = process.communicate() list_of_strings = stdout_data.decode('utf-8').splitlines() SPEED_PATTERN = r'^Avr:\s*([0-9]*)\s.*\|\s*([0-9]*)\s.*$' - VERSION_PATTERN = r'7-Zip (\d+\.\d+).*' + VERSION_PATTERN = r'7-Zip \(r\) (\d+\.\d+).*' VERSION = "" SPEED_C = "" diff --git a/7z/sevenzip_utils.py b/7z/sevenzip_utils.py index 32db283..5a37680 100644 --- a/7z/sevenzip_utils.py +++ b/7z/sevenzip_utils.py @@ -6,7 +6,7 @@ import shutil def copy_from_network_drive(): """Download 7zip from network drive""" - source = r"\\Labs\labs\01_Installers_Utilities\7ZIP\7zr_24.07.exe" + source = r"\\labs.lmg.gg\labs\01_Installers_Utilities\7ZIP\7zr_25.00.exe" root_dir = os.path.dirname(os.path.realpath(__file__)) - destination = os.path.join(root_dir, "7zr_24.07.exe") + destination = os.path.join(root_dir, "7zr_25.00.exe") shutil.copyfile(source, destination) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04319ce..4865aa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,20 @@ All notable changes to this project will be documented in this file. Changes are grouped by the date they are merged to the main branch of the repository and are ordered from newest to oldest. Dates use the ISO 8601 extended calendar date format, i.e. YYYY-MM-DD. +## 2025-07-15 +- Updated 7-Zip to 25.00 +- Updated Y-Cruncher to v0.8.6.9545 +- Updated Godot compile to 4.4.1-stable + ## 2025-04-02 - Fixed Keras not finding the FPS in Shadow of the Tomb Raider - Added a screenshot function for Vulkan games for Keras-OCR via DXcam - Added Keras functionality to Red Dead Redemption 2 - Added Strange Brigade (VK) to the team -- Updated PugetBench harness to include Davinci and After Effects +- Updated PugetBench harness to include DaVinci and After Effects - Updated PugetBench to more consistently find version numbers and include them in the report.json - Updated Rocket League harness to check what camera it is on and keep flipping through till it's on the correct one -- Updated Procyon AI harnesses to have verison numbers in report.json +- Updated Procyon AI harnesses to have version numbers in report.json - Replaced the hardcoded path for Cyberpunk2077 and instead use the get_app_install_location instead - Added DOTA 2 screenshotting for video config - Added beta harness of Marvel Rivals diff --git a/F1_24/f1_24.py b/F1_24/f1_24.py index 914c111..a25e1f8 100644 --- a/F1_24/f1_24.py +++ b/F1_24/f1_24.py @@ -40,6 +40,7 @@ intro_videos = [ os.path.join(VIDEO_PATH, "attract.bk2"), os.path.join(VIDEO_PATH, "cm_f1_sting.bk2") ] +user.FAILSAFE = False def find_latest_result_file(base_path): diff --git a/alanwake2/alanwake2.py b/alanwake2/alanwake2.py index fbfbeae..303f196 100644 --- a/alanwake2/alanwake2.py +++ b/alanwake2/alanwake2.py @@ -24,8 +24,9 @@ LOG_DIRECTORY = SCRIPT_DIRECTORY.joinpath("run") PROCESS_NAME = "alanwake2.exe" EXECUTABLE_PATH = find_epic_executable() GAME_ID = "c4763f236d08423eb47b4c3008779c84%3A93f2a8c3547846eda966cb3c152a026e%3Adc9d2e595d0e4650b35d659f90d41059?action=launch&silent=true" -gamefoldername = "AlanWake2" +GAMEFOLDERNAME = "AlanWake2" +user.FAILSAFE = False def setup_logging(): """default logging config""" @@ -169,7 +170,7 @@ try: "resolution": f"{width}x{height}", "start_time": round((start_time * 1000)), "end_time": round((end_time * 1000)), - "game_version": find_eg_game_version(gamefoldername) + "game_version": find_eg_game_version(GAMEFOLDERNAME) } am.create_manifest() diff --git a/aots-e/aotse.py b/aots-e/aotse.py index 1bd85e5..13bf0e1 100644 --- a/aots-e/aotse.py +++ b/aots-e/aotse.py @@ -6,13 +6,13 @@ import time import getpass import glob import os -from aotse_utils import read_current_resolution, find_score_in_log, delete_old_scores, get_args +from aotse_utils import read_current_resolution, find_score_in_log, delete_old_scores, get_args, replace_exe, restore_exe PARENT_DIR = str(Path(sys.path[0], "..")) sys.path.append(PARENT_DIR) from harness_utils.keras_service import KerasService -from harness_utils.steam import get_app_install_location, get_build_id, exec_steam_game +from harness_utils.steam import get_build_id, exec_steam_game from harness_utils.output import ( DEFAULT_DATE_FORMAT, DEFAULT_LOGGING_FORMAT, @@ -32,7 +32,7 @@ CONFIG_FILENAME = "settings.ini" STEAM_GAME_ID = 507490 SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" -EXECUTABLE = "AshesEscalation_DX12.exe" +EXECUTABLE = "StardockLauncher.exe" CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "GPU_Benchmark": { @@ -49,7 +49,6 @@ BENCHMARK_CONFIG = { } } CFG = f"{CONFIG_PATH}\\{CONFIG_FILENAME}" -GAME_DIR = get_app_install_location(STEAM_GAME_ID) def start_game(): """Launch the game with no launcher or start screen""" @@ -60,6 +59,7 @@ def run_benchmark(): """Start the benchmark""" # Start game via Steam and enter fullscreen mode setup_start_time = time.time() + replace_exe() start_game() time.sleep(10) @@ -85,16 +85,14 @@ def run_benchmark(): logging.info("Benchmark started. Waiting for benchmark to complete.") time.sleep(180) - # result = kerasService.wait_for_word("complete", timeout=240, interval=0.5) - # if not result: - # logging.info("Did not see the Benchmark Complete pop up. Did it run?") - # sys.exit(1) - test_end_time = time.time() - 2 + test_end_time = time.time() time.sleep(2) elapsed_test_time = round((test_end_time - test_start_time), 2) logging.info("Benchmark took %f seconds", elapsed_test_time) time.sleep(3) + restore_exe() + return test_start_time, test_end_time setup_log_directory(LOG_DIR) diff --git a/aots-e/aotse_utils.py b/aots-e/aotse_utils.py index 79e6820..b2a32a3 100644 --- a/aots-e/aotse_utils.py +++ b/aots-e/aotse_utils.py @@ -8,6 +8,7 @@ from pathlib import Path import psutil import glob import time +import shutil from argparse import ArgumentParser PARENT_DIR = str(Path(sys.path[0], "..")) @@ -15,14 +16,16 @@ sys.path.append(PARENT_DIR) sys.path.insert(1, os.path.join(sys.path[0], '..')) +from harness_utils.steam import get_app_install_location + USERNAME = getpass.getuser() SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") -PROCESS_NAME = "stellaris.exe" -STEAM_GAME_ID = 281990 +STEAM_GAME_ID = 507490 CONFIG_FILENAME = "settings.ini" USERNAME = getpass.getuser() CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Ashes of the Singularity - Escalation") +EXE_PATH = Path(get_app_install_location(STEAM_GAME_ID)) BENCHMARK_CONFIG = { "GPU_Benchmark": { "hardware": "GPU", @@ -119,3 +122,36 @@ def wait_for_benchmark_process(test_name, process_name, timeout=60): # Wait for 1 second before checking again time.sleep(1) + +def replace_exe(): + """Replaces the Strange Brigade launcher exe with the Vulkan exe for immediate launching + """ + check_backup = Path(f"{EXE_PATH}\\StardockLauncher_launcher.exe") + launcher_exe = Path(f"{EXE_PATH}\\StardockLauncher.exe") + dx12_exe = Path(f"{EXE_PATH}\\AshesEscalation_DX12.exe") + if not os.path.exists(check_backup): + os.rename(launcher_exe, check_backup) + shutil.copy(dx12_exe, launcher_exe) + logging.info("Replacing launcher file in %s", EXE_PATH) + elif os.path.exists(check_backup): + if not os.path.exists(launcher_exe): + shutil.copy(dx12_exe, launcher_exe) + logging.info("Replacing launcher file in %s", EXE_PATH) + else: + logging.info("Launcher already replaced with DX12 exe.") + +def restore_exe(): + """Restores the launcher exe back to the original exe name to close the loop. + """ + check_backup = Path(f"{EXE_PATH}\\StardockLauncher_launcher.exe") + launcher_exe = Path(f"{EXE_PATH}\\StardockLauncher.exe") + if not os.path.exists(check_backup): + logging.info("Launcher already restored or file does not exist.") + elif os.path.exists(check_backup): + if not os.path.exists(launcher_exe): + os.rename(check_backup, launcher_exe) + logging.info("Restoring launcher file in %s", EXE_PATH) + else: + os.remove(launcher_exe) + os.rename(check_backup, launcher_exe) + logging.info("Restoring launcher file in %s", EXE_PATH) diff --git a/aots-e/manifest.yaml b/aots-e/manifest.yaml index 04e505b..2612f7a 100644 --- a/aots-e/manifest.yaml +++ b/aots-e/manifest.yaml @@ -1,6 +1,6 @@ friendly_name: "Ashes of the Singularity: Escalation" executable: "aotse.py" -process_name: "AshesEscalation_DX12.exe" +process_name: "StardockLauncher.exe" output_dir: "run" options: - name: kerasHost diff --git a/assassins_creed_shadows/acshadows.py b/assassins_creed_shadows/acshadows.py index 8ae6f5a..541a0e5 100644 --- a/assassins_creed_shadows/acshadows.py +++ b/assassins_creed_shadows/acshadows.py @@ -1,39 +1,42 @@ -#pylint: disable=missing-module-docstring -from argparse import ArgumentParser +# pylint: disable=missing-module-docstring import logging -import os from pathlib import Path import time import sys import re import pydirectinput as user +import getpass +sys.path.insert(1, str(Path(sys.path[0]).parent)) -sys.path.insert(1, os.path.join(sys.path[0], '..')) - -#pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position from harness_utils.process import terminate_processes from harness_utils.output import ( format_resolution, - setup_log_directory, + setup_logging, write_report_json, seconds_to_milliseconds, - DEFAULT_LOGGING_FORMAT, - DEFAULT_DATE_FORMAT ) from harness_utils.steam import get_build_id, exec_steam_game from harness_utils.keras_service import KerasService from harness_utils.artifacts import ArtifactManager, ArtifactType -from harness_utils.misc import press_n_times +from harness_utils.misc import ( + press_n_times, + int_time, + find_word, + keras_args) -SCRIPT_DIR = Path(__file__).resolve().parent -LOG_DIR = SCRIPT_DIR.joinpath("run") -PROCESS_NAME = "ACShadows.exe" +USERNAME = getpass.getuser() STEAM_GAME_ID = 3159330 -CONFIG_LOCATION = "C:\\Users\\Administrator\\Documents\\Assassin's Creed Shadows" +SCRIPT_DIR = Path(__file__).resolve().parent +LOG_DIR = SCRIPT_DIR / "run" +PROCESS_NAME = "ACShadows.exe" + +CONFIG_LOCATION = f"C:\\Users\\{USERNAME}\\Documents\\Assassin's Creed Shadows" CONFIG_FILENAME = "ACShadows.ini" user.FAILSAFE = False + def read_current_resolution(): """Reads resolutions settings from local game file""" height_pattern = re.compile(r"FullscreenWidth=(\d+)") @@ -52,115 +55,132 @@ def read_current_resolution(): width_value = width_match.group(1) return (height_value, width_value) -def find_word(keras_service, word, msg, timeout = 30, interval = 1): - """function to call keras """ - if keras_service.wait_for_word(word = word, timeout = timeout, interval = interval) is None: - logging.info(msg) - sys.exit(1) -def int_time(): - """rounds time to int""" - return int(time.time()) - def delete_videos(): """deletes intro videos""" - base_dir = r"C:\Program Files (x86)\Steam\steamapps\common\Assassin's Creed Shadows" - videos_dir = os.path.join(base_dir, "videos") - videos_en_dir = os.path.join(videos_dir, "en") + base_dir = Path( + r"C:\Program Files (x86)\Steam\steamapps\common\Assassin's Creed Shadows") + videos_dir = base_dir / "videos" + videos_en_dir = videos_dir / "en" # List of video files to delete videos_to_delete = [ - os.path.join(videos_dir, "ANVIL_Logo.webm"), - os.path.join(videos_dir, "INTEL_Logo.webm"), - os.path.join(videos_dir, "HUB_Bootflow_FranchiseIntro.webm"), - os.path.join(videos_dir, "UbisoftLogo.webm"), - os.path.join(videos_en_dir, "Epilepsy.webm"), - os.path.join(videos_en_dir, "warning_disclaimer.webm"), - os.path.join(videos_en_dir, "WarningSaving.webm") + videos_dir / "ANVIL_Logo.webm", + videos_dir / "INTEL_Logo.webm", + videos_dir / "HUB_Bootflow_FranchiseIntro.webm", + videos_dir / "HUB_Bootflow_AbstergoIntro.webm", + videos_dir / "UbisoftLogo.webm", + videos_en_dir / "Epilepsy.webm", + videos_en_dir / "warning_disclaimer.webm", + videos_en_dir / "WarningSaving.webm" ] for file_path in videos_to_delete: - if os.path.exists(file_path): + if file_path.exists(): try: - os.remove(file_path) - logging.info("Deleted: %f", file_path) + file_path.unlink() + logging.info("Deleted: %s", file_path) except Exception as e: - logging.error("Error deleting %f: %e", file_path, e) + logging.error("Error deleting %s: %s", file_path, e) + def move_benchmark_file(): """moves html benchmark results to log folder""" - src_dir = r"C:\Users\Administrator\Documents\Assassin's Creed Shadows\benchmark_reports" + src_dir = Path( + f"C:\\Users\\{USERNAME}\\Documents\\Assassin's Creed Shadows\\benchmark_reports") - for filename in os.listdir(src_dir): - src_path = os.path.join(src_dir, filename) - dest_path = os.path.join(LOG_DIR, filename) + for src_path in src_dir.iterdir(): + dest_path = LOG_DIR / src_path.name - if os.path.isfile(src_path): + if src_path.is_file(): try: - os.rename(src_path, dest_path) + src_path.rename(dest_path) logging.info("Benchmark HTML moved") except Exception as e: - logging.error("Failed to move %s: %e", src_path, e) + logging.error("Failed to move %s: %s", src_path, e) else: logging.error("Benchmark HTML not found.") + def start_game(): """Starts the game process""" exec_steam_game(STEAM_GAME_ID) logging.info("Launching Game from Steam") + def navi_settings(am): """navigates and takes pictures of settings""" user.press("space") time.sleep(1) - am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "display settings 1") + am.take_screenshot( + "display1.png", ArtifactType.CONFIG_IMAGE, "display settings 1") press_n_times("down", 13, 0.3) - am.take_screenshot("display2.png", ArtifactType.CONFIG_IMAGE, "display settings 2") + am.take_screenshot( + "display2.png", ArtifactType.CONFIG_IMAGE, "display settings 2") press_n_times("down", 4, 0.3) - am.take_screenshot("display3.png", ArtifactType.CONFIG_IMAGE, "display settings 3") + am.take_screenshot( + "display3.png", ArtifactType.CONFIG_IMAGE, "display settings 3") user.press("c") time.sleep(1) - am.take_screenshot("scalability1.png", ArtifactType.CONFIG_IMAGE, "scalability settings 1") + am.take_screenshot( + "scalability1.png", ArtifactType.CONFIG_IMAGE, + "scalability settings 1") press_n_times("down", 10, 0.3) - am.take_screenshot("scalability2.png", ArtifactType.CONFIG_IMAGE, "scalability settings 2") + am.take_screenshot( + "scalability2.png", ArtifactType.CONFIG_IMAGE, + "scalability settings 2") press_n_times("down", 6, 0.3) - am.take_screenshot("scalability3.png", ArtifactType.CONFIG_IMAGE, "scalability settings 3") + am.take_screenshot( + "scalability3.png", ArtifactType.CONFIG_IMAGE, + "scalability settings 3") press_n_times("down", 5, 0.3) - am.take_screenshot("scalability4.png", ArtifactType.CONFIG_IMAGE, "scalability settings 4") + am.take_screenshot( + "scalability4.png", ArtifactType.CONFIG_IMAGE, + "scalability settings 4") user.press("esc") + def run_benchmark(keras_service): """runs the benchmark""" delete_videos() start_game() setup_start_time = int_time() am = ArtifactManager(LOG_DIR) - time.sleep(20) + time.sleep(15) - if keras_service.wait_for_word(word="animus", timeout=30, interval = 1) is None: + if keras_service.wait_for_word( + word="hardware", timeout=30, interval=1) is None: + logging.info("did not find hardware") + else: + user.mouseDown() + time.sleep(0.2) + user.press("space") + + if keras_service.wait_for_word( + word="animus", timeout=130, interval=1) is None: logging.info("did not find main menu") sys.exit(1) user.press("f1") - find_word(keras_service, "system", "couldn't find system") + find_word(keras_service, "system", "Couldn't find 'System' button") user.press("down") @@ -168,10 +188,16 @@ def run_benchmark(keras_service): user.press("space") - find_word(keras_service, "benchmark", "couldn't find benchmark") + find_word( + keras_service, "benchmark", + "couldn't find 'benchmark' on screen before settings") navi_settings(am) + find_word( + keras_service, "benchmark", + "couldn't find 'benchmark' on screen after settings") + user.press("down") time.sleep(1) @@ -180,9 +206,10 @@ def run_benchmark(keras_service): setup_end_time = int_time() elapsed_setup_time = setup_end_time - setup_start_time - logging.info("Setup took %f seconds", elapsed_setup_time) + logging.info("Setup took %d seconds", elapsed_setup_time) - if keras_service.wait_for_word(word = "benchmark", timeout = 30, interval = 1) is None: + if keras_service.wait_for_word( + word="benchmark", timeout=50, interval=1) is None: logging.info("did not find benchmark") sys.exit(1) @@ -190,16 +217,16 @@ def run_benchmark(keras_service): time.sleep(100) - if keras_service.wait_for_word(word = "results", timeout = 30, interval = 1) is None: - logging.info("did not find end screen") - sys.exit(1) + find_word(keras_service, "results", "did not find results screen", 60) - test_end_time = int_time() + test_end_time = int_time() - 2 elapsed_test_time = test_end_time - test_start_time - logging.info("Benchmark took %f seconds", elapsed_test_time) + logging.info("Benchmark took %d seconds", elapsed_test_time) - am.take_screenshot("benchmark_results.png", ArtifactType.RESULTS_IMAGE, "benchmark results") + am.take_screenshot( + "benchmark_results.png", ArtifactType.RESULTS_IMAGE, + "benchmark results") user.press("x") @@ -217,28 +244,11 @@ def run_benchmark(keras_service): return test_start_time, test_end_time -def setup_logging(): - """setup logging""" - setup_log_directory(LOG_DIR) - logging.basicConfig(filename=f'{LOG_DIR}/harness.log', - format=DEFAULT_LOGGING_FORMAT, - datefmt=DEFAULT_DATE_FORMAT, - level=logging.DEBUG) - console = logging.StreamHandler() - formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - def main(): """entry point""" - parser = ArgumentParser() - parser.add_argument("--kerasHost", dest="keras_host", - help="Host for Keras OCR service", required=True) - parser.add_argument("--kerasPort", dest="keras_port", - help="Port for Keras OCR service", required=True) - args = parser.parse_args() - keras_service = KerasService(args.keras_host, args.keras_port) + keras_service = KerasService( + keras_args().keras_host, keras_args().keras_port) start_time, endtime = run_benchmark(keras_service) height, width = read_current_resolution() report = { @@ -249,9 +259,10 @@ def main(): } write_report_json(LOG_DIR, "report.json", report) + if __name__ == "__main__": try: - setup_logging() + setup_logging(LOG_DIR) main() except Exception as ex: logging.error("Something went wrong running the benchmark!") diff --git a/blender_render/blender.py b/blender_render/blender.py index 3b8f6b5..63b7e3a 100644 --- a/blender_render/blender.py +++ b/blender_render/blender.py @@ -3,17 +3,17 @@ from pathlib import Path from blender_utils import BENCHMARK_CONFIG, find_blender, run_blender_render, download_scene from argparse import ArgumentParser import logging -import os.path import sys import time -sys.path.insert(1, os.path.join(sys.path[0], '..')) +sys.path.insert(1, str(Path(sys.path[0]).parent)) from harness_utils.output import DEFAULT_DATE_FORMAT, DEFAULT_LOGGING_FORMAT, write_report_json, seconds_to_milliseconds SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") + def setup_logging(): """default logging config""" LOG_DIR.mkdir(exist_ok=True) @@ -26,15 +26,18 @@ def setup_logging(): console.setFormatter(formatter) logging.getLogger('').addHandler(console) + VALID_DEVICES = ["CPU", "CUDA", "OPTIX", "HIP", "ONEAPI", "METAL"] + def main(): """entry point for test script""" parser = ArgumentParser() parser.add_argument("-d", "--device", dest="device", help="device", metavar="device", required=True) parser.add_argument( - "--benchmark", dest="benchmark", help="Benchmark test type", metavar="benchmark", required=True) + "--benchmark", dest="benchmark", help="Benchmark test type", + metavar="benchmark", required=True) args = parser.parse_args() if args.device not in VALID_DEVICES: raise Exception(f"invalid device selection: {args.device}") @@ -49,23 +52,25 @@ def main(): score = run_blender_render( executable_path, LOG_DIR, args.device.upper(), benchmark) end_time = time.time() - logging.info(f'Finished rendering {args.benchmark} in %d seconds', (end_time - start_time)) + logging.info( + f'Finished rendering {args.benchmark} in %d seconds', + (end_time - start_time)) if score is None: raise Exception("no duration was found in the log to use as the score") report = { - "test": f"Blender {args.benchmark} Render {args.device.upper()}", + "test": "Blender Render", + "test_parameter": args.benchmark, "score": score, "unit": "seconds", "version": version, "device": args.device, - "benchmark": args.benchmark, "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) if __name__ == "__main__": diff --git a/blender_render/blender_utils.py b/blender_render/blender_utils.py index c341915..3f60bd0 100644 --- a/blender_render/blender_utils.py +++ b/blender_render/blender_utils.py @@ -80,7 +80,7 @@ def download_scene(scene: BlenderScene) -> None: def copy_scene_from_network_drive(file_name, destination): """copy blend file from network drive""" - network_dir = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Blender Render") + network_dir = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Blender Render") source_path = network_dir.joinpath(file_name) logging.info("Copying %s from %s", file_name, source_path) shutil.copyfile(source_path, destination) @@ -97,16 +97,17 @@ def time_to_seconds(time_string): return seconds -def run_blender_render(executable_path: Path, log_directory: Path, device: str, benchmark: BlenderScene) -> str: +def run_blender_render(executable_path: Path, log_directory: Path, device: str, + benchmark: BlenderScene) -> str: """Execute the blender render of barbershop, returns the duration as string""" blend_log = log_directory.joinpath("blender.log") blend_path = SCRIPT_DIR.joinpath(benchmark.file_name) cmd_line = f'"{str(executable_path)}" -b -E CYCLES -y "{str(blend_path)}" -f 1 -- --cycles-device {device} --cycles-print-stats' - with open(blend_log,'w' , encoding="utf-8") as f_obj: + with open(blend_log, 'w', encoding="utf-8") as f_obj: subprocess.run(cmd_line, stdout=f_obj, text=True, check=True) # example: Time: 02:59.57 (Saving: 00:00.16) - time_regex = r"Time: (.*) \(Saving.*\)" + time_regex = r".*Time:\s+([\d:.]+)\s+\(Saving.*\)" time = None with open(blend_log, 'r', encoding="utf-8") as file: diff --git a/blenderbenchmark/blender.py b/blenderbenchmark/blender.py index 99c7dd2..9886685 100644 --- a/blenderbenchmark/blender.py +++ b/blenderbenchmark/blender.py @@ -117,10 +117,12 @@ for report in json_array: scene_report = { "timestamp": report['timestamp'], "version": blender_version, - "test": f"Blender Benchmark {report['scene']['label']} {DEVICE_TYPE}", + "test": "Blender Benchmark", + "test_parameter": f"{report['scene']['label']} ", "score": round(report['stats']['samples_per_minute'], 2), "unit": "samples per minute", - "device": report['device_info']['compute_devices'][0]['name'] + "device": report['device_info']['compute_devices'][0]['name'], + "device_type": DEVICE_TYPE, } logging.info(json.dumps(scene_report, indent=2)) diff --git a/cinebench_2024/cinebench.py b/cinebench_2024/cinebench.py index e94d705..24d1aaa 100644 --- a/cinebench_2024/cinebench.py +++ b/cinebench_2024/cinebench.py @@ -33,7 +33,8 @@ DURATION_OPTION = "g_CinebenchMinimumTestDuration=1" parser = ArgumentParser() parser.add_argument( - "-t", "--test", dest="test", help="Cinebench test type", required=True, choices=TEST_OPTIONS.keys()) + "-t", "--test", dest="test", help="Cinebench test type", required=True, + choices=TEST_OPTIONS.keys()) args = parser.parse_args() script_dir = Path(__file__).resolve().parent @@ -63,21 +64,30 @@ try: stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, - universal_newlines=True) as proc: - logging.info("Cinebench started. Waiting for setup to finish to set process priority.") + universal_newlines=True) as proc: + logging.info( + "Cinebench started. Waiting for setup to finish to set process priority.") + START_TIME = 0 + if proc.stdout is None: + logging.error("Cinebench process did not start correctly!") + sys.exit(1) for line in proc.stdout: if "BEFORERENDERING" in line: - elapsed_setup_time = round(time.time() - setup_start_time, 2) + elapsed_setup_time = round( + time.time() - setup_start_time, 2) logging.info("Setup took %.2f seconds", elapsed_setup_time) - logging.info("Setting Cinebench process priority to high (PID: %s)", proc.pid) + logging.info( + "Setting Cinebench process priority to high (PID: %s)", + proc.pid) process = psutil.Process(proc.pid) process.nice(psutil.HIGH_PRIORITY_CLASS) - start_time = time.time() + START_TIME = time.time() break out, _ = proc.communicate() if proc.returncode > 0: - logging.warning("Cinebench exited with return code %d", proc.returncode) + logging.warning( + "Cinebench exited with return code %d", proc.returncode) score = get_score(out) if score is None: @@ -85,19 +95,20 @@ try: sys.exit(1) end_time = time.time() - elapsed_test_time = round(end_time - start_time, 2) + elapsed_test_time = round(end_time - START_TIME, 2) logging.info("Benchmark took %.2f seconds", elapsed_test_time) report = { - "test": friendly_test_name(test_type), + "test": "Cinebench 2024", + "test_parameter": friendly_test_name(test_type), "score": score, "unit": "score", - "start_time": seconds_to_milliseconds(start_time), + "start_time": seconds_to_milliseconds(START_TIME), "end_time": seconds_to_milliseconds(end_time) } session_report.append(report) - write_report_json(log_dir, "report.json", session_report) + write_report_json(str(log_dir), "report.json", session_report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/cities_skylines_2/citiesskylines2.py b/cities_skylines_2/citiesskylines2.py index cb1aef3..e22b36c 100644 --- a/cities_skylines_2/citiesskylines2.py +++ b/cities_skylines_2/citiesskylines2.py @@ -20,7 +20,7 @@ from harness_utils.output import ( DEFAULT_DATE_FORMAT ) from harness_utils.steam import exec_steam_game, get_build_id -from harness_utils.keras_service import KerasService +from harness_utils.keras_service import KerasService, ScreenSplitConfig, ScreenShotDivideMethod, ScreenShotQuadrant from harness_utils.artifacts import ArtifactManager, ArtifactType from harness_utils.misc import mouse_scroll_n_times @@ -29,6 +29,10 @@ SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") PROCESS_NAME = "cities2.exe" STEAM_GAME_ID = 949230 +top_left_keras = ScreenSplitConfig( + divide_method=ScreenShotDivideMethod.QUADRANT, + quadrant=ScreenShotQuadrant.TOP_LEFT) + launcher_files = [ "bootstrapper-v2.exe", "launcher.exe", @@ -39,7 +43,6 @@ save_files = [ "Benchmark.cok.cid" ] config_files = [ - "continue_game.json", "UserState.coc" ] @@ -89,12 +92,41 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word("paradox", interval=0.5, timeout=100) if not result: - logging.info("Could not find the paused notification. Unable to mark start time!") + logging.info("Could not find the Paradox logo. Did the game launch?") sys.exit(1) user.press("esc") user.press("esc") user.press("esc") - time.sleep(20) + time.sleep(15) + + result = keras_service.wait_for_word("new", interval=0.5, timeout=100) + if not result: + logging.info("Did not find the main menu. Did the game crash?") + sys.exit(1) + + result = keras_service.look_for_word("load", attempts=10, interval=1) + if not result: + logging.info("Did not find the load game option. Did the save game copy?") + sys.exit(1) + + # Navigate to load save menu + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.click() + time.sleep(0.2) + + result = keras_service.look_for_word("benchmark", attempts=10, interval=1, split_config=top_left_keras) + if not result: + logging.info("Did not find the save game original date. Did the keras click correctly?") + sys.exit(1) + + # Loading the game + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.click() + time.sleep(0.2) + user.press("enter") + time.sleep(10) result = keras_service.wait_for_word("grand", interval=0.5, timeout=100) if not result: @@ -102,6 +134,8 @@ def run_benchmark(keras_service): sys.exit(1) elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) logging.info("Setup took %f seconds", elapsed_setup_time) + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) time.sleep(2) logging.info('Starting benchmark') user.press("3") diff --git a/cities_skylines_2/citiesskylines2_utils.py b/cities_skylines_2/citiesskylines2_utils.py index bc345b2..ac330fe 100644 --- a/cities_skylines_2/citiesskylines2_utils.py +++ b/cities_skylines_2/citiesskylines2_utils.py @@ -18,7 +18,7 @@ LAUNCHCONFIG_LOCATION = Path(f"{LOCALAPPDATA}\\Paradox Interactive") INSTALL_LOCATION = Path(get_app_install_location(STEAM_GAME_ID)) APPDATA = os.getenv("APPDATA") CONFIG_LOCATION = Path(f"{APPDATA}\\..\\LocalLow\\Colossal Order\\Cities Skylines II") -SAVE_LOCATION = Path(f"{CONFIG_LOCATION}\\Saves") +SAVE_LOCATION = Path(f"{CONFIG_LOCATION}\\Saves\\76561199517889423") CONFIG_FILENAME = "launcher-settings.json" @@ -37,7 +37,7 @@ def read_current_resolution(): def copy_continuegame(config_files: list[str]) -> None: - """Copy launcher files to game directory""" + """Copy continue game files to config directory""" for file in config_files: try: src_path = SCRIPT_DIRECTORY / "config" / file @@ -84,7 +84,7 @@ def copy_launcherpath(): shutil.copy(src_path, dest_path) #os.chmod(dest_path, stat.S_IREAD) except OSError as err: - logging.error("Could not copy the launcherpath file. %s", e) + logging.error("Could not copy the launcherpath file. %s", err) raise err @@ -98,5 +98,5 @@ def copy_benchmarksave(save_files: list[str]) -> None: logging.info("Copying: %s -> %s", file, dest_path) shutil.copy(src_path, dest_path) except OSError as err: - logging.error("Could not copy launcher files. %s", err) + logging.error("Could not copy the save game. %s", err) raise err diff --git a/cities_skylines_2/config/UserState.coc b/cities_skylines_2/config/UserState.coc index 41356e4..a5d6789 100644 --- a/cities_skylines_2/config/UserState.coc +++ b/cities_skylines_2/config/UserState.coc @@ -1,5 +1,5 @@ User Settings { - "lastSaveGameMetadata": "4d6c3ccb7ecaebb43efcf0913bb053b0", + "lastSaveGameMetadata": "e284bbf6f86bd366ca930783985b849c", "naturalDisasters": false } diff --git a/cities_skylines_2/config/continue_game.json b/cities_skylines_2/config/continue_game.json deleted file mode 100644 index 4ee682e..0000000 --- a/cities_skylines_2/config/continue_game.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "title": "Benchmark", - "desc": "Population: 99766 Money: \u00a280542653", - "date": "2024-08-20T14:35:47", - "rawGameVersion": "1.1.7f1" -} \ No newline at end of file diff --git a/cities_skylines_2/launcher/notlauncher-options.json b/cities_skylines_2/launcher/notlauncher-options.json index 47733d5..fecac5f 100644 --- a/cities_skylines_2/launcher/notlauncher-options.json +++ b/cities_skylines_2/launcher/notlauncher-options.json @@ -1,5 +1,5 @@ { - "continuelastsave": true, + "continuelastsave": false, "noworkshop": false, "disablemods": false, "nolog": false, diff --git a/cities_skylines_2/save/Benchmark.cok b/cities_skylines_2/save/Benchmark.cok index 17c3cac..9c6aa18 100644 Binary files a/cities_skylines_2/save/Benchmark.cok and b/cities_skylines_2/save/Benchmark.cok differ diff --git a/cities_skylines_2/save/Benchmark.cok.cid b/cities_skylines_2/save/Benchmark.cok.cid index 5785f84..47d8eb0 100644 --- a/cities_skylines_2/save/Benchmark.cok.cid +++ b/cities_skylines_2/save/Benchmark.cok.cid @@ -1 +1 @@ -9a03f3dfd36a585d8397c021d92d4184 \ No newline at end of file +582572b506bb32028036437e27f475ae \ No newline at end of file diff --git a/counterstrike2/cs2.py b/counterstrike2/cs2.py index 6401bc1..e9f63c7 100644 --- a/counterstrike2/cs2.py +++ b/counterstrike2/cs2.py @@ -29,7 +29,14 @@ PROCESS_NAME = "cs2.exe" STEAM_GAME_ID = 730 STEAM_USER_ID = get_registry_active_user() -cfg = Path(get_steam_folder_path(), "userdata", str(STEAM_USER_ID), str(STEAM_GAME_ID), "local", "cfg", "cs2_video.txt") +cfg = Path( + get_steam_folder_path(), + "userdata", str(STEAM_USER_ID), + str(STEAM_GAME_ID), + "local", "cfg", "cs2_video.txt") + +user.FAILSAFE = False + def setup_logging(): """default logging config""" @@ -66,7 +73,7 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word("play", timeout=30, interval=0.1) if not result: logging.info("Did not find the play menu. Did the game load?") - sys.exit(1) + raise RuntimeError height, width = get_resolution() location = None @@ -74,14 +81,25 @@ def run_benchmark(keras_service): # We check the resolution so we know which screenshot to use for the locate on screen function match width: case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1080.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_1080.png", minSearchTime=5, confidence=0.6) case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1440.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_1440.png", minSearchTime=5, confidence=0.6) case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_2160.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_2160.png", minSearchTime=5, confidence=0.6) case _: - logging.error("Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) + logging.error( + "Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", + height, width) + raise RuntimeError + + if location is None: + logging.error( + "Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", + height, width) + raise RuntimeError click_me = gui.center(location) gui.moveTo(click_me.x, click_me.y) @@ -93,7 +111,7 @@ def run_benchmark(keras_service): result = keras_service.look_for_word(word="video", attempts=10, interval=1) if not result: logging.info("Did not find the video menu button. Did Keras enter settings correctly?") - sys.exit(1) + raise RuntimeError gui.moveTo(result["x"], result["y"]) gui.mouseDown() @@ -103,14 +121,14 @@ def run_benchmark(keras_service): if keras_service.wait_for_word(word="brightness", timeout=30, interval=1) is None: logging.info("Did not find the video settings menu. Did the menu get stuck?") - sys.exit(1) + raise RuntimeError am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") result = keras_service.look_for_word(word="advanced", attempts=10, interval=1) if not result: logging.info("Did not find the advanced video menu. Did Keras click correctly?") - sys.exit(1) + raise RuntimeError gui.moveTo(result["x"], result["y"]) gui.mouseDown() @@ -118,12 +136,14 @@ def run_benchmark(keras_service): gui.mouseUp() time.sleep(0.2) - am.take_screenshot("advanced_video_1.png", ArtifactType.CONFIG_IMAGE, "first picture of advanced video settings") + am.take_screenshot("advanced_video_1.png", ArtifactType.CONFIG_IMAGE, + "first picture of advanced video settings") result = keras_service.look_for_word(word="boost", attempts=10, interval=1) if not result: - logging.info("Did not find the keyword 'Boost' in the advanced video menu. Did Keras click correctly?") - sys.exit(1) + logging.info( + "Did not find the keyword 'Boost' in the advanced video menu. Did Keras click correctly?") + raise RuntimeError gui.moveTo(result["x"], result["y"]) time.sleep(1) @@ -131,9 +151,11 @@ def run_benchmark(keras_service): time.sleep(1) if keras_service.wait_for_word(word="particle", timeout=30, interval=1) is None: - logging.info("Did not find the keyword 'Particle' in advanced video menu. Did Keras scroll correctly?") - sys.exit(1) - am.take_screenshot("advanced_video_2.png", ArtifactType.CONFIG_IMAGE, "second picture of advanced video settings") + logging.info( + "Did not find the keyword 'Particle' in advanced video menu. Did Keras scroll correctly?") + raise RuntimeError + am.take_screenshot("advanced_video_2.png", ArtifactType.CONFIG_IMAGE, + "second picture of advanced video settings") logging.info('Starting benchmark') user.press("`") @@ -149,7 +171,7 @@ def run_benchmark(keras_service): time.sleep(3) if keras_service.wait_for_word(word="benchmark", timeout=30, interval=0.1) is None: logging.error("Didn't see the title of the benchmark. Did the map load?") - sys.exit(1) + raise RuntimeError setup_end_time = int(time.time()) elapsed_setup_time = round(setup_end_time - setup_start_time, 2) @@ -166,7 +188,7 @@ def run_benchmark(keras_service): test_start_time = int(time.time()) logging.info("Saw \'lets roll\'! Marking the time.") - time.sleep(112) # sleep duration during gameplay + time.sleep(112) # sleep duration during gameplay # Default fallback end time test_end_time = int(time.time()) @@ -174,10 +196,10 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word(word="console", timeout=30, interval=0.1) if result is None: logging.error("The console didn't open. Please check settings and try again.") - sys.exit(1) - else: - test_end_time = int(time.time()) - logging.info("The console opened. Marking end time.") + raise RuntimeError + + test_end_time = int(time.time()) + logging.info("The console opened. Marking end time.") # allow time for result screen to populate time.sleep(8) @@ -227,3 +249,5 @@ if __name__ == "__main__": logging.error("something went wrong running the benchmark!") logging.exception(ex) sys.exit(1) + finally: + terminate_processes(PROCESS_NAME) diff --git a/cspell.json b/cspell.json index c55018f..00786ae 100644 --- a/cspell.json +++ b/cspell.json @@ -46,7 +46,8 @@ "Stellaris", "shusaura", "wukong", - "vgamepad" + "vgamepad", + "DaVinci" ], "ignoreRegExpList": [ "import .*" diff --git a/cyberpunk2077/cyberpunk2077.py b/cyberpunk2077/cyberpunk2077.py index 549aeb3..87369a6 100644 --- a/cyberpunk2077/cyberpunk2077.py +++ b/cyberpunk2077/cyberpunk2077.py @@ -27,6 +27,8 @@ LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") PROCESS_NAME = "cyberpunk2077.exe" +user.FAILSAFE = False + def start_game(): """Launch the game with no launcher or start screen""" return exec_steam_game(STEAM_GAME_ID, game_params=["--launcher-skip", "-skipStartScreen"]) @@ -85,10 +87,9 @@ def navigate_settings() -> None: am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "graphics menu 1") user.press("down") - - rast = kerasService.wait_for_word("view", interval=1, timeout=2) - if rast: - press_n_times("up", 2, 0.2) #gets you to film grain + time.sleep(0.5) + user.press("down") #gets you to film grain + time.sleep(0.5) dlss = kerasService.wait_for_word("dlss", interval=1, timeout=2) if dlss: diff --git a/cyberpunk2077/cyberpunk_utils.py b/cyberpunk2077/cyberpunk_utils.py index da3a1f4..f125155 100644 --- a/cyberpunk2077/cyberpunk_utils.py +++ b/cyberpunk2077/cyberpunk_utils.py @@ -27,7 +27,8 @@ def get_args() -> any: def copy_from_network_drive(): """Copies mod file from network drive to harness folder""" - src_path = Path(r"\\Labs\labs\03_ProcessingFiles\Cyberpunk 2077\basegame_no_intro_videos.archive") + src_path = Path( + r"\\labs.lmg.gg\labs\03_ProcessingFiles\Cyberpunk 2077\basegame_no_intro_videos.archive") dest_path = SCRIPT_DIRECTORY / "basegame_no_intro_videos.archive" shutil.copyfile(src_path, dest_path) diff --git a/doomdarkages/README.md b/doomdarkages/README.md new file mode 100644 index 0000000..2307030 --- /dev/null +++ b/doomdarkages/README.md @@ -0,0 +1,21 @@ +# Doom: The Dark Ages + +This script navigates through in-game menus to the built in benchmark and runs it with the current settings. It then waits for a results screen while running the Abyssal Forest benchmark. + +## Prerequisites + +- Python 3.10+ +- Doom: The Dark Ages installed +- Keras OCR service + +## Options + +- `kerasHost`: string representing the IP address of the Keras service. e.x. `0.0.0.0` +- `kerasPort`: string representing the port of the Keras service. e.x. `8080` + +## Output + +report.json +- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080` +- `start_time`: number representing a timestamp of the test's start time in milliseconds +- `end_time`: number representing a timestamp of the test's end time in milliseconds \ No newline at end of file diff --git a/doomdarkages/doomdarkages.py b/doomdarkages/doomdarkages.py new file mode 100644 index 0000000..54da02b --- /dev/null +++ b/doomdarkages/doomdarkages.py @@ -0,0 +1,256 @@ +"""Doom: The Dark Ages test script""" +import logging +from argparse import ArgumentParser +from pathlib import Path +import os.path +import time +import sys +import pydirectinput as user +from doomdarkages_utils import get_resolution + +sys.path.insert(1, os.path.join(sys.path[0], "..")) + +from doomdarkages_utils import copy_launcher_config +from harness_utils.steam import exec_steam_game, get_build_id +from harness_utils.keras_service import KerasService +from harness_utils.misc import press_n_times, mouse_scroll_n_times +from harness_utils.process import terminate_processes +from harness_utils.output import ( + format_resolution, + seconds_to_milliseconds, + setup_log_directory, + write_report_json, + DEFAULT_LOGGING_FORMAT, + DEFAULT_DATE_FORMAT, +) +from harness_utils.artifacts import ArtifactManager, ArtifactType + + + +SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") +PROCESS_NAME = "DOOMTheDarkAges" +STEAM_GAME_ID = 3017860 +username = os.getlogin() +BENCHMARK_RESULTS_PATH = f"C:\\Users\\{username}\\Saved Games\\id Software\\DOOMTheDarkAges\\base\\benchmark" + +user.FAILSAFE = False + +def start_game(): + """Launch the game with no launcher or start screen""" + copy_launcher_config() + return exec_steam_game(STEAM_GAME_ID, game_params=["+com_skipIntroVideo", "1"]) + +def find_latest_result_file(base_path): + """Look for files in the benchmark results path that match the pattern. + Returns the most recent benchmark file.""" + base_path = Path(base_path) + + files = list(base_path.glob("benchmark-*.json")) + if not files: + raise ValueError(f"No benchmark-*.json files found in {base_path}") + + return max(files, key=lambda p: p.stat().st_mtime) + +def run_benchmark(): + """Runs the actual benchmark.""" + start_game() + am = ArtifactManager(LOG_DIRECTORY) + + setup_start_time = int(time.time()) + time.sleep(25) + # Press space to proceed to the main menu + result = kerasService.wait_for_word_vulkan("press", timeout=80) + if not result: + logging.info("Didn't see title screen. Check settings and try again.") + sys.exit(1) + + logging.info("Hit the title screen. Continuing") + time.sleep(2) + user.press("space") + time.sleep(4) + + # Navigate menus and take screenshots using the artifact manager + result = kerasService.wait_for_word_vulkan("campaign", interval=3, timeout=60) + if not result: + logging.info("Didn't land on the main menu!") + sys.exit(1) + + logging.info("Saw the main menu. Proceeding.") + time.sleep(1) + + press_n_times("down", 3, 0.2) + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("daze", interval=3, timeout=15) + if not result: + logging.info("Didn't see the game settings. Did it navigate correctly?") + sys.exit(1) + + logging.info("Saw the game settings. Proceeding.") + press_n_times("q", 2, 0.2) + time.sleep(1) + + # Screenshotting the display settings + result = kerasService.wait_for_word_vulkan("display", interval=3, timeout=15) + if not result: + logging.info("Didn't find the video settings. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video1.png", ArtifactType.CONFIG_IMAGE, "1st screenshot of video settings menu") + mouse_scroll_n_times(6, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("nvidia", interval=3, timeout=15) + if not result: + logging.info("Didn't find the NVIDIA Reflex setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video2.png", ArtifactType.CONFIG_IMAGE, "2nd screenshot of video settings menu") + mouse_scroll_n_times(6, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("advanced", interval=3, timeout=15) + if not result: + logging.info("Didn't find the advanced heading. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video3.png", ArtifactType.CONFIG_IMAGE, "3rd screenshot of video settings menu") + mouse_scroll_n_times(5, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("shading", interval=3, timeout=15) + if not result: + logging.info("Didn't find the shading quality setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video4.png", ArtifactType.CONFIG_IMAGE, "4th screenshot of video settings menu") + mouse_scroll_n_times(5, -220, 0.2) + time.sleep(0.2) + + result = kerasService.wait_for_word_vulkan("brightness", interval=3, timeout=15) + if not result: + logging.info("Didn't find the brightness setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video5.png", ArtifactType.CONFIG_IMAGE, "5th screenshot of video settings menu") + user.press("escape") + time.sleep(0.2) + + # Navigating to the benchmark + result = kerasService.wait_for_word_vulkan("campaign", interval=3, timeout=20) + if not result: + logging.info("Didn't land on the main menu!") + sys.exit(1) + + logging.info("Saw the main menu. Proceeding.") + time.sleep(1) + + user.press("up") + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("benchmarks", interval=3, timeout=15) + if not result: + logging.info("Didn't navigate to the extras menu. Did it navigate properly?") + sys.exit(1) + + logging.info("Saw the extras menu. Proceeding.") + time.sleep(1) + + user.press("up") + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("abyssal", interval=3, timeout=15) + if not result: + logging.info("Don't see the Abyssal Forest benchmark option. Did it navigate properly?") + sys.exit(1) + + logging.info("See the benchmarks. Starting the Abyssal Forest benchmark level.") + time.sleep(1) + + press_n_times("down", 2, 0.2) + user.press("enter") + time.sleep(1) + + elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) + logging.info("Setup took %f seconds", elapsed_setup_time) + + result = kerasService.wait_for_word_vulkan("frame", interval=0.5, timeout=90) + if not result: + logging.info("Benchmark didn't start. Did the game crash?") + sys.exit(1) + + logging.info("Benchmark started. Waiting for benchmark to complete.") + test_start_time = int(time.time()) + 8 + + # Sleeping for the duration of the benchmark + time.sleep(110) + + test_end_time = None + + result = kerasService.wait_for_word_vulkan("results", interval=0.5, timeout=90) + if result: + logging.info("Found the results screen. Marking the out time.") + test_end_time = int(time.time()) - 2 + time.sleep(2) + else: + logging.info("Results screen was not found!" + + "Did harness not wait long enough? Or test was too long?") + sys.exit(1) + + logging.info("Results screen was found! Finishing benchmark.") + results_file = find_latest_result_file(BENCHMARK_RESULTS_PATH) + am.take_screenshot_vulkan("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of results") + am.copy_file(results_file, ArtifactType.RESULTS_TEXT, "benchmark results/settings xml file") + + elapsed_test_time = round(test_end_time - test_start_time, 2) + logging.info("Benchmark took %f seconds", elapsed_test_time) + + terminate_processes(PROCESS_NAME) + am.create_manifest() + + return test_start_time, test_end_time + + +setup_log_directory(LOG_DIRECTORY) + +logging.basicConfig( + filename=f"{LOG_DIRECTORY}/harness.log", + format=DEFAULT_LOGGING_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + level=logging.DEBUG, +) +console = logging.StreamHandler() +formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) +console.setFormatter(formatter) +logging.getLogger("").addHandler(console) + +parser = ArgumentParser() +parser.add_argument( + "--kerasHost", dest="keras_host", help="Host for Keras OCR service", required=True +) +parser.add_argument( + "--kerasPort", dest="keras_port", help="Port for Keras OCR service", required=True +) +args = parser.parse_args() +kerasService = KerasService(args.keras_host, args.keras_port) + +try: + start_time, end_time = run_benchmark() + width, height = get_resolution() + report = { + "resolution": format_resolution(width, height), + "start_time": seconds_to_milliseconds(start_time), + "end_time": seconds_to_milliseconds(end_time), + "version": get_build_id(STEAM_GAME_ID) + } + + write_report_json(LOG_DIRECTORY, "report.json", report) +except Exception as e: + logging.error("Something went wrong running the benchmark!") + logging.exception(e) + terminate_processes(PROCESS_NAME) + sys.exit(1) diff --git a/doomdarkages/doomdarkages_utils.py b/doomdarkages/doomdarkages_utils.py new file mode 100644 index 0000000..abb6770 --- /dev/null +++ b/doomdarkages/doomdarkages_utils.py @@ -0,0 +1,62 @@ +"""Utility functions supporting Doom: The Dark Ages test script.""" +import os +import re +from pathlib import Path +import sys +import logging +import shutil +import json + +sys.path.insert(1, os.path.join(sys.path[0], '..')) +from harness_utils.steam import get_app_install_location + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +RUN_DIR = SCRIPT_DIRECTORY / "run" +STEAM_GAME_ID = 3017860 +username = os.getlogin() +BENCHMARK_PATH = f"C:\\Users\\{username}\\Saved Games\\id Software\\DOOMTheDarkAges\\base\\benchmark" +RES_REGEX = re.compile(r'\s*(\d+)\s*[x×]\s*(\d+)') + +def get_resolution() -> tuple[int, int]: + """Gets resolution width and height from local xml file created by game.""" + try: + bench_file = max( + RUN_DIR.glob("benchmark-*.json"), + key=lambda p: p.stat().st_mtime + ) + except ValueError as exc: + # No files matched, propagate as a clearer FileNotFoundError + raise FileNotFoundError( + f"No benchmark-*.json files in {RUN_DIR}" + ) from exc + + if not bench_file.is_file(): + raise FileNotFoundError("Benchmark file not found.") + + with bench_file.open(encoding="utf-8") as f: + data = json.load(f) + + res_string = data.get("resolution", "") + m = RES_REGEX.search(res_string) + if not m: + raise ValueError( + f"Cannot parse 'resolution' in {bench_file.name}: {res_string!r}" + ) + + width, height = map(int, m.groups()) + return width, height + +def copy_launcher_config() -> None: + """Copy launcher config to doom launcher config folder""" + try: + launcherconfig_path = Path(get_app_install_location(STEAM_GAME_ID), "launcherData\\base\\configs") + launcherconfig_path.mkdir(parents=True, exist_ok=True) + + src_path = SCRIPT_DIRECTORY / "launcher.cfg" + dest_path = launcherconfig_path / "launcher.cfg" + + logging.info("Copying: %s -> %s", src_path, dest_path) + shutil.copy(src_path, dest_path) + except OSError as err: + logging.error("Could not copy config file.") + raise err diff --git a/doomdarkages/launcher.cfg b/doomdarkages/launcher.cfg new file mode 100644 index 0000000..b9734c2 --- /dev/null +++ b/doomdarkages/launcher.cfg @@ -0,0 +1,16 @@ +rgl_driverMustBeExactMatch 0 +rgl_minNvidiaDriverVersion 57680 +rgl_minAMDDriverMajorVersion 25 +rgl_minAMDDriverMinorVersion 5 +rgl_minAMDDriverPatchVersion 1 +rgl_minAMDDriverMajorVersionWin8 25 +rgl_minAMDDriverMinorVersionWin8 6 +rgl_minAMDDriverPatchVersionWin8 1 +rgl_minAMDDriverMajorVersionWin7 25 +rgl_minAMDDriverMinorVersionWin7 6 +rgl_minAMDDriverPatchVersionWin7 1 +rgl_minIntelDriverMajorVersion 101 +rgl_minIntelDriverMinorVersion 6732 +rgl_showAMDStartupWarning 0 +rgl_showIntelStartupWarning 0 +rgl_showNvidiaStartupWarning 0 diff --git a/doomdarkages/manifest.yaml b/doomdarkages/manifest.yaml new file mode 100644 index 0000000..38d1024 --- /dev/null +++ b/doomdarkages/manifest.yaml @@ -0,0 +1,10 @@ +friendly_name: "Doom: The Dark Ages" +executable: "doomdarkages.py" +process_name: "DOOMTheDarkAges.exe" +hidden: 0 +output_dir: "run" +options: + - name: kerasHost + type: input + - name: kerasPort + type: input \ No newline at end of file diff --git a/dota2/dota2.py b/dota2/dota2.py index 2d935ee..ec6ed78 100644 --- a/dota2/dota2.py +++ b/dota2/dota2.py @@ -1,13 +1,13 @@ """Dota 2 test script""" import logging -import os +from pathlib import Path import time import pyautogui as gui import pydirectinput as user import sys from dota2_utils import get_resolution, copy_replay, copy_config, get_args -sys.path.insert(1, os.path.join(sys.path[0], '..')) +sys.path.insert(1, str(Path(sys.path[0]).parent)) from harness_utils.output import ( setup_log_directory, @@ -22,12 +22,12 @@ from harness_utils.steam import exec_steam_game from harness_utils.artifacts import ArtifactManager, ArtifactType -SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) -LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +LOG_DIRECTORY = SCRIPT_DIRECTORY / "run" PROCESS_NAME = "dota2.exe" STEAM_GAME_ID = 570 -setup_log_directory(LOG_DIRECTORY) +setup_log_directory(str(LOG_DIRECTORY)) logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log', format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -40,10 +40,12 @@ logging.getLogger('').addHandler(console) args = get_args() kerasService = KerasService(args.keras_host, args.keras_port) +user.FAILSAFE = False def start_game(): """Launch the game with console enabled and FPS unlocked""" - return exec_steam_game(STEAM_GAME_ID, game_params=["-console", "+fps_max 0"]) + return exec_steam_game( + STEAM_GAME_ID, game_params=["-console", "+fps_max 0"]) def console_command(command): @@ -68,25 +70,43 @@ def run_benchmark(): time.sleep(1) # waiting about a minute for the main menu to appear - if kerasService.wait_for_word(word="heroes", timeout=80, interval=1) is None: - logging.error("Game didn't start in time. Check settings and try again.") + if kerasService.wait_for_word( + word="heroes", timeout=80, interval=1) is None: + logging.error( + "Game didn't start in time. Check settings and try again.") sys.exit(1) - height, width = get_resolution() - location = None + time.sleep(15) # wait for main menu + screen_height, screen_width = get_resolution() + location = None + click_multiple = 0 # We check the resolution so we know which screenshot to use for the locate on screen function - match width: + match screen_width: case "1280": - location = gui.locateOnScreen(f"{SCRIPT_DIRECTORY}\\screenshots\\settings_720.png", confidence=0.9) + location = gui.locateOnScreen( + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_720.png", + confidence=0.9) + click_multiple = 0.8 case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1080.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1080.png", + confidence=0.9) + click_multiple = 1 case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1440.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1440.png", + confidence=0.9) + click_multiple = 1.5 case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIRECTORY}\\screenshots\\settings_2160.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_2160.png", + confidence=0.9) + click_multiple = 2 case _: - logging.error("Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) + logging.error( + "Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", + screen_height, screen_width) sys.exit(1) # navigating to the video config section @@ -99,21 +119,47 @@ def run_benchmark(): result = kerasService.look_for_word(word="video", attempts=10, interval=1) if not result: - logging.info("Did not find the video menu button. Did Keras enter settings correctly?") + logging.info( + "Did not find the video menu button. Did Keras enter settings correctly?") sys.exit(1) - gui.moveTo(result["x"] + 10, result["y"] + 8) + gui.moveTo(result["x"] + int(50 * click_multiple), + result["y"] + int(20 * click_multiple)) gui.mouseDown() time.sleep(0.2) gui.mouseUp() time.sleep(0.2) - if kerasService.wait_for_word(word="resolution", timeout=30, interval=1) is None: - logging.info("Did not find the video settings menu. Did the menu get stuck?") + if kerasService.wait_for_word( + word="resolution", timeout=30, interval=1) is None: + logging.info( + "Did not find the video settings menu. Did the menu get stuck?") sys.exit(1) - am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") + am.take_screenshot("video1.png", ArtifactType.CONFIG_IMAGE, + "picture of video settings") + user.press("down") + + if kerasService.wait_for_word( + word="api", timeout=30, interval=1) is None: + logging.info( + "Did not find the video settings menu. Did the menu get stuck?") + sys.exit(1) + + am.take_screenshot("video2.png", ArtifactType.CONFIG_IMAGE, + "picture of video settings") + + user.press("down") + + if kerasService.wait_for_word( + word="direct", timeout=30, interval=1) is None: + logging.info( + "Did not find the video settings menu. Did the menu get stuck?") + sys.exit(1) + + am.take_screenshot("video3.png", ArtifactType.CONFIG_IMAGE, + "picture of video settings") # starting the benchmark user.press("escape") logging.info('Starting benchmark') @@ -124,7 +170,8 @@ def run_benchmark(): user.press("\\") time.sleep(5) - if kerasService.wait_for_word(word="directed", timeout=30, interval=0.1) is None: + if kerasService.wait_for_word( + word="directed", timeout=30, interval=0.1) is None: logging.error("Didn't see directed camera. Did the replay load?") sys.exit(1) @@ -138,26 +185,29 @@ def run_benchmark(): result = kerasService.wait_for_word(word="2560", timeout=30, interval=0.1) if result is None: - logging.error("Unable to find Leshrac's HP. Using default start time value.") + logging.error( + "Unable to find Leshrac's HP. Using default start time value.") else: test_start_time = int(time.time()) logging.info("Found Leshrac's HP! Marking the start time accordingly.") - time.sleep(73) # sleep duration during gameplay + time.sleep(73) # sleep duration during gameplay # Default fallback end time test_end_time = int(time.time()) result = kerasService.wait_for_word(word="1195", timeout=30, interval=0.1) if result is None: - logging.error("Unable to find gold count of 1195. Using default end time value.") + logging.error( + "Unable to find gold count of 1195. Using default end time value.") else: test_end_time = int(time.time()) logging.info("Found the gold. Marking end time.") time.sleep(2) - if kerasService.wait_for_word(word="heroes", timeout=25, interval=1) is None: + if kerasService.wait_for_word( + word="heroes", timeout=25, interval=1) is None: logging.error("Main menu after running benchmark not found, exiting") sys.exit(1) @@ -173,14 +223,14 @@ def run_benchmark(): try: start_time, end_time = run_benchmark() - height, width = get_resolution() + res_height, res_width = get_resolution() report = { - "resolution": format_resolution(width, height), + "resolution": format_resolution(int(res_width), int(res_height)), "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIRECTORY, "report.json", report) + write_report_json(str(LOG_DIRECTORY), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/dota2/dota2_utils.py b/dota2/dota2_utils.py index 09afce8..f44896a 100644 --- a/dota2/dota2_utils.py +++ b/dota2/dota2_utils.py @@ -37,7 +37,7 @@ def get_install_path(): def copy_replay_from_network_drive(): """Copies replay file from network drive to harness folder""" - src_path = Path(r"\\Labs\labs\03_ProcessingFiles\Dota2\benchmark.dem") + src_path = Path(r"\\labs.lmg.gg\labs\03_ProcessingFiles\Dota2\benchmark.dem") dest_path = SCRIPT_DIRECTORY / "benchmark.dem" shutil.copyfile(src_path, dest_path) @@ -84,13 +84,18 @@ def copy_config() -> None: def read_config() -> list[str] | None: """Looks for config file and returns contents if found""" - userdata_path = Path(get_steam_folder_path(), "userdata", str(STEAM_USER_ID), str(STEAM_GAME_ID), "local", "cfg", "video.txt") + userdata_path = Path( + get_steam_folder_path(), + "userdata", str(STEAM_USER_ID), + str(STEAM_GAME_ID), + "local", "cfg", "video.txt") install_path = Path(get_install_path(), "game", "dota", "cfg", "video.txt") try: with open(userdata_path, encoding="utf-8") as f: return f.readlines() except OSError: - logging.error("Did not find config file at path %s. Trying path %s", userdata_path, install_path) + logging.error("Did not find config file at path %s. Trying path %s", + userdata_path, install_path) try: with open(install_path, encoding="utf-8") as f: return f.readlines() @@ -118,6 +123,6 @@ def get_resolution(): height = height_match.group(1) if width_match is not None: width = width_match.group(1) - if height != 0 and width !=0: + if height != 0 and width != 0: return (height, width) return (height, width) diff --git a/dota2/screenshots/settings_1080.png b/dota2/screenshots/settings_1080.png index 84cee45..e5e84d9 100644 Binary files a/dota2/screenshots/settings_1080.png and b/dota2/screenshots/settings_1080.png differ diff --git a/dota2/screenshots/settings_1440.png b/dota2/screenshots/settings_1440.png index c8e7ca2..515bdd2 100644 Binary files a/dota2/screenshots/settings_1440.png and b/dota2/screenshots/settings_1440.png differ diff --git a/dota2/screenshots/settings_2160.png b/dota2/screenshots/settings_2160.png index af3032f..c896f8e 100644 Binary files a/dota2/screenshots/settings_2160.png and b/dota2/screenshots/settings_2160.png differ diff --git a/dota2/screenshots/settings_720.png b/dota2/screenshots/settings_720.png index 50f1dd8..9df5e44 100644 Binary files a/dota2/screenshots/settings_720.png and b/dota2/screenshots/settings_720.png differ diff --git a/farcry6/farcry6.py b/farcry6/farcry6.py index c128e13..61dd49b 100644 --- a/farcry6/farcry6.py +++ b/farcry6/farcry6.py @@ -1,4 +1,6 @@ """Far Cry 6 test script""" + +# pylint: disable = C0116, W0621 import os import logging import time @@ -30,25 +32,31 @@ LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") PROCESS_NAME = "FarCry6.exe" GAME_ID = 5266 username = os.getlogin() -xml_file = rf"C:\Users\{username}\Documents\My Games\Far Cry 6\gamerprofile.xml" +XML_FILE = rf"C:\Users\{username}\Documents\My Games\Far Cry 6\gamerprofile.xml" + + +user.FAILSAFE = False + def start_game(): - subprocess.run(f'start uplay://launch/{GAME_ID}/0', shell=True) + subprocess.run(f'start uplay://launch/{GAME_ID}/0', shell=True, check=True) + def skip_logo_screens() -> None: """Simulate input to skip logo screens""" logging.info("Skipping logo screens") - #skipping the logo screens + # skipping the logo screens press_n_times("escape", 8, 1) + def run_benchmark(): am = ArtifactManager(LOG_DIRECTORY) start_game() setup_start_time = int(time.time()) time.sleep(25) - #skipping game intros + # skipping game intros result = kerasService.look_for_word("warning", attempts=20, interval=1) if not result: logging.info("Did not see warnings. Did the game start?") @@ -66,7 +74,7 @@ def run_benchmark(): time.sleep(2) - #navigating the menus to get to the video settings + # navigating the menus to get to the video settings result = kerasService.look_for_word("later", attempts=5, interval=1) if result: user.press("escape") @@ -95,10 +103,11 @@ def run_benchmark(): gui.mouseUp() time.sleep(2) - #grabbing screenshots of all the video settings + # grabbing screenshots of all the video settings result = kerasService.look_for_word("adapter", attempts=10, interval=1) if not result: - logging.info("Did not find the Video Adapter setting in the monitor options. Did keras navigate wrong?") + logging.info( + "Did not find the Video Adapter setting in the monitor options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") @@ -109,18 +118,20 @@ def run_benchmark(): result = kerasService.look_for_word("filtering", attempts=10, interval=1) if not result: - logging.info("Did not find the Texture Filtering setting in the quality options. Did keras navigate wrong?") + logging.info( + "Did not find the Texture Filtering setting in the quality options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("quality1.png", ArtifactType.CONFIG_IMAGE, "1st picture of quality settings") time.sleep(2) - mouse_scroll_n_times(8, -800, 0.2) + mouse_scroll_n_times(8, -800, 0.2) result = kerasService.look_for_word("shading", attempts=10, interval=1) if not result: - logging.info("Did not find the FidelityFX Variable Shading setting in the quality options. Did keras navigate wrong?") + logging.info( + "Did not find the FidelityFX Variable Shading setting in the quality options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("quality2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of quality settings") @@ -131,12 +142,13 @@ def run_benchmark(): result = kerasService.look_for_word("lock", attempts=10, interval=1) if not result: - logging.info("Did not find the Enable Framerate Lock setting in the advanced options. Did keras navigate wrong?") + logging.info( + "Did not find the Enable Framerate Lock setting in the advanced options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("advanced.png", ArtifactType.CONFIG_IMAGE, "picture of advanced settings") - #starting the benchmark + # starting the benchmark time.sleep(2) user.press("f5") elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) @@ -148,7 +160,7 @@ def run_benchmark(): sys.exit(1) test_start_time = int(time.time()) - time.sleep(60) # wait for benchmark to complete + time.sleep(60) # wait for benchmark to complete result = kerasService.wait_for_word("results", interval=0.5, timeout=100) if not result: @@ -165,11 +177,12 @@ def run_benchmark(): # Exit terminate_processes(PROCESS_NAME) - am.copy_file(xml_file, ArtifactType.CONFIG_TEXT, "config file") + am.copy_file(XML_FILE, ArtifactType.CONFIG_TEXT, "config file") am.create_manifest() return test_start_time, test_end_time + setup_log_directory(LOG_DIRECTORY) logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log', @@ -204,4 +217,4 @@ except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) terminate_processes(PROCESS_NAME) - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/forza5/forza5.py b/forza5/forza5.py index 3d8943f..cb62db7 100644 --- a/forza5/forza5.py +++ b/forza5/forza5.py @@ -34,6 +34,7 @@ CONFIG_LOCATION = ( CONFIG_FILENAME = "UserConfigSelections" PROCESSES = ["ForzaHorizon5.exe", "RTSS.exe"] +user.FAILSAFE = False def start_rtss(): """Sets up the RTSS process""" diff --git a/godot_compile/godot_compile.py b/godot_compile/godot_compile.py index 5abb906..12120fd 100644 --- a/godot_compile/godot_compile.py +++ b/godot_compile/godot_compile.py @@ -78,11 +78,11 @@ def main(): report = { "start_time": start_time, - "version": "4.3-stable", + "version": "4.4.1-stable", "end_time": end_time, "score": score, "unit": "seconds", - "test": "Godot 4.3 Compile" + "test": "Godot 4.4.1 Compile" } write_report_json(LOG_DIR, "report.json", report) diff --git a/godot_compile/godot_compile_utils.py b/godot_compile/godot_compile_utils.py index f78f604..a64cad2 100644 --- a/godot_compile/godot_compile_utils.py +++ b/godot_compile/godot_compile_utils.py @@ -14,7 +14,7 @@ MINGW_ZIP = "x86_64-13.2.0-release-posix-seh-msvcrt-rt_v11-rev1.zip" MINGW_FOLDER = SCRIPT_DIR.joinpath("mingw64") MINICONDA_EXECUTABLE_PATH = Path("C:\\ProgramData\\miniconda3\\_conda.exe") CONDA_ENV_NAME = "godotbuild" -GODOT_DIR = "godot-4.3-stable" +GODOT_DIR = "godot-4.4.1-stable" def install_mingw() -> str: @@ -24,7 +24,7 @@ def install_mingw() -> str: if str(MINGW_FOLDER) not in original_path: os.environ['PATH'] = str(MINGW_FOLDER.joinpath('bin')) + os.pathsep + original_path return "existing mingw installation detected" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\MinGW\\").joinpath(MINGW_ZIP) + source = Path("\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\MinGW\\").joinpath(MINGW_ZIP) destination = SCRIPT_DIR.joinpath(MINGW_ZIP) shutil.copyfile(source, destination) with ZipFile(destination, 'r') as zip_object: @@ -36,7 +36,8 @@ def install_mingw() -> str: def copy_miniconda_from_network_drive(): """copies miniconda installer from network drive""" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\Miniconda\\").joinpath(MINICONDA_INSTALLER) + source = Path("\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\Miniconda\\").joinpath( + MINICONDA_INSTALLER) destination = SCRIPT_DIR.joinpath(MINICONDA_INSTALLER) shutil.copyfile(source, destination) @@ -49,15 +50,15 @@ def install_miniconda() -> str: copy_miniconda_from_network_drive() except Exception as err: raise Exception("could not copy miniconda from network drive") from err - command =[ + command = [ "powershell", - "start-process", + "start-process", "-FilePath", f'"{str(SCRIPT_DIR.joinpath(MINICONDA_INSTALLER))}"', "-ArgumentList", '"/S"', "-Wait" - ] + ] try: output = subprocess.check_output(command, stderr=subprocess.PIPE, text=True) except Exception as err: @@ -71,14 +72,14 @@ def copy_godot_source_from_network_drive() -> str: if SCRIPT_DIR.joinpath(GODOT_DIR).is_dir(): return "existing godot source directory detected" zip_name = f"{GODOT_DIR}.zip" - source = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Godot Files\\").joinpath(zip_name) + source = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Godot Files\\").joinpath(zip_name) destination = SCRIPT_DIR.joinpath(zip_name) shutil.copyfile(source, destination) with ZipFile(destination, 'r') as zip_object: try: zip_object.extractall(path=SCRIPT_DIR) except Exception as ex: - raise Exception ("error extracting godot zip") from ex + raise Exception("error extracting godot zip") from ex return "godot source copied and unpacked from network drive" @@ -90,7 +91,8 @@ def check_conda_environment_exists() -> bool: "-n", CONDA_ENV_NAME ] - process = subprocess.run(" ".join(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + process = subprocess.run(" ".join(command), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=False) if process.returncode == 1: return False return True @@ -131,6 +133,6 @@ def convert_duration_string_to_seconds(duration: str) -> int: hours=int(duration.split(':')[0]), minutes=int(duration.split(':')[1]), seconds=float(duration.split('.')[0].split(':')[2]), - milliseconds=int(float('0.' + duration.split('.')[1])*1000)) + milliseconds=int(float('0.' + duration.split('.')[1]) * 1000)) return round(time_obj.total_seconds()) diff --git a/gravitymark/gravitymark.py b/gravitymark/gravitymark.py index 6eee1ec..c5b3898 100644 --- a/gravitymark/gravitymark.py +++ b/gravitymark/gravitymark.py @@ -4,7 +4,7 @@ import getpass import subprocess import sys from pathlib import Path -from gravitymark_utils import friendly_test_name, get_args, get_score, create_gravitymark_command +from gravitymark_utils import friendly_test_param, get_args, get_score, create_gravitymark_command PARENT_DIR = str(Path(sys.path[0], "..")) sys.path.append(PARENT_DIR) @@ -19,7 +19,7 @@ GRAVITYMARK_PATH = Path("C:/", "Program Files", "GravityMark", "bin") GRAVITYMARK_EXE = GRAVITYMARK_PATH / "GravityMark.exe" args = get_args() -api = f"-{args.api}" +API = f"-{args.api}" script_dir = Path(__file__).resolve().parent log_dir = script_dir / "run" @@ -36,9 +36,11 @@ formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger("").addHandler(console) -gravitymark_log_path = Path("C:/Users", getpass.getuser(), ".GravityMark", "GravityMark.log") +gravitymark_log_path = Path( + "C:/Users", getpass.getuser(), + ".GravityMark", "GravityMark.log") image_path = log_dir / "result.png" -command = create_gravitymark_command(GRAVITYMARK_EXE, api, image_path) +command = create_gravitymark_command(GRAVITYMARK_EXE, API, image_path) try: logging.info('Starting benchmark!') @@ -47,7 +49,8 @@ try: result = subprocess.run(command, check=True, cwd=GRAVITYMARK_PATH) if result.returncode > 0: - logging.error("GravityMark exited with return code %d", result.returncode) + logging.error("GravityMark exited with return code %d", + result.returncode) sys.exit(1) score = get_score(gravitymark_log_path) @@ -57,12 +60,13 @@ try: sys.exit(1) report = { - "test": friendly_test_name(args.api), + "test": "GravityMark", + "test_parameter": friendly_test_param(args.api), "score": score, "unit": "score" } - write_report_json(log_dir, "report.json", report) + write_report_json(str(log_dir), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/gravitymark/gravitymark_utils.py b/gravitymark/gravitymark_utils.py index 3a0d0b8..899bebf 100644 --- a/gravitymark/gravitymark_utils.py +++ b/gravitymark/gravitymark_utils.py @@ -23,16 +23,16 @@ CLI_OPTIONS = { "-status": "1" } -def friendly_test_name(api: str) -> str: +def friendly_test_param(api: str) -> str: """return a friendlier string given the API harness argument""" if api == "vulkan": - return "GravityMark Vulkan" + return "Vulkan" if api == "opengl": - return "GravityMark OpenGL" + return "OpenGL" if api == "direct3d12": - return "GravityMark DX12" + return "DX12" if api == "direct3d11": - return "GravityMark DX11" + return "DX11" return api def get_args() -> Namespace: diff --git a/grid_legends/grid_legends.py b/grid_legends/grid_legends.py index c8d4b73..558cf7b 100644 --- a/grid_legends/grid_legends.py +++ b/grid_legends/grid_legends.py @@ -33,6 +33,8 @@ CONFIG_PATH = f"C:\\Users\\{username}\\Documents\\My Games\\GRID Legends\\hardwa CONFIG_FILENAME = "hardware_settings_config.xml" CONFIG_FULL_PATH = f"{CONFIG_PATH}\\{CONFIG_FILENAME}" +user.FAILSAFE = False + def get_resolution() -> tuple[int]: """Gets resolution width and height from local xml file created by game.""" resolution = re.compile(r" bool: def copy_handbrake_from_network_drive(): """copy handbrake cli from network drive""" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\Handbrake\\X86\\HandBrakeCLI-1.9.1-win-x86_64\\") + source = Path( + "\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\Handbrake\\X86\\HandBrakeCLI-1.9.1-win-x86_64\\") copy_souce = source / HANDBRAKE_EXECUTABLE destination = SCRIPT_DIR / HANDBRAKE_EXECUTABLE shutil.copyfile(copy_souce, destination) @@ -30,7 +31,7 @@ def is_video_source_present() -> bool: def copy_video_source(): """copy big buck bunny source video to local from network drive""" - source = r"\\Labs\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m" + source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, SOURCE_VIDEO_NAME) shutil.copyfile(source, destination) diff --git a/harness_utils/artifacts.py b/harness_utils/artifacts.py index ad26758..e84ffd8 100644 --- a/harness_utils/artifacts.py +++ b/harness_utils/artifacts.py @@ -81,7 +81,7 @@ class ArtifactManager: The newly created artifact's `type` and `description` fields are set to the given `artifact_type` and `description` arguments respectively while the artifact's `filename` is set to the basename of `src`. - + Raises a `ValueError` if `src` points to a directory instead of a file. """ src_path = Path(src) @@ -108,7 +108,7 @@ class ArtifactManager: The newly created artifact's `filename`, `type` and `description` fields are set to the given `filename`, `artifact_type` and `description` arguments respectively. - Raises a `ValueError` if `artifact_type` is not one of the `ArtifactType` values which represents an image. + Raises a `ValueError` if `artifact_type` is not one of the `ArtifactType` values which represents an image. """ if artifact_type not in _IMAGE_ARTIFACT_TYPES: raise ValueError("artifact_type should be a type that represents an image artifact") diff --git a/harness_utils/misc.py b/harness_utils/misc.py index 24b28d8..833cdd1 100644 --- a/harness_utils/misc.py +++ b/harness_utils/misc.py @@ -1,4 +1,5 @@ """Misc utility functions""" +from argparse import ArgumentParser import logging import os from pathlib import Path @@ -10,6 +11,9 @@ import requests import vgamepad as vg import json import re +import sys + +user.FAILSAFE = False class LTTGamePad360(vg.VX360Gamepad): """ @@ -19,7 +23,8 @@ class LTTGamePad360(vg.VX360Gamepad): This class extension provides some useful functions to make your code look a little cleaner when implemented in our harnesses. """ - def single_press(self, button = vg.XUSB_BUTTON.XUSB_GAMEPAD_DPAD_DOWN, pause = 0.1): + + def single_press(self, button=vg.XUSB_BUTTON.XUSB_GAMEPAD_DPAD_DOWN, pause=0.1): """ Custom function to perform a single press of a specified gamepad button @@ -59,6 +64,7 @@ class LTTGamePad360(vg.VX360Gamepad): self.single_press(button) time.sleep(pause) + class LTTGamePadDS4(vg.VDS4Gamepad): """ Class extension for the virtual game pad library @@ -67,7 +73,8 @@ class LTTGamePadDS4(vg.VDS4Gamepad): This class extension provides some useful functions to make your code look a little cleaner when implemented in our harnesses. """ - def single_button_press(self, button = vg.DS4_BUTTONS.DS4_BUTTON_CROSS, fastpause = 0.05): + + def single_button_press(self, button=vg.DS4_BUTTONS.DS4_BUTTON_CROSS, fastpause=0.05): """ Custom function to perform a single press of a specified gamepad digital button @@ -95,7 +102,6 @@ class LTTGamePadDS4(vg.VDS4Gamepad): time.sleep(fastpause) self.release_button(button=button) self.update() - def button_press_n_times(self, button: vg.DS4_BUTTONS, n: int, pause: float): """ @@ -105,7 +111,7 @@ class LTTGamePadDS4(vg.VDS4Gamepad): self.single_button_press(button) time.sleep(pause) - def single_dpad_press(self, direction = vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, pause = 0.1): + def single_dpad_press(self, direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, pause=0.1): """ Custom function to perform a single press of a specified gamepad button @@ -139,6 +145,7 @@ class LTTGamePadDS4(vg.VDS4Gamepad): self.single_dpad_press(direction) time.sleep(pause) + def clickme(x: int, y: int): """Pyautogui's click function sucks, this should do the trick""" gui.moveTo(x, y) @@ -147,10 +154,11 @@ def clickme(x: int, y: int): time.sleep(0.2) gui.mouseUp() + def mouse_scroll_n_times(n: int, scroll_amount: int, pause: float): """ Pyautogui's mouse scroll function often fails to actually scroll in game menus, this functions solves that problem - + n --> the number of times you want to scroll, should be a positive integer scroll_amount --> positive is scroll up, negative is scroll down pause --> the amount of time to pause betwee subsequent scrolls @@ -159,12 +167,19 @@ def mouse_scroll_n_times(n: int, scroll_amount: int, pause: float): gui.vscroll(scroll_amount) time.sleep(pause) -def press_n_times(key: str, n: int, pause: float): + +def int_time() -> int: + """Returns the current time in seconds since epoch as an integer""" + return int(time.time()) + + +def press_n_times(key: str, n: int, pause: float = 0.5): """A helper function press the same button multiple times""" for _ in range(n): user.press(key) time.sleep(pause) + def remove_files(paths: list[str]) -> None: """Removes files specified by provided list of file paths. Does nothing for a path that does not exist. @@ -199,7 +214,7 @@ def extract_file_from_archive(zip_file: Path, member_path: str, destination_dir: def find_eg_game_version(gamefoldername: str) -> str: """Find the version of the specific game (e.g., AlanWake2) from the launcher installed data.""" installerdat = r"C:\ProgramData\Epic\UnrealEngineLauncher\LauncherInstalled.dat" - + try: # Open the file and read its entire content with open(installerdat, encoding="utf-8") as file: @@ -213,7 +228,7 @@ def find_eg_game_version(gamefoldername: str) -> str: # Extract the InstallationList part from the file installation_list_json = installation_list_match.group(1) - + # Load the installation list as JSON installation_list = json.loads(installation_list_json) @@ -228,3 +243,20 @@ def find_eg_game_version(gamefoldername: str) -> str: print(f"Error: {e}") return None + + +def find_word(keras_service, word, msg, timeout=30, interval=1): + """Function to call Keras service to find a word in the screen""" + if keras_service.wait_for_word(word=word, timeout=timeout, interval=interval) is None: + logging.error(msg) + sys.exit(1) + + +def keras_args(): + """helper function to get args for keras""" + parser = ArgumentParser() + parser.add_argument("--kerasHost", dest="keras_host", + help="Host for Keras OCR service", required=True) + parser.add_argument("--kerasPort", dest="keras_port", + help="Port for Keras OCR service", required=True) + return parser.parse_args() diff --git a/harness_utils/output.py b/harness_utils/output.py index 875751d..72d28dd 100644 --- a/harness_utils/output.py +++ b/harness_utils/output.py @@ -1,17 +1,20 @@ """Functions related to logging and formatting output from test harnesses.""" import json import os +import logging DEFAULT_LOGGING_FORMAT = '%(asctime)s %(levelname)-s %(message)s' DEFAULT_DATE_FORMAT = '%m-%d %H:%M' + def setup_log_directory(log_dir: str) -> None: """Creates the log directory for a harness if it does not already exist""" if not os.path.isdir(log_dir): os.mkdir(log_dir) -def write_report_json(log_dir: str, report_name: str, report_json: any) -> None: +def write_report_json( + log_dir: str, report_name: str, report_json: dict) -> None: """Writes the json output of a harness to the log directory""" with open(os.path.join(log_dir, report_name), "w", encoding="utf-8") as file: file.write(json.dumps(report_json)) @@ -25,3 +28,17 @@ def format_resolution(width: int, height: int) -> str: def seconds_to_milliseconds(seconds: float | int) -> int: """Convert seconds to milliseconds""" return round((seconds * 1000)) + + +def setup_logging(log_directory: str) -> None: + """Sets up logging for the harness""" + setup_log_directory(log_directory) + + logging.basicConfig(filename=f'{log_directory}/harness.log', + format=DEFAULT_LOGGING_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + level=logging.DEBUG) + console = logging.StreamHandler() + formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) diff --git a/marvelrivals/README.md b/marvelrivals/README.md index 88b0e29..a6e8a6b 100644 --- a/marvelrivals/README.md +++ b/marvelrivals/README.md @@ -1,12 +1,11 @@ # Marvel Rivals -This benchmark runs a replay of a Season 1 tournament Double Elimination round game between SendHelp and BeerLovers +This benchmark runs a canned benchmark built into the Marvel Rivals settings. ## Prerequisites - Python 3.10+ - Marvel Rivals installed on Steam - Keras OCR service -- Favoriting replay ID 10518740076 ## Options diff --git a/marvelrivals/marvelrivals.py b/marvelrivals/marvelrivals.py index d1244e9..07f4f38 100644 --- a/marvelrivals/marvelrivals.py +++ b/marvelrivals/marvelrivals.py @@ -7,7 +7,7 @@ import time import pyautogui as gui import pydirectinput as user import sys -from marvelrivals_utils import read_resolution +from marvelrivals_utils import read_resolution, find_latest_benchmarkcsv import subprocess sys.path.insert(1, os.path.join(sys.path[0], '..')) @@ -33,7 +33,9 @@ LAUNCHER_NAME = "MarvelRivals_Launcher.exe" APPDATA = os.getenv("LOCALAPPDATA") CONFIG_LOCATION = f"{APPDATA}\\Marvel\\Saved\\Config\\Windows" CONFIG_FILENAME = "GameUserSettings.ini" -cfg = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}" +CFG = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}" + +user.FAILSAFE = False am = ArtifactManager(LOG_DIR) @@ -49,19 +51,20 @@ def setup_logging(): console.setFormatter(formatter) logging.getLogger('').addHandler(console) + + def start_game(): """Starts the game process""" game_path = get_app_install_location(STEAM_GAME_ID) process_path = os.path.join(game_path, LAUNCHER_NAME) # Full path to the executable - logging.info(f"Starting game: {process_path}") - process = subprocess.Popen([process_path], cwd=game_path) + logging.info("Starting game: %s", process_path) + process = subprocess.Popen([process_path], cwd=game_path) # pylint: disable=R1732 return process def run_benchmark(keras_service): """Run Marvel Rivals benchmark""" setup_start_time = int(time.time()) start_game() - #wait for launcher to launch then click the launch button to launch the launcher into the game that we were launching time.sleep(20) @@ -84,7 +87,13 @@ def run_benchmark(keras_service): gui.mouseDown() time.sleep(0.2) gui.mouseUp() - time.sleep(0.5) + time.sleep(20) + + #checking if a marketing notification has come up + result = keras_service.wait_for_word("view", timeout=15, interval=1) + if result: + user.press("escape") + time.sleep(0.5) #navigating to the video settings and taking screenshots result = keras_service.wait_for_word("play", timeout=30, interval=1) @@ -125,42 +134,14 @@ def run_benchmark(keras_service): time.sleep(1) #navigate to the player profile - user.press("escape") + mouse_scroll_n_times(10, 800, 0.2) time.sleep(1) - result = keras_service.wait_for_word("play", timeout=30, interval=1) + + result = keras_service.wait_for_word("run", timeout=30, interval=1) if not result: - logging.info("Did not find the play menu. Did it press escape?") + logging.info("Did not find the Performance Test. Did it scroll back up properly?") sys.exit(1) - time.sleep(1) - height, width = read_resolution() - location = None - - # We check the resolution so we know which screenshot to use for the locate on screen function - match width: - case "1280": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_720.png", confidence=0.9) - case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_1080.png", confidence=0.9) - case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_1440.png", confidence=0.9) - case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_2160.png", confidence=0.9) - case _: - logging.error("Could not find the profile icon. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) - click_me = gui.center(location) - gui.moveTo(click_me.x, click_me.y) - gui.mouseDown() - time.sleep(0.2) - gui.mouseUp() - time.sleep(0.5) - - #navigate to the replays section - result = keras_service.wait_for_word("favorites", timeout=30, interval=1) - if not result: - logging.info("Did not find the favorites menu. Did it navigate properly to it?") - sys.exit(1) gui.moveTo(result["x"], result["y"]) time.sleep(0.2) gui.mouseDown() @@ -168,69 +149,42 @@ def run_benchmark(keras_service): gui.mouseUp() time.sleep(1) - result = keras_service.wait_for_word("match", timeout=30, interval=1) + result = keras_service.wait_for_word("start", timeout=30, interval=1) if not result: - logging.info("Did not find the match replays menu. Did it click correctly?") + logging.info("Did not find the Start Test button. Keras click correctly?") sys.exit(1) + gui.moveTo(result["x"], result["y"]) time.sleep(0.2) gui.mouseDown() time.sleep(0.2) gui.mouseUp() time.sleep(1) - - #starting the benchmark replay - result = keras_service.wait_for_word("shibuya", timeout=30, interval=1) - if not result: - logging.info("Did not find the replay we were looking for. Is it not saved in the favorites?") - sys.exit(1) - match width: - case "1280": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_720.png", confidence=0.9) - case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_1080.png", confidence=0.9) - case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_1440.png", confidence=0.9) - case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_2160.png", confidence=0.9) - case _: - logging.error("Could not find the play button. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) - click_me = gui.center(location) - gui.moveTo(click_me.x, click_me.y) - gui.mouseDown() - time.sleep(0.2) - gui.mouseUp() - time.sleep(0.5) - #marking the in-time + #marking the end time setup_end_time = int(time.time()) elapsed_setup_time = round(setup_end_time - setup_start_time, 2) logging.info("Harness setup took %f seconds", elapsed_setup_time) time.sleep(2) - - #looking for the player name to start wait timer till we get into the actual game - result = keras_service.wait_for_word("dluo", timeout=30, interval=1) - if not result: - logging.info("Did not find the player Dluo. Did the replay start?") - sys.exit(1) - time.sleep(90) - #looking for landmark to mark benchmark start time and then wait for first round to finish - if keras_service.wait_for_word(word="defend", timeout=30, interval=1) is None: - logging.info("Didn't see the defend waypoint. Did the game crash?") + #looking for the FPS data graph + result = keras_service.wait_for_word("fps", timeout=30, interval=1) + if not result: + logging.info("Did not find the FPS graph. Did the replay start?") sys.exit(1) - test_start_time = int(time.time()) + 2 - time.sleep(460) + + test_start_time = int(time.time()) + time.sleep(98) #checking that first round has finished - result = keras_service.wait_for_word("complete", timeout=30, interval=1) + result = keras_service.wait_for_word("again", timeout=30, interval=1) if not result: - logging.info("First round doesn't appear to have finished. Did the replay start?") + logging.info("Didn't see the results screen. Did the test crash?") sys.exit(1) test_end_time = int(time.time()) - - am.copy_file(Path(cfg), ArtifactType.CONFIG_TEXT, "Marvel Rivals Video Config") + + am.copy_file(Path(CFG), ArtifactType.CONFIG_TEXT, "Marvel Rivals Video Config") + am.copy_file(Path(find_latest_benchmarkcsv()), ArtifactType.CONFIG_TEXT, "Marvel Rivals Benchmark CSV") logging.info("Run completed. Closing game.") time.sleep(2) @@ -274,4 +228,4 @@ if __name__ == "__main__": except Exception as ex: logging.error("something went wrong running the benchmark!") logging.exception(ex) - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/marvelrivals/marvelrivals_utils.py b/marvelrivals/marvelrivals_utils.py index c50abed..9f93aff 100644 --- a/marvelrivals/marvelrivals_utils.py +++ b/marvelrivals/marvelrivals_utils.py @@ -29,4 +29,13 @@ def read_resolution(): height = height_match.group(1) if width_match is not None: width = width_match.group(1) - return (height, width) \ No newline at end of file + return (height, width) + +def find_latest_benchmarkcsv(): + """find latest log from the benchmark""" + appdata_path = os.getenv('LOCALAPPDATA') + benchmarkcsv_dir = Path(appdata_path) / "Marvel" / "Saved" / "Benchmark" + files = [os.path.join(benchmarkcsv_dir, file) for file in os.listdir( + benchmarkcsv_dir) if os.path.isfile(os.path.join(benchmarkcsv_dir, file))] + latest_file = max(files, key=os.path.getmtime) + return latest_file diff --git a/marvelrivals/screenshots/play_1080.png b/marvelrivals/screenshots/play_1080.png deleted file mode 100644 index 49621b8..0000000 Binary files a/marvelrivals/screenshots/play_1080.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_1440.png b/marvelrivals/screenshots/play_1440.png deleted file mode 100644 index 72ad6e4..0000000 Binary files a/marvelrivals/screenshots/play_1440.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_2160.png b/marvelrivals/screenshots/play_2160.png deleted file mode 100644 index 2c52b1a..0000000 Binary files a/marvelrivals/screenshots/play_2160.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_720.png b/marvelrivals/screenshots/play_720.png deleted file mode 100644 index 0e7d1d9..0000000 Binary files a/marvelrivals/screenshots/play_720.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_1080.png b/marvelrivals/screenshots/profile_1080.png deleted file mode 100644 index 5b42fdc..0000000 Binary files a/marvelrivals/screenshots/profile_1080.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_1440.png b/marvelrivals/screenshots/profile_1440.png deleted file mode 100644 index bc68225..0000000 Binary files a/marvelrivals/screenshots/profile_1440.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_2160.png b/marvelrivals/screenshots/profile_2160.png deleted file mode 100644 index e16adf7..0000000 Binary files a/marvelrivals/screenshots/profile_2160.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_720.png b/marvelrivals/screenshots/profile_720.png deleted file mode 100644 index 649452b..0000000 Binary files a/marvelrivals/screenshots/profile_720.png and /dev/null differ diff --git a/procyon_ai/ulprocai.py b/procyon_ai/ulprocai.py index 37104f2..467ab2d 100644 --- a/procyon_ai/ulprocai.py +++ b/procyon_ai/ulprocai.py @@ -1,4 +1,5 @@ """UL Procyon Computer Vision test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -32,7 +33,7 @@ from harness_utils.procyoncmd import ( get_cuda_devices, ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -48,74 +49,87 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "AMD_CPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_cpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": "CPU", - "device_id": "CPU", # TODO: Find a good way to report the CPU name here. - "test_name": "WinML CPU (FLOAT32)" + # TODO: Find a good way to report the CPU name here. + "device_id": "CPU", + "test_name": "cpu_float32", + "api": "winml" }, "AMD_GPU0": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[0], - "test_name": "WinML GPU (FLOAT32)" + "test_name": "gpu_float32", + "api": "winml" }, "AMD_GPU1": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0], - "test_name": "WinML GPU (FLOAT32)" + "test_name": "gpu_float32", + "api": "winml" }, "Intel_CPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_cpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "CPU", "device_name": OPENVINO_DEVICES["CPU"], - "test_name": "Intel OpenVINO CPU (FLOAT32)" + "test_name": "cpu_float32", + "api": "openvino" }, "Intel_GPU0": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO GPU 0 (FLOAT32)" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "gpu_float32", + "api": "openvino" }, "Intel_GPU1": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO GPU 1 (FLOAT32)" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "gpu_float32", + "api": "openvino" }, "Intel_NPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_npu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "NPU", "device_name": OPENVINO_DEVICES.get("NPU", "None"), - "test_name": "Intel OpenVINO NPU (FLOAT32)" + "test_name": "npu_float32", + "api": "openvino" }, "NVIDIA_GPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_tensorrt.def\"", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "process_name": "TensorRT.exe", - "test_name": "NVIDIA TensorRT (FLOAT32)" + "process_name": "TensorRT.exe", + "test_name": "gpu_float32", + "api": "tensorrt" }, "Qualcomm_HTP": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_snpe.def\"", "device_id": "CPU", "device_name": "CPU", - "process_name": "SNPE.exe", - "test_name": "Qualcomm SNPE (INTEGER)" + "process_name": "SNPE.exe", + "test_name": "htp_integer", + "api": "snpe" }, } + + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -130,7 +144,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -160,16 +175,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() logging.info("Detected Windows ML Devices: %s", str(WINML_DEVICES)) @@ -203,15 +219,17 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI CV", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], + "api": BENCHMARK_CONFIG[args.engine]["api"], "test_version": find_test_version(), "device_name": BENCHMARK_CONFIG[args.engine]["device_name"], "procyon_version": find_procyon_version(), "unit": "score", - "score": score + "score": score } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/procyon_ai_img_gen/ulprocai_img_gen.py b/procyon_ai_img_gen/ulprocai_img_gen.py index 91ee41f..6a389c2 100644 --- a/procyon_ai_img_gen/ulprocai_img_gen.py +++ b/procyon_ai_img_gen/ulprocai_img_gen.py @@ -1,4 +1,5 @@ """UL Procyon AI Image Generation test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -26,7 +27,7 @@ from harness_utils.output import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -41,102 +42,117 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "AMD_GPU0_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"", - "process_name": "ort-directml.exe", + "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": "0", - "test_name": "ONNX Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "onnx" }, "AMD_GPU1_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"", - "process_name": "ort-directml.exe", + "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": "1" if len(list(WINML_DEVICES.values())) > 1 else "0", - "test_name": "ONNX Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "onnx" }, "AMD_GPU0_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"", "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": "0", - "test_name": "ONNX Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "onnx" }, "AMD_GPU1_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"", "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0], - "test_name": "ONNX Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "onnx" }, "Intel_GPU0_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion INT8" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_int8", + "api": "openvino" }, "Intel_GPU0_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion FP16" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_fp16", + "api": "openvino" }, "Intel_GPU0_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion FP16 XL" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_fp16_xl", + "api": "openvino" }, "Intel_GPU1_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion INT8" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_int8", + "api": "openvino" }, "Intel_GPU1_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion FP16" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_fp16", + "api": "openvino" }, "Intel_GPU1_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion FP16 XL" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_fp16_xl", + "api": "openvino" }, "NVIDIA_GPU_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion INT8" + "test_name": "stable_diffusion_int8", + "api": "tensorrt" }, "NVIDIA_GPU_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "tensorrt" }, "NVIDIA_GPU_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "tensorrt" } } + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -151,7 +167,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -179,16 +196,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() logging.info("Detected Windows ML Devices: %s", str(WINML_DEVICES)) @@ -197,7 +215,9 @@ try: args = get_arguments() option = BENCHMARK_CONFIG[args.engine]["config"] - cmd = create_procyon_command(option, BENCHMARK_CONFIG[args.engine]["process_name"], BENCHMARK_CONFIG[args.engine]["device_id"]) + cmd = create_procyon_command( + option, BENCHMARK_CONFIG[args.engine]["process_name"], + BENCHMARK_CONFIG[args.engine]["device_id"]) logging.info('Starting benchmark!') logging.info(cmd) start_time = time.time() @@ -221,16 +241,18 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI Image Generation", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], + "api": BENCHMARK_CONFIG[args.engine]["api"], "test_version": find_test_version(), "device_name": BENCHMARK_CONFIG[args.engine]["device_name"], "procyon_version": find_procyon_version(), "unit": "score", "score": score - + } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/procyon_ai_text_generation/ulprocai_text_gen.py b/procyon_ai_text_generation/ulprocai_text_gen.py index 2655b38..5fe9fb2 100644 --- a/procyon_ai_text_generation/ulprocai_text_gen.py +++ b/procyon_ai_text_generation/ulprocai_text_gen.py @@ -1,4 +1,5 @@ """UL Procyon AI Text Generation test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -20,7 +21,7 @@ from harness_utils.output import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -31,71 +32,83 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "All_Models_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_all.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "All LLM Model Text Generation" + "test_name": "all_models", + "api": "onnx" }, "Llama_2_13B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2.def\"", "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "LLama 2 Text Generation" + "test_name": "llama_2_13b", + "api": "onnx" }, - "Llama_3_1_8B_ONNX": { + "Llama_3_1_8B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Llama 3.1 Text Generation" + "test_name": "llama_3_1_8b", + "api": "onnx" }, "Mistral_7B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Mistral Text Generation" + "test_name": "mistral_7b", + "api": "onnx" }, "Phi_3_5_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Phi Text Generation" + "test_name": "phi_3_5", + "api": "onnx" }, "All_Models_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_all_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "All LLM Model Text Generation" + "test_name": "all_models", + "api": "openvino" }, "Llama_2_13B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2_openvino.def\"", "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "LLama 2 Text Generation" + "test_name": "llama_2_13b", + "api": "openvino" }, - "Llama_3_1_8B_OPENVINO": { + "Llama_3_1_8B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Llama 3.1 Text Generation" + "test_name": "llama_3_1_8b", + "api": "openvino" }, "Mistral_7B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Mistral Text Generation" + "test_name": "mistral_7b", + "api": "openvino" }, "Phi_3_5_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Phi Text Generation" + "test_name": "phi_3_5", + "api": "openvino" } } + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -110,7 +123,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -129,16 +143,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() args = get_arguments() @@ -165,7 +180,8 @@ try: sys.exit(1) report = { - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI Text Generation", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], "unit": "score", "score": score, "start_time": seconds_to_milliseconds(start_time), @@ -175,7 +191,7 @@ try: logging.info("Benchmark took %.2f seconds", elapsed_test_time) logging.info("Score was %s", score) - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) else: session_report = [] @@ -198,19 +214,19 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": test_type[0], + "test": "Procyon AI Text Generation", + "test_parameter": test_type[1]["test_name"], + "api": test_type[1]["api"], "test_version": find_test_version(), "procyon_version": find_procyon_version(), "unit": "score", "score": score - - } - + } session_report.append(report) - write_report_json(LOG_DIR, "report.json", session_report) + write_report_json(str(LOG_DIR), "report.json", session_report) except Exception as e: logging.error("Something went wrong running the benchmark!") diff --git a/procyon_ai_text_generation/utils.py b/procyon_ai_text_generation/utils.py index c060062..9557076 100644 --- a/procyon_ai_text_generation/utils.py +++ b/procyon_ai_text_generation/utils.py @@ -10,6 +10,7 @@ import logging SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" + def is_process_running(process_name): """check if given process is running""" for process in psutil.process_iter(['pid', 'name']): @@ -17,6 +18,7 @@ def is_process_running(process_name): return process return None + def regex_find_score_in_xml(result_regex): """Reads score from local game log""" score_pattern = re.compile(result_regex) @@ -30,26 +32,29 @@ def regex_find_score_in_xml(result_regex): score_value = score_match.group(1) return score_value + def get_install_path() -> str: """Gets the path to the Steam installation directory from the SteamPath registry key""" reg_path = r"Software\UL\Procyon" - reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, reg_path, 0, winreg.KEY_READ) - value, _ = winreg.QueryValueEx(reg_key, "InstallDir") + reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + reg_path, 0, winreg.KEY_READ) + value, _ = winreg.QueryValueEx(reg_key, "InstallDir") return value + def find_procyon_version() -> str: """Gets the version of an executable located in the install path.""" install_path = get_install_path() - + if not install_path: logging.info("Installation path not found.") - return None + return "" exe_path = os.path.join(install_path, "ProcyonCmd.exe") if not os.path.exists(exe_path): - logging.info(f"Executable not found at {exe_path}") - return None + logging.info("Executable not found at %s", exe_path) + return "" try: # Get all file version info @@ -61,7 +66,7 @@ def find_procyon_version() -> str: if ms is None or ls is None: logging.info("No FileVersionMS or FileVersionLS found.") - return None + return "" # Convert to human-readable version: major.minor.build.revision major = ms >> 16 @@ -73,29 +78,31 @@ def find_procyon_version() -> str: return version except Exception as e: - logging.info(f"Error retrieving version info from {exe_path}: {e}") - return None # Return None if version info retrieval fails + logging.info("Error retrieving version info from %s: %s", exe_path, e) + return "" # Return empty string if version info retrieval fails + def find_test_version() -> str: """Gets the version of an executable located in the chops path.""" chops_path = "C:\\ProgramData\\UL\\Procyon\\chops\\dlc\\ai-textgeneration-benchmark\\x64" - logging.info(f"The install path for the test is {chops_path}") - + logging.info("The install path for the test is %s", chops_path) + if not chops_path: logging.info("Installation path not found.") - return None + return "" exe_path = os.path.join(chops_path, "Handler.exe") if not os.path.exists(exe_path): - logging.info(f"Executable 'Handler.exe' not found at {exe_path}") - return None + logging.info("Executable 'Handler.exe' not found at %s", exe_path) + return "" try: - lang, codepage = win32api.GetFileVersionInfo(exe_path, "\\VarFileInfo\\Translation")[0] + lang, codepage = win32api.GetFileVersionInfo( + exe_path, "\\VarFileInfo\\Translation")[0] str_info_path = f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion" - return win32api.GetFileVersionInfo(exe_path, str_info_path) + return str(win32api.GetFileVersionInfo(exe_path, str_info_path)) except Exception as e: - logging.info(f"Error retrieving version info from {exe_path}: {e}") - return None # Return None if version info retrieval fails \ No newline at end of file + logging.info("Error retrieving version info from %s: %s", exe_path, e) + return "" # Return empty string if version info retrieval fails diff --git a/pugetbench/manifest.yaml b/pugetbench/manifest.yaml index 1d554ea..c33f0a4 100644 --- a/pugetbench/manifest.yaml +++ b/pugetbench/manifest.yaml @@ -15,6 +15,6 @@ options: - aftereffects - resolve tooltip: Select which test to run - - name: benchmark + - name: benchmark_version type: input tooltip: Version of benchmark to run diff --git a/pugetbench/pugetbench.py b/pugetbench/pugetbench.py index 129d1d2..5be174d 100644 --- a/pugetbench/pugetbench.py +++ b/pugetbench/pugetbench.py @@ -69,9 +69,9 @@ def run_benchmark(application: str, app_version: str, benchmark_version: str): "Standard", "--app_version", f"{app_version}"] command = None if application == "premierepro": - command = [executable_path] + command_args + ["--app", "photoshop"] - elif application == "photoshop": command = [executable_path] + command_args + ["--app", "premierepro"] + elif application == "photoshop": + command = [executable_path] + command_args + ["--app", "photoshop"] elif application == "aftereffects": command = [executable_path] + command_args + ["--app", "aftereffects"] elif application == "resolve": @@ -136,19 +136,19 @@ def main(): score = 0 test = "" if args.app == "premierepro": - test = "PugetBench Adobe Premiere Pro" + test = "Adobe Premiere Pro" if version is None: version = get_premierepro_version() elif args.app == "photoshop": - test = "PugetBench Adobe Photoshop" + test = "Adobe Photoshop" if version is None: version = get_photoshop_version() elif args.app == "aftereffects": - test = "PugetBench Adobe After Effects" + test = "Adobe After Effects" if version is None: version = get_aftereffects_version() elif args.app == "resolve": - test = "PugetBench Davinci Resolve Studio" + test = "Davinci Resolve Studio" if version is None: version = get_davinci_version() + "-studio" @@ -163,7 +163,8 @@ def main(): report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": test, + "test": "PugetBench", + "test_parameter": test, "app_version": version, "benchmark_version": args.benchmark_version, "pugetbench_version": get_pugetbench_version(), diff --git a/reddeadredemption2/reddeadredemption2.py b/reddeadredemption2/reddeadredemption2.py index 895d1b1..b016c1e 100644 --- a/reddeadredemption2/reddeadredemption2.py +++ b/reddeadredemption2/reddeadredemption2.py @@ -32,6 +32,7 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") CONFIG_FULL_PATH = Path("C:/Users/", getpass.getuser(), "Documents", "Rockstar Games", "Red Dead Redemption 2", "Settings", "system.xml") +user.FAILSAFE = False def run_benchmark(): """Starts the benchmark""" @@ -39,8 +40,15 @@ def run_benchmark(): setup_start_time = int(time.time()) exec_steam_run_command(STEAM_GAME_ID) am = ArtifactManager(LOG_DIRECTORY) + time.sleep(80) + # patch to look for seasonal popup + result = kerasService.look_for_word_vulkan("strange", attempts=30, interval=1) + if result: + user.press("enter") + time.sleep(3) + # Press Z to enter settings result = kerasService.look_for_word_vulkan("settings", attempts=30, interval=1) if not result: diff --git a/rocket_league/rocket_league.py b/rocket_league/rocket_league.py index c85d6e2..96532e7 100644 --- a/rocket_league/rocket_league.py +++ b/rocket_league/rocket_league.py @@ -30,7 +30,7 @@ CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Rocket League\\ PROCESS_NAME = "rocketleague.exe" EXECUTABLE_PATH = find_epic_executable() GAME_ID = "9773aa1aa54f4f7b80e44bef04986cea%3A530145df28a24424923f5828cc9031a1%3ASugar?action=launch&silent=true" -gamefoldername = "rocketleague" +GAMEFOLDERNAME = "rocketleague" am = ArtifactManager(LOG_DIRECTORY) gamepad = LTTGamePadDS4() @@ -63,7 +63,7 @@ def camera_cycle(max_attempts=10): :param check_duration: How long (in seconds) to look for the word before pressing a button. :param button: The gamepad button to press if word is not found. """ - for attempt in range(max_attempts): + for _ in range(max_attempts): # Try finding the word within check_duration seconds found = kerasService.look_for_word(word="player", attempts=2, interval=0.2) @@ -136,6 +136,11 @@ def run_benchmark(): gamepad.single_dpad_press(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH) time.sleep(0.5) + if kerasService.look_for_word(word="club", attempts=5, interval=0.2): + logging.info('Saw Create a Club. Navigating accordingly.') + gamepad.single_dpad_press(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH) + time.sleep(0.5) + gamepad.dpad_press_n_times(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, n=2, pause=0.8) time.sleep(0.5) gamepad.single_button_press(button=vg.DS4_BUTTONS.DS4_BUTTON_CROSS) @@ -239,7 +244,7 @@ try: "resolution": format_resolution(width, height), "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "game_version": find_eg_game_version(gamefoldername) + "game_version": find_eg_game_version(GAMEFOLDERNAME) } write_report_json(LOG_DIRECTORY, "report.json", report) diff --git a/shadowofthetombraider/shadowofthetombraider.py b/shadowofthetombraider/shadowofthetombraider.py index 1f8717d..85d41c6 100644 --- a/shadowofthetombraider/shadowofthetombraider.py +++ b/shadowofthetombraider/shadowofthetombraider.py @@ -27,6 +27,7 @@ PROCESS_NAME = "SOTTR.exe" SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") +user.FAILSAFE = False def setup_logging(): """default logging config""" diff --git a/stellaris/settings/pdx_settings.txt b/stellaris/settings/pdx_settings.txt index ac632d6..b596e26 100644 --- a/stellaris/settings/pdx_settings.txt +++ b/stellaris/settings/pdx_settings.txt @@ -20,7 +20,7 @@ version=0 } "refreshRate"={ - value="50" + value="60" version=0 } } diff --git a/stellaris/settings/settings.txt b/stellaris/settings/settings.txt new file mode 100644 index 0000000..8cd768c --- /dev/null +++ b/stellaris/settings/settings.txt @@ -0,0 +1,73 @@ +force_pow2_textures=no +language="l_english" +graphics= +{ + size= + { + x=1920 + y=1080 + } + + gui_scale=1.000000 + gui_safe_ratio=1.000000 + refreshRate=60 + fullScreen=yes + borderless=no + display_index=0 + renderer=1 + shadowSize=2048 + multi_sampling=4 + multi_sampling_quality=0 + maxanisotropy=16 + gamma=50.000000 + vsync=yes +} +sound_fx_volume=50.000000 +music_volume=50.000000 +scroll_speed=50.000000 +camera_rotation_speed=50.000000 +zoom_speed=50.000000 +mouse_speed=50.000000 +soundgroup="l_english" +master_volume=100.000000 +ambient_volume=50.000000 +dev_master_volume=75.000000 +input_type=0 +precise_mouse_wheel=no +mouse_wheel_acceleration=0.000000 +mouse_wheel_base_speed=1.000000 +input_type=0 +crisis_conversation_speech=yes +voice_volume=50.000000 +tts_volume=75.000000 +camera_look_sensitivity=1.000000 +camera_speed=50.000000 +autosave=0 +tutorial=0 +completed_tutorial_missions=0 +gfx_quality=2 +bloom= +{ + quality=2 + lens_flare=yes +} +cvaa_settings= +{ + mouse_side_button_mode=0 + cvaa_tts_enabled=no + cvaa_stt_enabled=no + general_tts_enabled=no + tooltip_tts_enabled=no + cvaa_chat_visual_alerts=no + cvaa_stt_hotkey_enabled=no + cvaa_chat_large_fonts=no + cvaa_stt_hotkey="" +} +mp_max_ticks_ahead=30 +mapmode_sectors=no +show_startup_game_info="Phoenix v4.0" +hyperlane_opacity=0.300000 +hotkey_activation_delay=0.100000 +hotkey_actualization_delay=0.100000 +hide_unowned_content=no +transfer_speed="fast" diff --git a/stellaris/settings/standard_settings.txt b/stellaris/settings/standard_settings.txt deleted file mode 100644 index 1368e66..0000000 --- a/stellaris/settings/standard_settings.txt +++ /dev/null @@ -1,45 +0,0 @@ -force_pow2_textures=no -language="l_english" -graphics={ - size={ - x=1920 - y=1080 - } - - gui_scale=1.000000 - gui_safe_ratio=1.000000 - refreshRate=50 - fullScreen=yes - borderless=no - display_index=0 - renderer=0 - shadowSize=2048 - multi_sampling=0 - multi_sampling_quality=0 - maxanisotropy=16 - gamma=50.000000 - vsync=yes -} -sound_fx_volume=20.000000 -music_volume=20.000000 -scroll_speed=50.000000 -camera_rotation_speed=50.000000 -zoom_speed=50.000000 -mouse_speed=50.000000 -soundgroup="l_english" -master_volume=52.000000 -ambient_volume=21.000000 -dev_master_volume=75.000000 -input_type=0 -voice_volume=22.000000 -tts_volume=75.000000 -camera_look_sensitivity=1.000000 -camera_speed=50.000000 -autosave=0 -tutorial=0 -completed_tutorial_missions=0 -gfx_quality=0 -bloom={ - quality=0 - lens_flare=no -} \ No newline at end of file diff --git a/stellaris/stellaris.py b/stellaris/stellaris.py index 9d8d135..9b084eb 100644 --- a/stellaris/stellaris.py +++ b/stellaris/stellaris.py @@ -23,6 +23,7 @@ from harness_utils.output import ( ) from harness_utils.steam import get_app_install_location from harness_utils.keras_service import KerasService +from harness_utils.artifacts import ArtifactManager, ArtifactType SCRIPT_DIR = Path(__file__).resolve().parent @@ -67,12 +68,38 @@ def run_benchmark(keras_host, keras_port): start_game() setup_start_time = int(time.time()) time.sleep(5) + am = ArtifactManager(LOG_DIR) + + patchnotes = keras_service.wait_for_word("close", interval=0.5, timeout=100) + if patchnotes: + gui.moveTo(patchnotes["x"], patchnotes["y"]) + time.sleep(0.2) + gui.mouseDown() + time.sleep(0.2) + gui.mouseUp() + time.sleep(0.2) result = keras_service.wait_for_word("credits", interval=0.5, timeout=100) if not result: logging.info("Could not find the paused notification. Unable to mark start time!") sys.exit(1) + result = keras_service.look_for_word("settings", attempts=10, interval=1) + if not result: + logging.info("Did not find the settings button. Is there something wrong on the screen?") + sys.exit(1) + + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.mouseDown() + time.sleep(0.2) + gui.mouseUp() + time.sleep(0.5) + am.take_screenshot("settings.png", ArtifactType.CONFIG_IMAGE, "settings") + + time.sleep(0.2) + user.press("esc") + result = keras_service.look_for_word("load", attempts=10, interval=1) if not result: logging.info("Did not find the load save menu. Is there something wrong on the screen?") @@ -102,9 +129,9 @@ def run_benchmark(keras_host, keras_port): logging.info("Could not find the paused notification. Unable to mark start time!") sys.exit(1) - result = keras_service.look_for_word("government", attempts=10, interval=1) + result = keras_service.look_for_word("overview", attempts=10, interval=1) if not result: - logging.info("Did not find the load latest save button. Did keras click correctly?") + logging.info("Did not find the overview in the corner. Did the game load?") sys.exit(1) gui.moveTo(result["x"], result["y"]) @@ -141,6 +168,8 @@ def run_benchmark(keras_host, keras_port): score = find_score_in_log() logging.info("The one year passed in %s seconds", score) terminate_processes(PROCESS_NAME) + am.create_manifest() + return test_start_time, test_end_time, score diff --git a/stellaris/stellaris_utils.py b/stellaris/stellaris_utils.py index 4b88b8b..7a0c00d 100644 --- a/stellaris/stellaris_utils.py +++ b/stellaris/stellaris_utils.py @@ -16,15 +16,16 @@ PROCESS_NAME = "stellaris.exe" STEAM_GAME_ID = 281990 CONFIG_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris") LOG_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\logs") -BENCHMARK_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\save games\\BENCHMARK") -CONFIG_FILENAME = "standard_settings.txt" +BENCHMARK_LOCATION = Path( + f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\save games\\BENCHMARK") +CONFIG_FILENAME = "settings.txt" LOG_FILE = "game.log" benchmark_files = [ "benchmark.ini", "pdx_settings.txt", - "standard_settings.txt" + "settings.txt" ] @@ -76,7 +77,7 @@ def copy_benchmarkfiles() -> None: def copy_save_from_network_drive(file_name, destination): """copy save file from network drive""" - network_dir = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Stellaris") + network_dir = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Stellaris") source_path = network_dir.joinpath(file_name) logging.info("Copying %s from %s", file_name, source_path) shutil.copyfile(source_path, destination) diff --git a/superposition/superposition.py b/superposition/superposition.py index f9af792..12026ad 100644 --- a/superposition/superposition.py +++ b/superposition/superposition.py @@ -43,21 +43,22 @@ formatter = logging.Formatter(LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger('').addHandler(console) -cmd = f'{INSTALL_DIR}\\{EXECUTABLE}' -argstr = f"-fullscreen 1 -mode default -api {args.api} -quality {args.preset} -iterations 1" -argstr += f" -log_txt {log_dir}\\log.txt" +CMD = f'{INSTALL_DIR}\\{EXECUTABLE}' +ARGSTR = f"-fullscreen 1 -mode default -api {args.api} -quality {args.preset} -iterations 1" +ARGSTR += f" -log_txt {log_dir}\\log.txt" -logging.info(cmd) -logging.info(argstr) -argies = argstr.split(" ") -cmd = cmd.rstrip() -with Popen([cmd, *argies]) as process: +logging.info(CMD) +logging.info(ARGSTR) +argies = ARGSTR.split(" ") +CMD = CMD.rstrip() +with Popen([CMD, *argies]) as process: EXIT_CODE = process.wait() if EXIT_CODE > 0: logging.error("Test failed!") sys.exit(EXIT_CODE) +SCORE = "" pattern = re.compile(r"Score: (\d+)") log_path = os.path.join(log_dir, "log.txt") with open(log_path, encoding="utf-8") as log: @@ -65,11 +66,13 @@ with open(log_path, encoding="utf-8") as log: for line in lines: match = pattern.search(line) if match: - score = match.group(1) + SCORE = match.group(1) report = { - "test": f"Unigine Superposition 2017 {args.preset} ${args.api}", - "score": score, + "test": "Unigine Superposition", + "test_parameter": f"{args.api}", + "test_preset": args.preset, + "score": SCORE, "unit": "score" } diff --git a/the_last_of_us_part_i/the_last_of_us_part_i.py b/the_last_of_us_part_i/the_last_of_us_part_i.py index 0e67136..b760049 100644 --- a/the_last_of_us_part_i/the_last_of_us_part_i.py +++ b/the_last_of_us_part_i/the_last_of_us_part_i.py @@ -20,8 +20,8 @@ from harness_utils.output import ( ) from harness_utils.process import terminate_processes from harness_utils.steam import ( - get_registry_active_user, - exec_steam_run_command, + get_registry_active_user, + exec_steam_run_command, ) from harness_utils.misc import press_n_times @@ -34,19 +34,20 @@ PROCESS_NAME = "tlou" user.FAILSAFE = False + def take_screenshots(am: ArtifactManager) -> None: """Take screenshots of the benchmark settings""" logging.info("Taking screenshots of benchmark settings") - press_n_times("s",2,0.2 ) + press_n_times("s", 2, 0.2) user.press("enter") - press_n_times("s",4,0.2 ) + press_n_times("s", 4, 0.2) user.press("enter") am.take_screenshot("video1.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings1") - press_n_times("s",15,0.2) + press_n_times("s", 15, 0.2) am.take_screenshot("video2.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings2") - press_n_times("s",6, 0.2) + press_n_times("s", 6, 0.2) am.take_screenshot("video3.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings3") user.press("backspace") @@ -69,6 +70,7 @@ def take_screenshots(am: ArtifactManager) -> None: user.press("backspace") press_n_times("w", 2, 0.2) + def navigate_main_menu(am: ArtifactManager) -> None: """Input to navigate main menu""" logging.info("Navigating main menu") @@ -174,7 +176,7 @@ try: start_time, end_time = run_benchmark() steam_id = get_registry_active_user() config_path = os.path.join( - os.environ["HOMEPATH"], "Saved Games" ,"The Last of Us Part I", + os.environ["HOMEPATH"], "Saved Games", "The Last of Us Part I", "users", str(steam_id), "screeninfo.cfg" ) height, width = get_resolution(config_path) @@ -183,8 +185,8 @@ try: "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIRECTORY, "report.json", report) + except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/the_last_of_us_part_ii/manifest.yaml b/the_last_of_us_part_ii/manifest.yaml new file mode 100644 index 0000000..f530266 --- /dev/null +++ b/the_last_of_us_part_ii/manifest.yaml @@ -0,0 +1,9 @@ +friendly_name: "The Last of Us Part II" +executable: "tlou2.py" +process_name: "tlou-ii.exe" +output_dir: "run" +options: + - name: kerasHost + type: input + - name: kerasPort + type: input diff --git a/the_last_of_us_part_ii/tlou2.py b/the_last_of_us_part_ii/tlou2.py new file mode 100644 index 0000000..6c23d3a --- /dev/null +++ b/the_last_of_us_part_ii/tlou2.py @@ -0,0 +1,286 @@ +"""The Last of Us Part I test script""" +import logging +from pathlib import Path +import time +import sys +import pydirectinput as user +import getpass + +import winreg # for accessing settings, including resolution, in the registry + +import shutil + +sys.path.insert(1, str(Path(sys.path[0]).parent)) + +from harness_utils.keras_service import KerasService +from harness_utils.output import ( + format_resolution, + seconds_to_milliseconds, + write_report_json, + setup_logging, +) +from harness_utils.process import terminate_processes +from harness_utils.steam import ( + exec_steam_run_command, +) + +from harness_utils.artifacts import ArtifactManager, ArtifactType + +from harness_utils.misc import ( + int_time, + find_word, + press_n_times, + keras_args) + +USERNAME = getpass.getuser() +STEAM_GAME_ID = 2531310 +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +LOG_DIRECTORY = SCRIPT_DIRECTORY / "run" +PROCESS_NAME = "tlou-ii.exe" + +user.FAILSAFE = False + + +def reset_savedata(): + """ + Deletes the savegame folder from the local directory and replaces it with a new one from the network drive. + """ + local_savegame_path = Path( + f"C:\\Users\\{USERNAME}\\Documents\\The Last of Us Part II\\76561199405246658\\savedata") # make this global + network_savegame_path = Path( + r"\\labs.lmg.gg\Labs\03_ProcessingFiles\The Last of Us Part II\savedata") + + # Delete the local savedata folder if it exists + if local_savegame_path.exists() and local_savegame_path.is_dir(): + shutil.rmtree(local_savegame_path) + logging.info("Deleted local savedata folder: %s", local_savegame_path) + + # Copy the savedata folder from the network drive + try: + shutil.copytree(network_savegame_path, local_savegame_path) + logging.info("Copied savedata folder from %s to %s", + network_savegame_path, local_savegame_path) + except Exception as e: + logging.error("Failed to copy savedata folder: %s", e) + + # Check if the newly copied directory contains a folder called SAVEFILE0A + + +def delete_autosave(): + """ + Deletes the autosave folder from the local directory if it exists. + """ + local_savegame_path = Path( + f"C:\\Users\\{USERNAME}\\Documents\\The Last of Us Part II\\76561199405246658\\savedata") + savefile_path = local_savegame_path / "SAVEFILE0A" # check for autosaved file, delete if exists + if savefile_path.exists() and savefile_path.is_dir(): + shutil.rmtree(savefile_path) + logging.info("Deleted folder: %s", savefile_path) + + +def get_current_resolution(): + """ + Returns: + tuple: (width, height) + Reads resolutions settings from registry + """ + key_path = r"Software\Naughty Dog\The Last of Us Part II\Graphics" + fullscreen_width = read_registry_value(key_path, "FullscreenWidth") + fullscreen_height = read_registry_value(key_path, "FullscreenHeight") + + return (fullscreen_width, fullscreen_height) + + +def read_registry_value(key_path, value_name): + """ + Reads value from registry + A helper function for get_current_resolution + """ + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path) as key: + value, _ = winreg.QueryValueEx(key, value_name) + return value + except FileNotFoundError: + logging.error("Registry key not found: %s", value_name) + return None + except OSError as e: + logging.error("Error reading registry value: %s", e) + return None + + +def run_benchmark(keras_service: KerasService) -> tuple: + """Starts Game, Sets Settings, and Runs Benchmark""" + exec_steam_run_command(STEAM_GAME_ID) + setup_start_time = int_time() + am = ArtifactManager(LOG_DIRECTORY) + + if keras_service.wait_for_word(word="sony", timeout=60, interval=0.2) is None: + logging.error("Couldn't find 'sony'") + else: + user.press("escape") + + find_word(keras_service, "story", "Couldn't find main menu : 'story'") + + press_n_times("down", 2) + + # navigate settings + navigate_settings(am, keras_service) + + find_word(keras_service, "story", "Couldn't find main menu the second time : 'story'") + + press_n_times("up", 2) + + user.press("space") + + time.sleep(0.3) + + user.press("space") + + if keras_service.wait_for_word(word="continue", timeout=5, interval=0.2) is None: + user.press("down") + else: + press_n_times("down", 2) + + delete_autosave() + + time.sleep(0.3) + + user.press("space") + + time.sleep(0.3) + + if keras_service.wait_for_word(word="autosave", timeout=5, interval=0.2) is None: + + user.press("space") + + else: + user.press("up") + + time.sleep(0.3) + + user.press("space") + + time.sleep(0.3) + + user.press("left") + + time.sleep(0.3) + + user.press("space") + + setup_end_time = test_start_time = test_end_time = int_time() + + elapsed_setup_time = setup_end_time - setup_start_time + logging.info("Setup took %f seconds", elapsed_setup_time) + + # time of benchmark usually is 4:23 = 263 seconds + + if keras_service.wait_for_word(word="man", timeout=100, interval=0.2) is not None: + + test_start_time = int_time() - 14 + time.sleep(240) + + else: + + logging.error("couldn't find 'man'") + time.sleep(150) + + if keras_service.wait_for_word(word="rush", timeout=100, interval=0.2) is not None: + + time.sleep(3) + test_end_time = int_time() + + else: + + logging.error("couldn't find 'rush', marks end of benchmark") + test_end_time = int_time() + + elapsed_test_time = test_end_time - test_start_time + logging.info("Test took %f seconds", elapsed_test_time) + + terminate_processes(PROCESS_NAME) + + am.create_manifest() + + return test_start_time, test_end_time + + +def navigate_settings(am: ArtifactManager, keras: KerasService) -> None: + """Navigate through settings and take screenshots. + Exits to main menu after taking screenshots. + """ + + user.press("space") + + find_word(keras, "display", "Couldn't find display") + + time.sleep(5) # slow cards may miss the first down + + press_n_times("down", 4) + + user.press("space") + + time.sleep(0.5) + + find_word(keras, "resolution", "Couldn't find resolution") + + am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "display settings 1") + + user.press("up") + + find_word(keras, "brightness", "Couldn't find brightness") + + am.take_screenshot("display2.png", ArtifactType.CONFIG_IMAGE, "display settings 2") + + user.press("q") # swaps to graphics settings + + time.sleep(0.5) + + find_word(keras, "preset", "Couldn't find preset") + + am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "graphics settings 1") + + user.press("up") + + find_word(keras, "dirt", "Couldn't find dirt") + + am.take_screenshot("graphics3.png", ArtifactType.CONFIG_IMAGE, + "graphics settings 3") # is at the bottom of the menu + + press_n_times("up", 13) + + find_word(keras, "scattering", "Couldn't find scattering") + + am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "graphics settings 2") + + press_n_times("escape", 2) + + +def main(): + """Main function to run the benchmark""" + try: + logging.info("Starting The Last of Us Part II benchmark") + + keras_service = KerasService(keras_args().keras_host, keras_args().keras_port) + + reset_savedata() + + start_time, end_time = run_benchmark(keras_service) + resolution_tuple = get_current_resolution() + report = { + "resolution": format_resolution(resolution_tuple[0], resolution_tuple[1]), + "start_time": seconds_to_milliseconds(start_time), # secconds to miliseconds + "end_time": seconds_to_milliseconds(end_time), + } + write_report_json(LOG_DIRECTORY, "report.json", report) + + except Exception as e: + logging.error("An error occurred: %s", e) + logging.exception(e) + terminate_processes(PROCESS_NAME) + sys.exit(1) + + +if __name__ == "__main__": + setup_logging(LOG_DIRECTORY) + main() diff --git a/tinytinaswonderland/tinytinaswonderland.py b/tinytinaswonderland/tinytinaswonderland.py index 85dd0c2..d40d067 100644 --- a/tinytinaswonderland/tinytinaswonderland.py +++ b/tinytinaswonderland/tinytinaswonderland.py @@ -22,6 +22,8 @@ LOG_DIRECTORY = SCRIPT_DIRECTORY.joinpath("run") STEAM_GAME_ID = 1286680 EXECUTABLE = "Wonderlands.exe" +user.FAILSAFE = False + def setup_logging(): """default logging config""" setup_log_directory(LOG_DIRECTORY) diff --git a/xz/xz_utils.py b/xz/xz_utils.py index ea340e1..e3ba847 100644 --- a/xz/xz_utils.py +++ b/xz/xz_utils.py @@ -16,12 +16,12 @@ def xz_executable_exists() -> bool: def copy_from_network_drive(): """Download xz from network drive""" - source = r"\\Labs\labs\01_Installers_Utilities\xz\xz_5.6.2_x86_64.exe" + source = r"\\labs.lmg.gg\labs\01_Installers_Utilities\xz\xz_5.6.2_x86_64.exe" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, XZ_EXECUTABLE) shutil.copyfile(source, destination) - source = r"\\Labs\labs\03_ProcessingFiles\Compression\tq_dlss_explained_1080p.mp4" + source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Compression\tq_dlss_explained_1080p.mp4" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, "tq_dlss_explained_1080p.mp4") shutil.copyfile(source, destination) diff --git a/ycruncher/ycruncher.py b/ycruncher/ycruncher.py index bf775ea..328da42 100644 --- a/ycruncher/ycruncher.py +++ b/ycruncher/ycruncher.py @@ -12,8 +12,8 @@ sys.path.insert(1, os.path.join(sys.path[0], "..")) from harness_utils.output import write_report_json, DEFAULT_LOGGING_FORMAT, DEFAULT_DATE_FORMAT SCRIPT_DIR = Path(__file__).resolve().parent -LOG_DIR = SCRIPT_DIR.joinpath("run") -EXECUTABLE_PATH = SCRIPT_DIR.joinpath(YCRUNCHER_FOLDER_NAME, "y-cruncher.exe") +LOG_DIR = SCRIPT_DIR / "run" +EXECUTABLE_PATH = SCRIPT_DIR / YCRUNCHER_FOLDER_NAME / "y-cruncher.exe" def setup_logging(): @@ -79,7 +79,7 @@ def main(): report = { "start_time": start_time, - "version": "v0.8.5.9543", + "version": "v0.8.5.9545b", "end_time": end_time, "score": avg_score, "unit": "seconds", diff --git a/ycruncher/ycruncher_utils.py b/ycruncher/ycruncher_utils.py index 67eb9b2..fd42828 100644 --- a/ycruncher/ycruncher_utils.py +++ b/ycruncher/ycruncher_utils.py @@ -4,11 +4,11 @@ from zipfile import ZipFile from pathlib import Path import requests -YCRUNCHER_FOLDER_NAME = "y-cruncher v0.8.5.9543" -YCRUNCHER_ZIP_NAME = "y-cruncher.v0.8.5.9543.zip" - +YCRUNCHER_FOLDER_NAME = "y-cruncher v0.8.6.9545" +YCRUNCHER_ZIP_NAME = "y-cruncher.v0.8.6.9545b.zip" SCRIPT_DIR = Path(__file__).resolve().parent + def ycruncher_folder_exists() -> bool: """Check if ycruncher has been downloaded or not""" return SCRIPT_DIR.joinpath(YCRUNCHER_FOLDER_NAME).is_dir() @@ -16,7 +16,7 @@ def ycruncher_folder_exists() -> bool: def download_ycruncher(): """Download and extract Y-Cruncher""" - download_url = "https://github.com/Mysticial/y-cruncher/releases/download/v0.8.5.9543/y-cruncher.v0.8.5.9543.zip" + download_url = "https://github.com/Mysticial/y-cruncher/releases/download/v0.8.6.9545/y-cruncher.v0.8.6.9545b.zip" destination = SCRIPT_DIR / YCRUNCHER_ZIP_NAME response = requests.get(download_url, allow_redirects=True, timeout=180) with open(destination, 'wb') as file: @@ -24,7 +24,7 @@ def download_ycruncher(): with ZipFile(destination, 'r') as zip_object: zip_object.extractall(path=SCRIPT_DIR) + def current_time_ms(): """Get current timestamp in milliseconds since epoch""" return int(time.time() * 1000) - \ No newline at end of file diff --git a/deprecated/F1_22/README.md b/zdeprecated/F1_22/README.md similarity index 100% rename from deprecated/F1_22/README.md rename to zdeprecated/F1_22/README.md diff --git a/deprecated/F1_22/f1.py b/zdeprecated/F1_22/f1.py similarity index 100% rename from deprecated/F1_22/f1.py rename to zdeprecated/F1_22/f1.py diff --git a/deprecated/F1_22/f1_22_utils.py b/zdeprecated/F1_22/f1_22_utils.py similarity index 100% rename from deprecated/F1_22/f1_22_utils.py rename to zdeprecated/F1_22/f1_22_utils.py diff --git a/deprecated/F1_22/manifest.yaml b/zdeprecated/F1_22/manifest.yaml similarity index 100% rename from deprecated/F1_22/manifest.yaml rename to zdeprecated/F1_22/manifest.yaml diff --git a/F1_23/README.md b/zdeprecated/F1_23/README.md similarity index 100% rename from F1_23/README.md rename to zdeprecated/F1_23/README.md diff --git a/F1_23/f1_23.py b/zdeprecated/F1_23/f1_23.py similarity index 100% rename from F1_23/f1_23.py rename to zdeprecated/F1_23/f1_23.py diff --git a/F1_23/f1_23_utils.py b/zdeprecated/F1_23/f1_23_utils.py similarity index 100% rename from F1_23/f1_23_utils.py rename to zdeprecated/F1_23/f1_23_utils.py diff --git a/F1_23/manifest.yaml b/zdeprecated/F1_23/manifest.yaml similarity index 100% rename from F1_23/manifest.yaml rename to zdeprecated/F1_23/manifest.yaml diff --git a/deprecated/README.md b/zdeprecated/README.md similarity index 100% rename from deprecated/README.md rename to zdeprecated/README.md diff --git a/deprecated/aida64gpgpu/README.md b/zdeprecated/aida64gpgpu/README.md similarity index 100% rename from deprecated/aida64gpgpu/README.md rename to zdeprecated/aida64gpgpu/README.md diff --git a/deprecated/aida64gpgpu/aida64_gpgpu_benchmark.png b/zdeprecated/aida64gpgpu/aida64_gpgpu_benchmark.png similarity index 100% rename from deprecated/aida64gpgpu/aida64_gpgpu_benchmark.png rename to zdeprecated/aida64gpgpu/aida64_gpgpu_benchmark.png diff --git a/deprecated/aida64gpgpu/aida64gpgpu.py b/zdeprecated/aida64gpgpu/aida64gpgpu.py similarity index 90% rename from deprecated/aida64gpgpu/aida64gpgpu.py rename to zdeprecated/aida64gpgpu/aida64gpgpu.py index ef5dd8f..f36d370 100644 --- a/deprecated/aida64gpgpu/aida64gpgpu.py +++ b/zdeprecated/aida64gpgpu/aida64gpgpu.py @@ -23,8 +23,8 @@ logging.getLogger('').addHandler(console) executable = os.path.join(INSTALL_DIR, EXECUTABLE) report_dest = os.path.join(log_dir, "report.xml") -argstr = f"/GGBENCH {report_dest}" -result = subprocess.run([executable, "/GGBENCH", report_dest], check=False) +ARGSTR = f"/GGBENCH {report_dest}" +result = subprocess.run([executable, ARGSTR], check=False) if result.returncode > 0: logging.error("Aida failed with exit code {result.returncode}") diff --git a/deprecated/aida64gpgpu/manifest.yaml b/zdeprecated/aida64gpgpu/manifest.yaml similarity index 100% rename from deprecated/aida64gpgpu/manifest.yaml rename to zdeprecated/aida64gpgpu/manifest.yaml diff --git a/deprecated/aida64gpgpu/sample_gpgpu.xml b/zdeprecated/aida64gpgpu/sample_gpgpu.xml similarity index 100% rename from deprecated/aida64gpgpu/sample_gpgpu.xml rename to zdeprecated/aida64gpgpu/sample_gpgpu.xml diff --git a/zz_non_game_harness_template/harness.py b/zz_non_game_harness_template/harness.py new file mode 100644 index 0000000..eaa6f9f --- /dev/null +++ b/zz_non_game_harness_template/harness.py @@ -0,0 +1 @@ +# This is a non-game harness template