diff --git a/.gitignore b/.gitignore index 386cc3d..d91f24a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ flac-1.4.3-win.zip primesieve-12.3* y-cruncher v0.8.2.9522/ y-cruncher v0.8.2.9522.zip +y-cruncher v0.8.6.9545/ +y-cruncher.v0.8.6.9545b.zip basegame_no_intro_videos.archive *.blend 0001.png @@ -31,6 +33,8 @@ godot-4.2.1-stable/ godot-4.2.1-stable.zip godot-4.3-stable/ godot-4.3-stable.zip +godot-4.4.1-stable/ +godot-4.4.1-stable.zip mingw64/ # python diff --git a/3dmark/ul3dmark.py b/3dmark/ul3dmark.py index 576fcf9..ff60ab9 100644 --- a/3dmark/ul3dmark.py +++ b/3dmark/ul3dmark.py @@ -23,7 +23,7 @@ from harness_utils.process import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -34,25 +34,25 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "TimeSpy": { "config": CONFIG_DIR / "timespy.3dmdef", - "process_name": "3DMarkTimeSpy.exe", + "process_name": "3DMarkTimeSpy.exe", "score_name": "TimeSpyPerformanceGraphicsScore", "test_name": "3DMark Time Spy" }, "FireStrike": { "config": CONFIG_DIR / "firestrike.3dmdef", - "process_name": "3DMarkICFWorkload.exe", + "process_name": "3DMarkICFWorkload.exe", "score_name": "firestrikegraphicsscorep", "test_name": "3DMark Fire Strike" }, "PortRoyal": { "config": CONFIG_DIR / "portroyal.3dmdef", - "process_name": "3DMarkPortRoyal.exe", + "process_name": "3DMarkPortRoyal.exe", "score_name": "PortRoyalPerformanceGraphicsScore", "test_name": "3DMark Port Royal" }, "SolarBay": { "config": CONFIG_DIR / "solarbay.3dmdef", - "process_name": "3DMarkSolarBay.exe", + "process_name": "3DMarkSolarBay.exe", "score_name": "SolarBayPerformanceGraphicsScore", "test_name": "3DMark Solar Bay" } @@ -60,9 +60,10 @@ BENCHMARK_CONFIG = { RESULTS_FILENAME = "myresults.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -76,8 +77,9 @@ def setup_logging(): def get_arguments(): """get arguments""" parser = ArgumentParser() - parser.add_argument( - "--benchmark", dest="benchmark", help="Benchmark test type", required=True, choices=BENCHMARK_CONFIG.keys()) + parser.add_argument("--benchmark", dest="benchmark", + help="Benchmark test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -97,16 +99,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 30: #seconds + if elapsed >= 30: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() args = get_arguments() @@ -121,7 +124,9 @@ try: logging.error("3DMark exited with return code %d", pr.returncode) sys.exit(pr.returncode) - score = get_score(BENCHMARK_CONFIG[args.benchmark]["score_name"], REPORT_PATH) + score = get_score( + BENCHMARK_CONFIG[args.benchmark]["score_name"], + REPORT_PATH) if score is None: logging.error("Could not find average FPS output!") sys.exit(1) @@ -132,7 +137,8 @@ try: logging.info("Score was %s", score) report = { - "test": BENCHMARK_CONFIG[args.benchmark]["test_name"], + "test": "3DMark", + "test_parameter": args.benchmark, "unit": "score", "score": score, "start_time": seconds_to_milliseconds(strt), diff --git a/7z/sevenzip.py b/7z/sevenzip.py index 85dcdf7..a25b89a 100644 --- a/7z/sevenzip.py +++ b/7z/sevenzip.py @@ -26,7 +26,7 @@ formatter = logging.Formatter(LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger('').addHandler(console) -EXECUTABLE = "7zr_24.07.exe" +EXECUTABLE = "7zr_25.00.exe" ABS_EXECUTABLE_PATH = os.path.join( os.path.dirname(os.path.realpath(__file__)), EXECUTABLE) @@ -35,18 +35,18 @@ if os.path.isfile(ABS_EXECUTABLE_PATH) is False: "7-Zip executable not found, downloading from network drive") copy_from_network_drive() -command = f'{ABS_EXECUTABLE_PATH}' -command = command.rstrip() +COMMAND = f'{ABS_EXECUTABLE_PATH}' +COMMAND = COMMAND.rstrip() t1 = time.time() logging.info("Starting 7-Zip benchmark! This may take a minute or so...") -with Popen([command, "b", "3"], cwd=os.path.dirname( +with Popen([COMMAND, "b", "3"], cwd=os.path.dirname( os.path.realpath(__file__)), stdout=subprocess.PIPE) as process: stdout_data, stderr = process.communicate() list_of_strings = stdout_data.decode('utf-8').splitlines() SPEED_PATTERN = r'^Avr:\s*([0-9]*)\s.*\|\s*([0-9]*)\s.*$' - VERSION_PATTERN = r'7-Zip (\d+\.\d+).*' + VERSION_PATTERN = r'7-Zip \(r\) (\d+\.\d+).*' VERSION = "" SPEED_C = "" diff --git a/7z/sevenzip_utils.py b/7z/sevenzip_utils.py index 32db283..5a37680 100644 --- a/7z/sevenzip_utils.py +++ b/7z/sevenzip_utils.py @@ -6,7 +6,7 @@ import shutil def copy_from_network_drive(): """Download 7zip from network drive""" - source = r"\\Labs\labs\01_Installers_Utilities\7ZIP\7zr_24.07.exe" + source = r"\\labs.lmg.gg\labs\01_Installers_Utilities\7ZIP\7zr_25.00.exe" root_dir = os.path.dirname(os.path.realpath(__file__)) - destination = os.path.join(root_dir, "7zr_24.07.exe") + destination = os.path.join(root_dir, "7zr_25.00.exe") shutil.copyfile(source, destination) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b86bd3..ed238a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,16 +6,20 @@ Changes are grouped by the date they are merged to the main branch of the reposi ## 2025-05-13 - Add Evolve Benchmark test harness +## 2025-07-15 +- Updated 7-Zip to 25.00 +- Updated Y-Cruncher to v0.8.6.9545 +- Updated Godot compile to 4.4.1-stable ## 2025-04-02 - Fixed Keras not finding the FPS in Shadow of the Tomb Raider - Added a screenshot function for Vulkan games for Keras-OCR via DXcam - Added Keras functionality to Red Dead Redemption 2 - Added Strange Brigade (VK) to the team -- Updated PugetBench harness to include Davinci and After Effects +- Updated PugetBench harness to include DaVinci and After Effects - Updated PugetBench to more consistently find version numbers and include them in the report.json - Updated Rocket League harness to check what camera it is on and keep flipping through till it's on the correct one -- Updated Procyon AI harnesses to have verison numbers in report.json +- Updated Procyon AI harnesses to have version numbers in report.json - Replaced the hardcoded path for Cyberpunk2077 and instead use the get_app_install_location instead - Added DOTA 2 screenshotting for video config - Added beta harness of Marvel Rivals diff --git a/F1_24/f1_24.py b/F1_24/f1_24.py index 914c111..a25e1f8 100644 --- a/F1_24/f1_24.py +++ b/F1_24/f1_24.py @@ -40,6 +40,7 @@ intro_videos = [ os.path.join(VIDEO_PATH, "attract.bk2"), os.path.join(VIDEO_PATH, "cm_f1_sting.bk2") ] +user.FAILSAFE = False def find_latest_result_file(base_path): diff --git a/alanwake2/alanwake2.py b/alanwake2/alanwake2.py index fbfbeae..303f196 100644 --- a/alanwake2/alanwake2.py +++ b/alanwake2/alanwake2.py @@ -24,8 +24,9 @@ LOG_DIRECTORY = SCRIPT_DIRECTORY.joinpath("run") PROCESS_NAME = "alanwake2.exe" EXECUTABLE_PATH = find_epic_executable() GAME_ID = "c4763f236d08423eb47b4c3008779c84%3A93f2a8c3547846eda966cb3c152a026e%3Adc9d2e595d0e4650b35d659f90d41059?action=launch&silent=true" -gamefoldername = "AlanWake2" +GAMEFOLDERNAME = "AlanWake2" +user.FAILSAFE = False def setup_logging(): """default logging config""" @@ -169,7 +170,7 @@ try: "resolution": f"{width}x{height}", "start_time": round((start_time * 1000)), "end_time": round((end_time * 1000)), - "game_version": find_eg_game_version(gamefoldername) + "game_version": find_eg_game_version(GAMEFOLDERNAME) } am.create_manifest() diff --git a/aots-e/aotse.py b/aots-e/aotse.py index 1bd85e5..13bf0e1 100644 --- a/aots-e/aotse.py +++ b/aots-e/aotse.py @@ -6,13 +6,13 @@ import time import getpass import glob import os -from aotse_utils import read_current_resolution, find_score_in_log, delete_old_scores, get_args +from aotse_utils import read_current_resolution, find_score_in_log, delete_old_scores, get_args, replace_exe, restore_exe PARENT_DIR = str(Path(sys.path[0], "..")) sys.path.append(PARENT_DIR) from harness_utils.keras_service import KerasService -from harness_utils.steam import get_app_install_location, get_build_id, exec_steam_game +from harness_utils.steam import get_build_id, exec_steam_game from harness_utils.output import ( DEFAULT_DATE_FORMAT, DEFAULT_LOGGING_FORMAT, @@ -32,7 +32,7 @@ CONFIG_FILENAME = "settings.ini" STEAM_GAME_ID = 507490 SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" -EXECUTABLE = "AshesEscalation_DX12.exe" +EXECUTABLE = "StardockLauncher.exe" CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "GPU_Benchmark": { @@ -49,7 +49,6 @@ BENCHMARK_CONFIG = { } } CFG = f"{CONFIG_PATH}\\{CONFIG_FILENAME}" -GAME_DIR = get_app_install_location(STEAM_GAME_ID) def start_game(): """Launch the game with no launcher or start screen""" @@ -60,6 +59,7 @@ def run_benchmark(): """Start the benchmark""" # Start game via Steam and enter fullscreen mode setup_start_time = time.time() + replace_exe() start_game() time.sleep(10) @@ -85,16 +85,14 @@ def run_benchmark(): logging.info("Benchmark started. Waiting for benchmark to complete.") time.sleep(180) - # result = kerasService.wait_for_word("complete", timeout=240, interval=0.5) - # if not result: - # logging.info("Did not see the Benchmark Complete pop up. Did it run?") - # sys.exit(1) - test_end_time = time.time() - 2 + test_end_time = time.time() time.sleep(2) elapsed_test_time = round((test_end_time - test_start_time), 2) logging.info("Benchmark took %f seconds", elapsed_test_time) time.sleep(3) + restore_exe() + return test_start_time, test_end_time setup_log_directory(LOG_DIR) diff --git a/aots-e/aotse_utils.py b/aots-e/aotse_utils.py index 79e6820..b2a32a3 100644 --- a/aots-e/aotse_utils.py +++ b/aots-e/aotse_utils.py @@ -8,6 +8,7 @@ from pathlib import Path import psutil import glob import time +import shutil from argparse import ArgumentParser PARENT_DIR = str(Path(sys.path[0], "..")) @@ -15,14 +16,16 @@ sys.path.append(PARENT_DIR) sys.path.insert(1, os.path.join(sys.path[0], '..')) +from harness_utils.steam import get_app_install_location + USERNAME = getpass.getuser() SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") -PROCESS_NAME = "stellaris.exe" -STEAM_GAME_ID = 281990 +STEAM_GAME_ID = 507490 CONFIG_FILENAME = "settings.ini" USERNAME = getpass.getuser() CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Ashes of the Singularity - Escalation") +EXE_PATH = Path(get_app_install_location(STEAM_GAME_ID)) BENCHMARK_CONFIG = { "GPU_Benchmark": { "hardware": "GPU", @@ -119,3 +122,36 @@ def wait_for_benchmark_process(test_name, process_name, timeout=60): # Wait for 1 second before checking again time.sleep(1) + +def replace_exe(): + """Replaces the Strange Brigade launcher exe with the Vulkan exe for immediate launching + """ + check_backup = Path(f"{EXE_PATH}\\StardockLauncher_launcher.exe") + launcher_exe = Path(f"{EXE_PATH}\\StardockLauncher.exe") + dx12_exe = Path(f"{EXE_PATH}\\AshesEscalation_DX12.exe") + if not os.path.exists(check_backup): + os.rename(launcher_exe, check_backup) + shutil.copy(dx12_exe, launcher_exe) + logging.info("Replacing launcher file in %s", EXE_PATH) + elif os.path.exists(check_backup): + if not os.path.exists(launcher_exe): + shutil.copy(dx12_exe, launcher_exe) + logging.info("Replacing launcher file in %s", EXE_PATH) + else: + logging.info("Launcher already replaced with DX12 exe.") + +def restore_exe(): + """Restores the launcher exe back to the original exe name to close the loop. + """ + check_backup = Path(f"{EXE_PATH}\\StardockLauncher_launcher.exe") + launcher_exe = Path(f"{EXE_PATH}\\StardockLauncher.exe") + if not os.path.exists(check_backup): + logging.info("Launcher already restored or file does not exist.") + elif os.path.exists(check_backup): + if not os.path.exists(launcher_exe): + os.rename(check_backup, launcher_exe) + logging.info("Restoring launcher file in %s", EXE_PATH) + else: + os.remove(launcher_exe) + os.rename(check_backup, launcher_exe) + logging.info("Restoring launcher file in %s", EXE_PATH) diff --git a/aots-e/manifest.yaml b/aots-e/manifest.yaml index 04e505b..2612f7a 100644 --- a/aots-e/manifest.yaml +++ b/aots-e/manifest.yaml @@ -1,6 +1,6 @@ friendly_name: "Ashes of the Singularity: Escalation" executable: "aotse.py" -process_name: "AshesEscalation_DX12.exe" +process_name: "StardockLauncher.exe" output_dir: "run" options: - name: kerasHost diff --git a/assassins_creed_shadows/acshadows.py b/assassins_creed_shadows/acshadows.py index 5f60997..541a0e5 100644 --- a/assassins_creed_shadows/acshadows.py +++ b/assassins_creed_shadows/acshadows.py @@ -1,36 +1,36 @@ # pylint: disable=missing-module-docstring -from argparse import ArgumentParser import logging -import os from pathlib import Path import time import sys import re import pydirectinput as user import getpass - -sys.path.insert(1, os.path.join(sys.path[0], '..')) +sys.path.insert(1, str(Path(sys.path[0]).parent)) # pylint: disable=wrong-import-position from harness_utils.process import terminate_processes from harness_utils.output import ( format_resolution, - setup_log_directory, + setup_logging, write_report_json, seconds_to_milliseconds, - DEFAULT_LOGGING_FORMAT, - DEFAULT_DATE_FORMAT ) from harness_utils.steam import get_build_id, exec_steam_game from harness_utils.keras_service import KerasService from harness_utils.artifacts import ArtifactManager, ArtifactType -from harness_utils.misc import press_n_times -USERNAME = getpass.getuser() +from harness_utils.misc import ( + press_n_times, + int_time, + find_word, + keras_args) -SCRIPT_DIR = Path(__file__).resolve().parent -LOG_DIR = SCRIPT_DIR.joinpath("run") -PROCESS_NAME = "ACShadows.exe" +USERNAME = getpass.getuser() STEAM_GAME_ID = 3159330 +SCRIPT_DIR = Path(__file__).resolve().parent +LOG_DIR = SCRIPT_DIR / "run" +PROCESS_NAME = "ACShadows.exe" + CONFIG_LOCATION = f"C:\\Users\\{USERNAME}\\Documents\\Assassin's Creed Shadows" CONFIG_FILENAME = "ACShadows.ini" @@ -56,40 +56,30 @@ def read_current_resolution(): return (height_value, width_value) -def find_word(keras_service, word, msg, timeout=30, interval=1): - """function to call keras """ - if keras_service.wait_for_word( - word=word, timeout=timeout, interval=interval) is None: - logging.info(msg) - sys.exit(1) - - -def int_time(): - """rounds time to int""" - return int(time.time()) - def delete_videos(): """deletes intro videos""" - base_dir = r"C:\Program Files (x86)\Steam\steamapps\common\Assassin's Creed Shadows" - videos_dir = os.path.join(base_dir, "videos") - videos_en_dir = os.path.join(videos_dir, "en") + base_dir = Path( + r"C:\Program Files (x86)\Steam\steamapps\common\Assassin's Creed Shadows") + videos_dir = base_dir / "videos" + videos_en_dir = videos_dir / "en" # List of video files to delete videos_to_delete = [ - os.path.join(videos_dir, "ANVIL_Logo.webm"), - os.path.join(videos_dir, "INTEL_Logo.webm"), - os.path.join(videos_dir, "HUB_Bootflow_FranchiseIntro.webm"), - os.path.join(videos_dir, "UbisoftLogo.webm"), - os.path.join(videos_en_dir, "Epilepsy.webm"), - os.path.join(videos_en_dir, "warning_disclaimer.webm"), - os.path.join(videos_en_dir, "WarningSaving.webm") + videos_dir / "ANVIL_Logo.webm", + videos_dir / "INTEL_Logo.webm", + videos_dir / "HUB_Bootflow_FranchiseIntro.webm", + videos_dir / "HUB_Bootflow_AbstergoIntro.webm", + videos_dir / "UbisoftLogo.webm", + videos_en_dir / "Epilepsy.webm", + videos_en_dir / "warning_disclaimer.webm", + videos_en_dir / "WarningSaving.webm" ] for file_path in videos_to_delete: - if os.path.exists(file_path): + if file_path.exists(): try: - os.remove(file_path) + file_path.unlink() logging.info("Deleted: %s", file_path) except Exception as e: logging.error("Error deleting %s: %s", file_path, e) @@ -97,15 +87,15 @@ def delete_videos(): def move_benchmark_file(): """moves html benchmark results to log folder""" - src_dir = f"C:\\Users\\{USERNAME}\\Documents\\Assassin's Creed Shadows\\benchmark_reports" + src_dir = Path( + f"C:\\Users\\{USERNAME}\\Documents\\Assassin's Creed Shadows\\benchmark_reports") - for filename in os.listdir(src_dir): - src_path = os.path.join(src_dir, filename) - dest_path = os.path.join(LOG_DIR, filename) + for src_path in src_dir.iterdir(): + dest_path = LOG_DIR / src_path.name - if os.path.isfile(src_path): + if src_path.is_file(): try: - os.rename(src_path, dest_path) + src_path.rename(dest_path) logging.info("Benchmark HTML moved") except Exception as e: logging.error("Failed to move %s: %s", src_path, e) @@ -190,7 +180,7 @@ def run_benchmark(keras_service): user.press("f1") - find_word(keras_service, "system", "couldn't find system") + find_word(keras_service, "system", "Couldn't find 'System' button") user.press("down") @@ -198,10 +188,16 @@ def run_benchmark(keras_service): user.press("space") - find_word(keras_service, "benchmark", "couldn't find benchmark") + find_word( + keras_service, "benchmark", + "couldn't find 'benchmark' on screen before settings") navi_settings(am) + find_word( + keras_service, "benchmark", + "couldn't find 'benchmark' on screen after settings") + user.press("down") time.sleep(1) @@ -221,10 +217,7 @@ def run_benchmark(keras_service): time.sleep(100) - if keras_service.wait_for_word( - word="results", timeout=30, interval=1) is None: - logging.info("did not find end screen") - sys.exit(1) + find_word(keras_service, "results", "did not find results screen", 60) test_end_time = int_time() - 2 @@ -252,28 +245,10 @@ def run_benchmark(keras_service): return test_start_time, test_end_time -def setup_logging(): - """setup logging""" - setup_log_directory(LOG_DIR) - logging.basicConfig(filename=f'{LOG_DIR}/harness.log', - format=DEFAULT_LOGGING_FORMAT, - datefmt=DEFAULT_DATE_FORMAT, - level=logging.DEBUG) - console = logging.StreamHandler() - formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - - def main(): """entry point""" - parser = ArgumentParser() - parser.add_argument("--kerasHost", dest="keras_host", - help="Host for Keras OCR service", required=True) - parser.add_argument("--kerasPort", dest="keras_port", - help="Port for Keras OCR service", required=True) - args = parser.parse_args() - keras_service = KerasService(args.keras_host, args.keras_port) + keras_service = KerasService( + keras_args().keras_host, keras_args().keras_port) start_time, endtime = run_benchmark(keras_service) height, width = read_current_resolution() report = { @@ -287,7 +262,7 @@ def main(): if __name__ == "__main__": try: - setup_logging() + setup_logging(LOG_DIR) main() except Exception as ex: logging.error("Something went wrong running the benchmark!") diff --git a/blender_render/blender.py b/blender_render/blender.py index 3b8f6b5..63b7e3a 100644 --- a/blender_render/blender.py +++ b/blender_render/blender.py @@ -3,17 +3,17 @@ from pathlib import Path from blender_utils import BENCHMARK_CONFIG, find_blender, run_blender_render, download_scene from argparse import ArgumentParser import logging -import os.path import sys import time -sys.path.insert(1, os.path.join(sys.path[0], '..')) +sys.path.insert(1, str(Path(sys.path[0]).parent)) from harness_utils.output import DEFAULT_DATE_FORMAT, DEFAULT_LOGGING_FORMAT, write_report_json, seconds_to_milliseconds SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") + def setup_logging(): """default logging config""" LOG_DIR.mkdir(exist_ok=True) @@ -26,15 +26,18 @@ def setup_logging(): console.setFormatter(formatter) logging.getLogger('').addHandler(console) + VALID_DEVICES = ["CPU", "CUDA", "OPTIX", "HIP", "ONEAPI", "METAL"] + def main(): """entry point for test script""" parser = ArgumentParser() parser.add_argument("-d", "--device", dest="device", help="device", metavar="device", required=True) parser.add_argument( - "--benchmark", dest="benchmark", help="Benchmark test type", metavar="benchmark", required=True) + "--benchmark", dest="benchmark", help="Benchmark test type", + metavar="benchmark", required=True) args = parser.parse_args() if args.device not in VALID_DEVICES: raise Exception(f"invalid device selection: {args.device}") @@ -49,23 +52,25 @@ def main(): score = run_blender_render( executable_path, LOG_DIR, args.device.upper(), benchmark) end_time = time.time() - logging.info(f'Finished rendering {args.benchmark} in %d seconds', (end_time - start_time)) + logging.info( + f'Finished rendering {args.benchmark} in %d seconds', + (end_time - start_time)) if score is None: raise Exception("no duration was found in the log to use as the score") report = { - "test": f"Blender {args.benchmark} Render {args.device.upper()}", + "test": "Blender Render", + "test_parameter": args.benchmark, "score": score, "unit": "seconds", "version": version, "device": args.device, - "benchmark": args.benchmark, "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) if __name__ == "__main__": diff --git a/blender_render/blender_utils.py b/blender_render/blender_utils.py index c341915..3f60bd0 100644 --- a/blender_render/blender_utils.py +++ b/blender_render/blender_utils.py @@ -80,7 +80,7 @@ def download_scene(scene: BlenderScene) -> None: def copy_scene_from_network_drive(file_name, destination): """copy blend file from network drive""" - network_dir = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Blender Render") + network_dir = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Blender Render") source_path = network_dir.joinpath(file_name) logging.info("Copying %s from %s", file_name, source_path) shutil.copyfile(source_path, destination) @@ -97,16 +97,17 @@ def time_to_seconds(time_string): return seconds -def run_blender_render(executable_path: Path, log_directory: Path, device: str, benchmark: BlenderScene) -> str: +def run_blender_render(executable_path: Path, log_directory: Path, device: str, + benchmark: BlenderScene) -> str: """Execute the blender render of barbershop, returns the duration as string""" blend_log = log_directory.joinpath("blender.log") blend_path = SCRIPT_DIR.joinpath(benchmark.file_name) cmd_line = f'"{str(executable_path)}" -b -E CYCLES -y "{str(blend_path)}" -f 1 -- --cycles-device {device} --cycles-print-stats' - with open(blend_log,'w' , encoding="utf-8") as f_obj: + with open(blend_log, 'w', encoding="utf-8") as f_obj: subprocess.run(cmd_line, stdout=f_obj, text=True, check=True) # example: Time: 02:59.57 (Saving: 00:00.16) - time_regex = r"Time: (.*) \(Saving.*\)" + time_regex = r".*Time:\s+([\d:.]+)\s+\(Saving.*\)" time = None with open(blend_log, 'r', encoding="utf-8") as file: diff --git a/blenderbenchmark/blender.py b/blenderbenchmark/blender.py index 99c7dd2..9886685 100644 --- a/blenderbenchmark/blender.py +++ b/blenderbenchmark/blender.py @@ -117,10 +117,12 @@ for report in json_array: scene_report = { "timestamp": report['timestamp'], "version": blender_version, - "test": f"Blender Benchmark {report['scene']['label']} {DEVICE_TYPE}", + "test": "Blender Benchmark", + "test_parameter": f"{report['scene']['label']} ", "score": round(report['stats']['samples_per_minute'], 2), "unit": "samples per minute", - "device": report['device_info']['compute_devices'][0]['name'] + "device": report['device_info']['compute_devices'][0]['name'], + "device_type": DEVICE_TYPE, } logging.info(json.dumps(scene_report, indent=2)) diff --git a/cinebench_2024/cinebench.py b/cinebench_2024/cinebench.py index e94d705..24d1aaa 100644 --- a/cinebench_2024/cinebench.py +++ b/cinebench_2024/cinebench.py @@ -33,7 +33,8 @@ DURATION_OPTION = "g_CinebenchMinimumTestDuration=1" parser = ArgumentParser() parser.add_argument( - "-t", "--test", dest="test", help="Cinebench test type", required=True, choices=TEST_OPTIONS.keys()) + "-t", "--test", dest="test", help="Cinebench test type", required=True, + choices=TEST_OPTIONS.keys()) args = parser.parse_args() script_dir = Path(__file__).resolve().parent @@ -63,21 +64,30 @@ try: stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, - universal_newlines=True) as proc: - logging.info("Cinebench started. Waiting for setup to finish to set process priority.") + universal_newlines=True) as proc: + logging.info( + "Cinebench started. Waiting for setup to finish to set process priority.") + START_TIME = 0 + if proc.stdout is None: + logging.error("Cinebench process did not start correctly!") + sys.exit(1) for line in proc.stdout: if "BEFORERENDERING" in line: - elapsed_setup_time = round(time.time() - setup_start_time, 2) + elapsed_setup_time = round( + time.time() - setup_start_time, 2) logging.info("Setup took %.2f seconds", elapsed_setup_time) - logging.info("Setting Cinebench process priority to high (PID: %s)", proc.pid) + logging.info( + "Setting Cinebench process priority to high (PID: %s)", + proc.pid) process = psutil.Process(proc.pid) process.nice(psutil.HIGH_PRIORITY_CLASS) - start_time = time.time() + START_TIME = time.time() break out, _ = proc.communicate() if proc.returncode > 0: - logging.warning("Cinebench exited with return code %d", proc.returncode) + logging.warning( + "Cinebench exited with return code %d", proc.returncode) score = get_score(out) if score is None: @@ -85,19 +95,20 @@ try: sys.exit(1) end_time = time.time() - elapsed_test_time = round(end_time - start_time, 2) + elapsed_test_time = round(end_time - START_TIME, 2) logging.info("Benchmark took %.2f seconds", elapsed_test_time) report = { - "test": friendly_test_name(test_type), + "test": "Cinebench 2024", + "test_parameter": friendly_test_name(test_type), "score": score, "unit": "score", - "start_time": seconds_to_milliseconds(start_time), + "start_time": seconds_to_milliseconds(START_TIME), "end_time": seconds_to_milliseconds(end_time) } session_report.append(report) - write_report_json(log_dir, "report.json", session_report) + write_report_json(str(log_dir), "report.json", session_report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/cities_skylines_2/citiesskylines2.py b/cities_skylines_2/citiesskylines2.py index cb1aef3..e22b36c 100644 --- a/cities_skylines_2/citiesskylines2.py +++ b/cities_skylines_2/citiesskylines2.py @@ -20,7 +20,7 @@ from harness_utils.output import ( DEFAULT_DATE_FORMAT ) from harness_utils.steam import exec_steam_game, get_build_id -from harness_utils.keras_service import KerasService +from harness_utils.keras_service import KerasService, ScreenSplitConfig, ScreenShotDivideMethod, ScreenShotQuadrant from harness_utils.artifacts import ArtifactManager, ArtifactType from harness_utils.misc import mouse_scroll_n_times @@ -29,6 +29,10 @@ SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") PROCESS_NAME = "cities2.exe" STEAM_GAME_ID = 949230 +top_left_keras = ScreenSplitConfig( + divide_method=ScreenShotDivideMethod.QUADRANT, + quadrant=ScreenShotQuadrant.TOP_LEFT) + launcher_files = [ "bootstrapper-v2.exe", "launcher.exe", @@ -39,7 +43,6 @@ save_files = [ "Benchmark.cok.cid" ] config_files = [ - "continue_game.json", "UserState.coc" ] @@ -89,12 +92,41 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word("paradox", interval=0.5, timeout=100) if not result: - logging.info("Could not find the paused notification. Unable to mark start time!") + logging.info("Could not find the Paradox logo. Did the game launch?") sys.exit(1) user.press("esc") user.press("esc") user.press("esc") - time.sleep(20) + time.sleep(15) + + result = keras_service.wait_for_word("new", interval=0.5, timeout=100) + if not result: + logging.info("Did not find the main menu. Did the game crash?") + sys.exit(1) + + result = keras_service.look_for_word("load", attempts=10, interval=1) + if not result: + logging.info("Did not find the load game option. Did the save game copy?") + sys.exit(1) + + # Navigate to load save menu + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.click() + time.sleep(0.2) + + result = keras_service.look_for_word("benchmark", attempts=10, interval=1, split_config=top_left_keras) + if not result: + logging.info("Did not find the save game original date. Did the keras click correctly?") + sys.exit(1) + + # Loading the game + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.click() + time.sleep(0.2) + user.press("enter") + time.sleep(10) result = keras_service.wait_for_word("grand", interval=0.5, timeout=100) if not result: @@ -102,6 +134,8 @@ def run_benchmark(keras_service): sys.exit(1) elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) logging.info("Setup took %f seconds", elapsed_setup_time) + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) time.sleep(2) logging.info('Starting benchmark') user.press("3") diff --git a/cities_skylines_2/citiesskylines2_utils.py b/cities_skylines_2/citiesskylines2_utils.py index bc345b2..ac330fe 100644 --- a/cities_skylines_2/citiesskylines2_utils.py +++ b/cities_skylines_2/citiesskylines2_utils.py @@ -18,7 +18,7 @@ LAUNCHCONFIG_LOCATION = Path(f"{LOCALAPPDATA}\\Paradox Interactive") INSTALL_LOCATION = Path(get_app_install_location(STEAM_GAME_ID)) APPDATA = os.getenv("APPDATA") CONFIG_LOCATION = Path(f"{APPDATA}\\..\\LocalLow\\Colossal Order\\Cities Skylines II") -SAVE_LOCATION = Path(f"{CONFIG_LOCATION}\\Saves") +SAVE_LOCATION = Path(f"{CONFIG_LOCATION}\\Saves\\76561199517889423") CONFIG_FILENAME = "launcher-settings.json" @@ -37,7 +37,7 @@ def read_current_resolution(): def copy_continuegame(config_files: list[str]) -> None: - """Copy launcher files to game directory""" + """Copy continue game files to config directory""" for file in config_files: try: src_path = SCRIPT_DIRECTORY / "config" / file @@ -84,7 +84,7 @@ def copy_launcherpath(): shutil.copy(src_path, dest_path) #os.chmod(dest_path, stat.S_IREAD) except OSError as err: - logging.error("Could not copy the launcherpath file. %s", e) + logging.error("Could not copy the launcherpath file. %s", err) raise err @@ -98,5 +98,5 @@ def copy_benchmarksave(save_files: list[str]) -> None: logging.info("Copying: %s -> %s", file, dest_path) shutil.copy(src_path, dest_path) except OSError as err: - logging.error("Could not copy launcher files. %s", err) + logging.error("Could not copy the save game. %s", err) raise err diff --git a/cities_skylines_2/config/UserState.coc b/cities_skylines_2/config/UserState.coc index 41356e4..a5d6789 100644 --- a/cities_skylines_2/config/UserState.coc +++ b/cities_skylines_2/config/UserState.coc @@ -1,5 +1,5 @@ User Settings { - "lastSaveGameMetadata": "4d6c3ccb7ecaebb43efcf0913bb053b0", + "lastSaveGameMetadata": "e284bbf6f86bd366ca930783985b849c", "naturalDisasters": false } diff --git a/cities_skylines_2/config/continue_game.json b/cities_skylines_2/config/continue_game.json deleted file mode 100644 index 4ee682e..0000000 --- a/cities_skylines_2/config/continue_game.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "title": "Benchmark", - "desc": "Population: 99766 Money: \u00a280542653", - "date": "2024-08-20T14:35:47", - "rawGameVersion": "1.1.7f1" -} \ No newline at end of file diff --git a/cities_skylines_2/launcher/notlauncher-options.json b/cities_skylines_2/launcher/notlauncher-options.json index 47733d5..fecac5f 100644 --- a/cities_skylines_2/launcher/notlauncher-options.json +++ b/cities_skylines_2/launcher/notlauncher-options.json @@ -1,5 +1,5 @@ { - "continuelastsave": true, + "continuelastsave": false, "noworkshop": false, "disablemods": false, "nolog": false, diff --git a/cities_skylines_2/save/Benchmark.cok b/cities_skylines_2/save/Benchmark.cok index 17c3cac..9c6aa18 100644 Binary files a/cities_skylines_2/save/Benchmark.cok and b/cities_skylines_2/save/Benchmark.cok differ diff --git a/cities_skylines_2/save/Benchmark.cok.cid b/cities_skylines_2/save/Benchmark.cok.cid index 5785f84..47d8eb0 100644 --- a/cities_skylines_2/save/Benchmark.cok.cid +++ b/cities_skylines_2/save/Benchmark.cok.cid @@ -1 +1 @@ -9a03f3dfd36a585d8397c021d92d4184 \ No newline at end of file +582572b506bb32028036437e27f475ae \ No newline at end of file diff --git a/counterstrike2/cs2.py b/counterstrike2/cs2.py index 6401bc1..e9f63c7 100644 --- a/counterstrike2/cs2.py +++ b/counterstrike2/cs2.py @@ -29,7 +29,14 @@ PROCESS_NAME = "cs2.exe" STEAM_GAME_ID = 730 STEAM_USER_ID = get_registry_active_user() -cfg = Path(get_steam_folder_path(), "userdata", str(STEAM_USER_ID), str(STEAM_GAME_ID), "local", "cfg", "cs2_video.txt") +cfg = Path( + get_steam_folder_path(), + "userdata", str(STEAM_USER_ID), + str(STEAM_GAME_ID), + "local", "cfg", "cs2_video.txt") + +user.FAILSAFE = False + def setup_logging(): """default logging config""" @@ -66,7 +73,7 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word("play", timeout=30, interval=0.1) if not result: logging.info("Did not find the play menu. Did the game load?") - sys.exit(1) + raise RuntimeError height, width = get_resolution() location = None @@ -74,14 +81,25 @@ def run_benchmark(keras_service): # We check the resolution so we know which screenshot to use for the locate on screen function match width: case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1080.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_1080.png", minSearchTime=5, confidence=0.6) case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_1440.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_1440.png", minSearchTime=5, confidence=0.6) case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\settings_2160.png") + location = gui.locateOnScreen( + f"{SCRIPT_DIR}\\screenshots\\settings_2160.png", minSearchTime=5, confidence=0.6) case _: - logging.error("Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) + logging.error( + "Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", + height, width) + raise RuntimeError + + if location is None: + logging.error( + "Could not find the settings cog. The game resolution is currently %s, %s. Are you using a standard resolution?", + height, width) + raise RuntimeError click_me = gui.center(location) gui.moveTo(click_me.x, click_me.y) @@ -93,7 +111,7 @@ def run_benchmark(keras_service): result = keras_service.look_for_word(word="video", attempts=10, interval=1) if not result: logging.info("Did not find the video menu button. Did Keras enter settings correctly?") - sys.exit(1) + raise RuntimeError gui.moveTo(result["x"], result["y"]) gui.mouseDown() @@ -103,14 +121,14 @@ def run_benchmark(keras_service): if keras_service.wait_for_word(word="brightness", timeout=30, interval=1) is None: logging.info("Did not find the video settings menu. Did the menu get stuck?") - sys.exit(1) + raise RuntimeError am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") result = keras_service.look_for_word(word="advanced", attempts=10, interval=1) if not result: logging.info("Did not find the advanced video menu. Did Keras click correctly?") - sys.exit(1) + raise RuntimeError gui.moveTo(result["x"], result["y"]) gui.mouseDown() @@ -118,12 +136,14 @@ def run_benchmark(keras_service): gui.mouseUp() time.sleep(0.2) - am.take_screenshot("advanced_video_1.png", ArtifactType.CONFIG_IMAGE, "first picture of advanced video settings") + am.take_screenshot("advanced_video_1.png", ArtifactType.CONFIG_IMAGE, + "first picture of advanced video settings") result = keras_service.look_for_word(word="boost", attempts=10, interval=1) if not result: - logging.info("Did not find the keyword 'Boost' in the advanced video menu. Did Keras click correctly?") - sys.exit(1) + logging.info( + "Did not find the keyword 'Boost' in the advanced video menu. Did Keras click correctly?") + raise RuntimeError gui.moveTo(result["x"], result["y"]) time.sleep(1) @@ -131,9 +151,11 @@ def run_benchmark(keras_service): time.sleep(1) if keras_service.wait_for_word(word="particle", timeout=30, interval=1) is None: - logging.info("Did not find the keyword 'Particle' in advanced video menu. Did Keras scroll correctly?") - sys.exit(1) - am.take_screenshot("advanced_video_2.png", ArtifactType.CONFIG_IMAGE, "second picture of advanced video settings") + logging.info( + "Did not find the keyword 'Particle' in advanced video menu. Did Keras scroll correctly?") + raise RuntimeError + am.take_screenshot("advanced_video_2.png", ArtifactType.CONFIG_IMAGE, + "second picture of advanced video settings") logging.info('Starting benchmark') user.press("`") @@ -149,7 +171,7 @@ def run_benchmark(keras_service): time.sleep(3) if keras_service.wait_for_word(word="benchmark", timeout=30, interval=0.1) is None: logging.error("Didn't see the title of the benchmark. Did the map load?") - sys.exit(1) + raise RuntimeError setup_end_time = int(time.time()) elapsed_setup_time = round(setup_end_time - setup_start_time, 2) @@ -166,7 +188,7 @@ def run_benchmark(keras_service): test_start_time = int(time.time()) logging.info("Saw \'lets roll\'! Marking the time.") - time.sleep(112) # sleep duration during gameplay + time.sleep(112) # sleep duration during gameplay # Default fallback end time test_end_time = int(time.time()) @@ -174,10 +196,10 @@ def run_benchmark(keras_service): result = keras_service.wait_for_word(word="console", timeout=30, interval=0.1) if result is None: logging.error("The console didn't open. Please check settings and try again.") - sys.exit(1) - else: - test_end_time = int(time.time()) - logging.info("The console opened. Marking end time.") + raise RuntimeError + + test_end_time = int(time.time()) + logging.info("The console opened. Marking end time.") # allow time for result screen to populate time.sleep(8) @@ -227,3 +249,5 @@ if __name__ == "__main__": logging.error("something went wrong running the benchmark!") logging.exception(ex) sys.exit(1) + finally: + terminate_processes(PROCESS_NAME) diff --git a/cspell.json b/cspell.json index c55018f..00786ae 100644 --- a/cspell.json +++ b/cspell.json @@ -46,7 +46,8 @@ "Stellaris", "shusaura", "wukong", - "vgamepad" + "vgamepad", + "DaVinci" ], "ignoreRegExpList": [ "import .*" diff --git a/cyberpunk2077/cyberpunk2077.py b/cyberpunk2077/cyberpunk2077.py index 549aeb3..87369a6 100644 --- a/cyberpunk2077/cyberpunk2077.py +++ b/cyberpunk2077/cyberpunk2077.py @@ -27,6 +27,8 @@ LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") PROCESS_NAME = "cyberpunk2077.exe" +user.FAILSAFE = False + def start_game(): """Launch the game with no launcher or start screen""" return exec_steam_game(STEAM_GAME_ID, game_params=["--launcher-skip", "-skipStartScreen"]) @@ -85,10 +87,9 @@ def navigate_settings() -> None: am.take_screenshot("graphics_1.png", ArtifactType.CONFIG_IMAGE, "graphics menu 1") user.press("down") - - rast = kerasService.wait_for_word("view", interval=1, timeout=2) - if rast: - press_n_times("up", 2, 0.2) #gets you to film grain + time.sleep(0.5) + user.press("down") #gets you to film grain + time.sleep(0.5) dlss = kerasService.wait_for_word("dlss", interval=1, timeout=2) if dlss: diff --git a/cyberpunk2077/cyberpunk_utils.py b/cyberpunk2077/cyberpunk_utils.py index da3a1f4..f125155 100644 --- a/cyberpunk2077/cyberpunk_utils.py +++ b/cyberpunk2077/cyberpunk_utils.py @@ -27,7 +27,8 @@ def get_args() -> any: def copy_from_network_drive(): """Copies mod file from network drive to harness folder""" - src_path = Path(r"\\Labs\labs\03_ProcessingFiles\Cyberpunk 2077\basegame_no_intro_videos.archive") + src_path = Path( + r"\\labs.lmg.gg\labs\03_ProcessingFiles\Cyberpunk 2077\basegame_no_intro_videos.archive") dest_path = SCRIPT_DIRECTORY / "basegame_no_intro_videos.archive" shutil.copyfile(src_path, dest_path) diff --git a/doomdarkages/README.md b/doomdarkages/README.md new file mode 100644 index 0000000..2307030 --- /dev/null +++ b/doomdarkages/README.md @@ -0,0 +1,21 @@ +# Doom: The Dark Ages + +This script navigates through in-game menus to the built in benchmark and runs it with the current settings. It then waits for a results screen while running the Abyssal Forest benchmark. + +## Prerequisites + +- Python 3.10+ +- Doom: The Dark Ages installed +- Keras OCR service + +## Options + +- `kerasHost`: string representing the IP address of the Keras service. e.x. `0.0.0.0` +- `kerasPort`: string representing the port of the Keras service. e.x. `8080` + +## Output + +report.json +- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[height]", e.x. `1920x1080` +- `start_time`: number representing a timestamp of the test's start time in milliseconds +- `end_time`: number representing a timestamp of the test's end time in milliseconds \ No newline at end of file diff --git a/doomdarkages/doomdarkages.py b/doomdarkages/doomdarkages.py new file mode 100644 index 0000000..54da02b --- /dev/null +++ b/doomdarkages/doomdarkages.py @@ -0,0 +1,256 @@ +"""Doom: The Dark Ages test script""" +import logging +from argparse import ArgumentParser +from pathlib import Path +import os.path +import time +import sys +import pydirectinput as user +from doomdarkages_utils import get_resolution + +sys.path.insert(1, os.path.join(sys.path[0], "..")) + +from doomdarkages_utils import copy_launcher_config +from harness_utils.steam import exec_steam_game, get_build_id +from harness_utils.keras_service import KerasService +from harness_utils.misc import press_n_times, mouse_scroll_n_times +from harness_utils.process import terminate_processes +from harness_utils.output import ( + format_resolution, + seconds_to_milliseconds, + setup_log_directory, + write_report_json, + DEFAULT_LOGGING_FORMAT, + DEFAULT_DATE_FORMAT, +) +from harness_utils.artifacts import ArtifactManager, ArtifactType + + + +SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") +PROCESS_NAME = "DOOMTheDarkAges" +STEAM_GAME_ID = 3017860 +username = os.getlogin() +BENCHMARK_RESULTS_PATH = f"C:\\Users\\{username}\\Saved Games\\id Software\\DOOMTheDarkAges\\base\\benchmark" + +user.FAILSAFE = False + +def start_game(): + """Launch the game with no launcher or start screen""" + copy_launcher_config() + return exec_steam_game(STEAM_GAME_ID, game_params=["+com_skipIntroVideo", "1"]) + +def find_latest_result_file(base_path): + """Look for files in the benchmark results path that match the pattern. + Returns the most recent benchmark file.""" + base_path = Path(base_path) + + files = list(base_path.glob("benchmark-*.json")) + if not files: + raise ValueError(f"No benchmark-*.json files found in {base_path}") + + return max(files, key=lambda p: p.stat().st_mtime) + +def run_benchmark(): + """Runs the actual benchmark.""" + start_game() + am = ArtifactManager(LOG_DIRECTORY) + + setup_start_time = int(time.time()) + time.sleep(25) + # Press space to proceed to the main menu + result = kerasService.wait_for_word_vulkan("press", timeout=80) + if not result: + logging.info("Didn't see title screen. Check settings and try again.") + sys.exit(1) + + logging.info("Hit the title screen. Continuing") + time.sleep(2) + user.press("space") + time.sleep(4) + + # Navigate menus and take screenshots using the artifact manager + result = kerasService.wait_for_word_vulkan("campaign", interval=3, timeout=60) + if not result: + logging.info("Didn't land on the main menu!") + sys.exit(1) + + logging.info("Saw the main menu. Proceeding.") + time.sleep(1) + + press_n_times("down", 3, 0.2) + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("daze", interval=3, timeout=15) + if not result: + logging.info("Didn't see the game settings. Did it navigate correctly?") + sys.exit(1) + + logging.info("Saw the game settings. Proceeding.") + press_n_times("q", 2, 0.2) + time.sleep(1) + + # Screenshotting the display settings + result = kerasService.wait_for_word_vulkan("display", interval=3, timeout=15) + if not result: + logging.info("Didn't find the video settings. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video1.png", ArtifactType.CONFIG_IMAGE, "1st screenshot of video settings menu") + mouse_scroll_n_times(6, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("nvidia", interval=3, timeout=15) + if not result: + logging.info("Didn't find the NVIDIA Reflex setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video2.png", ArtifactType.CONFIG_IMAGE, "2nd screenshot of video settings menu") + mouse_scroll_n_times(6, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("advanced", interval=3, timeout=15) + if not result: + logging.info("Didn't find the advanced heading. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video3.png", ArtifactType.CONFIG_IMAGE, "3rd screenshot of video settings menu") + mouse_scroll_n_times(5, -200, 0.2) + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("shading", interval=3, timeout=15) + if not result: + logging.info("Didn't find the shading quality setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video4.png", ArtifactType.CONFIG_IMAGE, "4th screenshot of video settings menu") + mouse_scroll_n_times(5, -220, 0.2) + time.sleep(0.2) + + result = kerasService.wait_for_word_vulkan("brightness", interval=3, timeout=15) + if not result: + logging.info("Didn't find the brightness setting. Did it navigate correctly?") + sys.exit(1) + + am.take_screenshot_vulkan("video5.png", ArtifactType.CONFIG_IMAGE, "5th screenshot of video settings menu") + user.press("escape") + time.sleep(0.2) + + # Navigating to the benchmark + result = kerasService.wait_for_word_vulkan("campaign", interval=3, timeout=20) + if not result: + logging.info("Didn't land on the main menu!") + sys.exit(1) + + logging.info("Saw the main menu. Proceeding.") + time.sleep(1) + + user.press("up") + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("benchmarks", interval=3, timeout=15) + if not result: + logging.info("Didn't navigate to the extras menu. Did it navigate properly?") + sys.exit(1) + + logging.info("Saw the extras menu. Proceeding.") + time.sleep(1) + + user.press("up") + user.press("enter") + time.sleep(1) + + result = kerasService.wait_for_word_vulkan("abyssal", interval=3, timeout=15) + if not result: + logging.info("Don't see the Abyssal Forest benchmark option. Did it navigate properly?") + sys.exit(1) + + logging.info("See the benchmarks. Starting the Abyssal Forest benchmark level.") + time.sleep(1) + + press_n_times("down", 2, 0.2) + user.press("enter") + time.sleep(1) + + elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) + logging.info("Setup took %f seconds", elapsed_setup_time) + + result = kerasService.wait_for_word_vulkan("frame", interval=0.5, timeout=90) + if not result: + logging.info("Benchmark didn't start. Did the game crash?") + sys.exit(1) + + logging.info("Benchmark started. Waiting for benchmark to complete.") + test_start_time = int(time.time()) + 8 + + # Sleeping for the duration of the benchmark + time.sleep(110) + + test_end_time = None + + result = kerasService.wait_for_word_vulkan("results", interval=0.5, timeout=90) + if result: + logging.info("Found the results screen. Marking the out time.") + test_end_time = int(time.time()) - 2 + time.sleep(2) + else: + logging.info("Results screen was not found!" + + "Did harness not wait long enough? Or test was too long?") + sys.exit(1) + + logging.info("Results screen was found! Finishing benchmark.") + results_file = find_latest_result_file(BENCHMARK_RESULTS_PATH) + am.take_screenshot_vulkan("result.png", ArtifactType.RESULTS_IMAGE, "screenshot of results") + am.copy_file(results_file, ArtifactType.RESULTS_TEXT, "benchmark results/settings xml file") + + elapsed_test_time = round(test_end_time - test_start_time, 2) + logging.info("Benchmark took %f seconds", elapsed_test_time) + + terminate_processes(PROCESS_NAME) + am.create_manifest() + + return test_start_time, test_end_time + + +setup_log_directory(LOG_DIRECTORY) + +logging.basicConfig( + filename=f"{LOG_DIRECTORY}/harness.log", + format=DEFAULT_LOGGING_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + level=logging.DEBUG, +) +console = logging.StreamHandler() +formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) +console.setFormatter(formatter) +logging.getLogger("").addHandler(console) + +parser = ArgumentParser() +parser.add_argument( + "--kerasHost", dest="keras_host", help="Host for Keras OCR service", required=True +) +parser.add_argument( + "--kerasPort", dest="keras_port", help="Port for Keras OCR service", required=True +) +args = parser.parse_args() +kerasService = KerasService(args.keras_host, args.keras_port) + +try: + start_time, end_time = run_benchmark() + width, height = get_resolution() + report = { + "resolution": format_resolution(width, height), + "start_time": seconds_to_milliseconds(start_time), + "end_time": seconds_to_milliseconds(end_time), + "version": get_build_id(STEAM_GAME_ID) + } + + write_report_json(LOG_DIRECTORY, "report.json", report) +except Exception as e: + logging.error("Something went wrong running the benchmark!") + logging.exception(e) + terminate_processes(PROCESS_NAME) + sys.exit(1) diff --git a/doomdarkages/doomdarkages_utils.py b/doomdarkages/doomdarkages_utils.py new file mode 100644 index 0000000..abb6770 --- /dev/null +++ b/doomdarkages/doomdarkages_utils.py @@ -0,0 +1,62 @@ +"""Utility functions supporting Doom: The Dark Ages test script.""" +import os +import re +from pathlib import Path +import sys +import logging +import shutil +import json + +sys.path.insert(1, os.path.join(sys.path[0], '..')) +from harness_utils.steam import get_app_install_location + +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +RUN_DIR = SCRIPT_DIRECTORY / "run" +STEAM_GAME_ID = 3017860 +username = os.getlogin() +BENCHMARK_PATH = f"C:\\Users\\{username}\\Saved Games\\id Software\\DOOMTheDarkAges\\base\\benchmark" +RES_REGEX = re.compile(r'\s*(\d+)\s*[x×]\s*(\d+)') + +def get_resolution() -> tuple[int, int]: + """Gets resolution width and height from local xml file created by game.""" + try: + bench_file = max( + RUN_DIR.glob("benchmark-*.json"), + key=lambda p: p.stat().st_mtime + ) + except ValueError as exc: + # No files matched, propagate as a clearer FileNotFoundError + raise FileNotFoundError( + f"No benchmark-*.json files in {RUN_DIR}" + ) from exc + + if not bench_file.is_file(): + raise FileNotFoundError("Benchmark file not found.") + + with bench_file.open(encoding="utf-8") as f: + data = json.load(f) + + res_string = data.get("resolution", "") + m = RES_REGEX.search(res_string) + if not m: + raise ValueError( + f"Cannot parse 'resolution' in {bench_file.name}: {res_string!r}" + ) + + width, height = map(int, m.groups()) + return width, height + +def copy_launcher_config() -> None: + """Copy launcher config to doom launcher config folder""" + try: + launcherconfig_path = Path(get_app_install_location(STEAM_GAME_ID), "launcherData\\base\\configs") + launcherconfig_path.mkdir(parents=True, exist_ok=True) + + src_path = SCRIPT_DIRECTORY / "launcher.cfg" + dest_path = launcherconfig_path / "launcher.cfg" + + logging.info("Copying: %s -> %s", src_path, dest_path) + shutil.copy(src_path, dest_path) + except OSError as err: + logging.error("Could not copy config file.") + raise err diff --git a/doomdarkages/launcher.cfg b/doomdarkages/launcher.cfg new file mode 100644 index 0000000..b9734c2 --- /dev/null +++ b/doomdarkages/launcher.cfg @@ -0,0 +1,16 @@ +rgl_driverMustBeExactMatch 0 +rgl_minNvidiaDriverVersion 57680 +rgl_minAMDDriverMajorVersion 25 +rgl_minAMDDriverMinorVersion 5 +rgl_minAMDDriverPatchVersion 1 +rgl_minAMDDriverMajorVersionWin8 25 +rgl_minAMDDriverMinorVersionWin8 6 +rgl_minAMDDriverPatchVersionWin8 1 +rgl_minAMDDriverMajorVersionWin7 25 +rgl_minAMDDriverMinorVersionWin7 6 +rgl_minAMDDriverPatchVersionWin7 1 +rgl_minIntelDriverMajorVersion 101 +rgl_minIntelDriverMinorVersion 6732 +rgl_showAMDStartupWarning 0 +rgl_showIntelStartupWarning 0 +rgl_showNvidiaStartupWarning 0 diff --git a/doomdarkages/manifest.yaml b/doomdarkages/manifest.yaml new file mode 100644 index 0000000..38d1024 --- /dev/null +++ b/doomdarkages/manifest.yaml @@ -0,0 +1,10 @@ +friendly_name: "Doom: The Dark Ages" +executable: "doomdarkages.py" +process_name: "DOOMTheDarkAges.exe" +hidden: 0 +output_dir: "run" +options: + - name: kerasHost + type: input + - name: kerasPort + type: input \ No newline at end of file diff --git a/dota2/dota2.py b/dota2/dota2.py index 5968699..ec6ed78 100644 --- a/dota2/dota2.py +++ b/dota2/dota2.py @@ -1,13 +1,13 @@ """Dota 2 test script""" import logging -import os +from pathlib import Path import time import pyautogui as gui import pydirectinput as user import sys from dota2_utils import get_resolution, copy_replay, copy_config, get_args -sys.path.insert(1, os.path.join(sys.path[0], '..')) +sys.path.insert(1, str(Path(sys.path[0]).parent)) from harness_utils.output import ( setup_log_directory, @@ -22,12 +22,12 @@ from harness_utils.steam import exec_steam_game from harness_utils.artifacts import ArtifactManager, ArtifactType -SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) -LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +LOG_DIRECTORY = SCRIPT_DIRECTORY / "run" PROCESS_NAME = "dota2.exe" STEAM_GAME_ID = 570 -setup_log_directory(LOG_DIRECTORY) +setup_log_directory(str(LOG_DIRECTORY)) logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log', format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -40,6 +40,7 @@ logging.getLogger('').addHandler(console) args = get_args() kerasService = KerasService(args.keras_host, args.keras_port) +user.FAILSAFE = False def start_game(): """Launch the game with console enabled and FPS unlocked""" @@ -75,6 +76,8 @@ def run_benchmark(): "Game didn't start in time. Check settings and try again.") sys.exit(1) + time.sleep(15) # wait for main menu + screen_height, screen_width = get_resolution() location = None click_multiple = 0 @@ -87,15 +90,18 @@ def run_benchmark(): click_multiple = 0.8 case "1920": location = gui.locateOnScreen( - f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1080.png") + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1080.png", + confidence=0.9) click_multiple = 1 case "2560": location = gui.locateOnScreen( - f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1440.png") + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_1440.png", + confidence=0.9) click_multiple = 1.5 case "3840": location = gui.locateOnScreen( - f"{SCRIPT_DIRECTORY}\\screenshots\\settings_2160.png") + f"{SCRIPT_DIRECTORY}\\screenshots\\settings_2160.png", + confidence=0.9) click_multiple = 2 case _: logging.error( @@ -130,9 +136,30 @@ def run_benchmark(): "Did not find the video settings menu. Did the menu get stuck?") sys.exit(1) - am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, + am.take_screenshot("video1.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") + user.press("down") + + if kerasService.wait_for_word( + word="api", timeout=30, interval=1) is None: + logging.info( + "Did not find the video settings menu. Did the menu get stuck?") + sys.exit(1) + + am.take_screenshot("video2.png", ArtifactType.CONFIG_IMAGE, + "picture of video settings") + + user.press("down") + + if kerasService.wait_for_word( + word="direct", timeout=30, interval=1) is None: + logging.info( + "Did not find the video settings menu. Did the menu get stuck?") + sys.exit(1) + + am.take_screenshot("video3.png", ArtifactType.CONFIG_IMAGE, + "picture of video settings") # starting the benchmark user.press("escape") logging.info('Starting benchmark') @@ -196,14 +223,14 @@ def run_benchmark(): try: start_time, end_time = run_benchmark() - height, width = get_resolution() + res_height, res_width = get_resolution() report = { - "resolution": format_resolution(width, height), + "resolution": format_resolution(int(res_width), int(res_height)), "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIRECTORY, "report.json", report) + write_report_json(str(LOG_DIRECTORY), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/dota2/dota2_utils.py b/dota2/dota2_utils.py index 09afce8..f44896a 100644 --- a/dota2/dota2_utils.py +++ b/dota2/dota2_utils.py @@ -37,7 +37,7 @@ def get_install_path(): def copy_replay_from_network_drive(): """Copies replay file from network drive to harness folder""" - src_path = Path(r"\\Labs\labs\03_ProcessingFiles\Dota2\benchmark.dem") + src_path = Path(r"\\labs.lmg.gg\labs\03_ProcessingFiles\Dota2\benchmark.dem") dest_path = SCRIPT_DIRECTORY / "benchmark.dem" shutil.copyfile(src_path, dest_path) @@ -84,13 +84,18 @@ def copy_config() -> None: def read_config() -> list[str] | None: """Looks for config file and returns contents if found""" - userdata_path = Path(get_steam_folder_path(), "userdata", str(STEAM_USER_ID), str(STEAM_GAME_ID), "local", "cfg", "video.txt") + userdata_path = Path( + get_steam_folder_path(), + "userdata", str(STEAM_USER_ID), + str(STEAM_GAME_ID), + "local", "cfg", "video.txt") install_path = Path(get_install_path(), "game", "dota", "cfg", "video.txt") try: with open(userdata_path, encoding="utf-8") as f: return f.readlines() except OSError: - logging.error("Did not find config file at path %s. Trying path %s", userdata_path, install_path) + logging.error("Did not find config file at path %s. Trying path %s", + userdata_path, install_path) try: with open(install_path, encoding="utf-8") as f: return f.readlines() @@ -118,6 +123,6 @@ def get_resolution(): height = height_match.group(1) if width_match is not None: width = width_match.group(1) - if height != 0 and width !=0: + if height != 0 and width != 0: return (height, width) return (height, width) diff --git a/dota2/screenshots/settings_1080.png b/dota2/screenshots/settings_1080.png index 84cee45..e5e84d9 100644 Binary files a/dota2/screenshots/settings_1080.png and b/dota2/screenshots/settings_1080.png differ diff --git a/dota2/screenshots/settings_1440.png b/dota2/screenshots/settings_1440.png index c8e7ca2..515bdd2 100644 Binary files a/dota2/screenshots/settings_1440.png and b/dota2/screenshots/settings_1440.png differ diff --git a/dota2/screenshots/settings_2160.png b/dota2/screenshots/settings_2160.png index af3032f..c896f8e 100644 Binary files a/dota2/screenshots/settings_2160.png and b/dota2/screenshots/settings_2160.png differ diff --git a/dota2/screenshots/settings_720.png b/dota2/screenshots/settings_720.png index 50f1dd8..9df5e44 100644 Binary files a/dota2/screenshots/settings_720.png and b/dota2/screenshots/settings_720.png differ diff --git a/farcry6/farcry6.py b/farcry6/farcry6.py index c128e13..61dd49b 100644 --- a/farcry6/farcry6.py +++ b/farcry6/farcry6.py @@ -1,4 +1,6 @@ """Far Cry 6 test script""" + +# pylint: disable = C0116, W0621 import os import logging import time @@ -30,25 +32,31 @@ LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") PROCESS_NAME = "FarCry6.exe" GAME_ID = 5266 username = os.getlogin() -xml_file = rf"C:\Users\{username}\Documents\My Games\Far Cry 6\gamerprofile.xml" +XML_FILE = rf"C:\Users\{username}\Documents\My Games\Far Cry 6\gamerprofile.xml" + + +user.FAILSAFE = False + def start_game(): - subprocess.run(f'start uplay://launch/{GAME_ID}/0', shell=True) + subprocess.run(f'start uplay://launch/{GAME_ID}/0', shell=True, check=True) + def skip_logo_screens() -> None: """Simulate input to skip logo screens""" logging.info("Skipping logo screens") - #skipping the logo screens + # skipping the logo screens press_n_times("escape", 8, 1) + def run_benchmark(): am = ArtifactManager(LOG_DIRECTORY) start_game() setup_start_time = int(time.time()) time.sleep(25) - #skipping game intros + # skipping game intros result = kerasService.look_for_word("warning", attempts=20, interval=1) if not result: logging.info("Did not see warnings. Did the game start?") @@ -66,7 +74,7 @@ def run_benchmark(): time.sleep(2) - #navigating the menus to get to the video settings + # navigating the menus to get to the video settings result = kerasService.look_for_word("later", attempts=5, interval=1) if result: user.press("escape") @@ -95,10 +103,11 @@ def run_benchmark(): gui.mouseUp() time.sleep(2) - #grabbing screenshots of all the video settings + # grabbing screenshots of all the video settings result = kerasService.look_for_word("adapter", attempts=10, interval=1) if not result: - logging.info("Did not find the Video Adapter setting in the monitor options. Did keras navigate wrong?") + logging.info( + "Did not find the Video Adapter setting in the monitor options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("video.png", ArtifactType.CONFIG_IMAGE, "picture of video settings") @@ -109,18 +118,20 @@ def run_benchmark(): result = kerasService.look_for_word("filtering", attempts=10, interval=1) if not result: - logging.info("Did not find the Texture Filtering setting in the quality options. Did keras navigate wrong?") + logging.info( + "Did not find the Texture Filtering setting in the quality options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("quality1.png", ArtifactType.CONFIG_IMAGE, "1st picture of quality settings") time.sleep(2) - mouse_scroll_n_times(8, -800, 0.2) + mouse_scroll_n_times(8, -800, 0.2) result = kerasService.look_for_word("shading", attempts=10, interval=1) if not result: - logging.info("Did not find the FidelityFX Variable Shading setting in the quality options. Did keras navigate wrong?") + logging.info( + "Did not find the FidelityFX Variable Shading setting in the quality options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("quality2.png", ArtifactType.CONFIG_IMAGE, "2nd picture of quality settings") @@ -131,12 +142,13 @@ def run_benchmark(): result = kerasService.look_for_word("lock", attempts=10, interval=1) if not result: - logging.info("Did not find the Enable Framerate Lock setting in the advanced options. Did keras navigate wrong?") + logging.info( + "Did not find the Enable Framerate Lock setting in the advanced options. Did keras navigate wrong?") sys.exit(1) am.take_screenshot("advanced.png", ArtifactType.CONFIG_IMAGE, "picture of advanced settings") - #starting the benchmark + # starting the benchmark time.sleep(2) user.press("f5") elapsed_setup_time = round(int(time.time()) - setup_start_time, 2) @@ -148,7 +160,7 @@ def run_benchmark(): sys.exit(1) test_start_time = int(time.time()) - time.sleep(60) # wait for benchmark to complete + time.sleep(60) # wait for benchmark to complete result = kerasService.wait_for_word("results", interval=0.5, timeout=100) if not result: @@ -165,11 +177,12 @@ def run_benchmark(): # Exit terminate_processes(PROCESS_NAME) - am.copy_file(xml_file, ArtifactType.CONFIG_TEXT, "config file") + am.copy_file(XML_FILE, ArtifactType.CONFIG_TEXT, "config file") am.create_manifest() return test_start_time, test_end_time + setup_log_directory(LOG_DIRECTORY) logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log', @@ -204,4 +217,4 @@ except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) terminate_processes(PROCESS_NAME) - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/forza5/forza5.py b/forza5/forza5.py index 3d8943f..cb62db7 100644 --- a/forza5/forza5.py +++ b/forza5/forza5.py @@ -34,6 +34,7 @@ CONFIG_LOCATION = ( CONFIG_FILENAME = "UserConfigSelections" PROCESSES = ["ForzaHorizon5.exe", "RTSS.exe"] +user.FAILSAFE = False def start_rtss(): """Sets up the RTSS process""" diff --git a/godot_compile/godot_compile.py b/godot_compile/godot_compile.py index 5abb906..12120fd 100644 --- a/godot_compile/godot_compile.py +++ b/godot_compile/godot_compile.py @@ -78,11 +78,11 @@ def main(): report = { "start_time": start_time, - "version": "4.3-stable", + "version": "4.4.1-stable", "end_time": end_time, "score": score, "unit": "seconds", - "test": "Godot 4.3 Compile" + "test": "Godot 4.4.1 Compile" } write_report_json(LOG_DIR, "report.json", report) diff --git a/godot_compile/godot_compile_utils.py b/godot_compile/godot_compile_utils.py index f78f604..a64cad2 100644 --- a/godot_compile/godot_compile_utils.py +++ b/godot_compile/godot_compile_utils.py @@ -14,7 +14,7 @@ MINGW_ZIP = "x86_64-13.2.0-release-posix-seh-msvcrt-rt_v11-rev1.zip" MINGW_FOLDER = SCRIPT_DIR.joinpath("mingw64") MINICONDA_EXECUTABLE_PATH = Path("C:\\ProgramData\\miniconda3\\_conda.exe") CONDA_ENV_NAME = "godotbuild" -GODOT_DIR = "godot-4.3-stable" +GODOT_DIR = "godot-4.4.1-stable" def install_mingw() -> str: @@ -24,7 +24,7 @@ def install_mingw() -> str: if str(MINGW_FOLDER) not in original_path: os.environ['PATH'] = str(MINGW_FOLDER.joinpath('bin')) + os.pathsep + original_path return "existing mingw installation detected" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\MinGW\\").joinpath(MINGW_ZIP) + source = Path("\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\MinGW\\").joinpath(MINGW_ZIP) destination = SCRIPT_DIR.joinpath(MINGW_ZIP) shutil.copyfile(source, destination) with ZipFile(destination, 'r') as zip_object: @@ -36,7 +36,8 @@ def install_mingw() -> str: def copy_miniconda_from_network_drive(): """copies miniconda installer from network drive""" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\Miniconda\\").joinpath(MINICONDA_INSTALLER) + source = Path("\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\Miniconda\\").joinpath( + MINICONDA_INSTALLER) destination = SCRIPT_DIR.joinpath(MINICONDA_INSTALLER) shutil.copyfile(source, destination) @@ -49,15 +50,15 @@ def install_miniconda() -> str: copy_miniconda_from_network_drive() except Exception as err: raise Exception("could not copy miniconda from network drive") from err - command =[ + command = [ "powershell", - "start-process", + "start-process", "-FilePath", f'"{str(SCRIPT_DIR.joinpath(MINICONDA_INSTALLER))}"', "-ArgumentList", '"/S"', "-Wait" - ] + ] try: output = subprocess.check_output(command, stderr=subprocess.PIPE, text=True) except Exception as err: @@ -71,14 +72,14 @@ def copy_godot_source_from_network_drive() -> str: if SCRIPT_DIR.joinpath(GODOT_DIR).is_dir(): return "existing godot source directory detected" zip_name = f"{GODOT_DIR}.zip" - source = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Godot Files\\").joinpath(zip_name) + source = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Godot Files\\").joinpath(zip_name) destination = SCRIPT_DIR.joinpath(zip_name) shutil.copyfile(source, destination) with ZipFile(destination, 'r') as zip_object: try: zip_object.extractall(path=SCRIPT_DIR) except Exception as ex: - raise Exception ("error extracting godot zip") from ex + raise Exception("error extracting godot zip") from ex return "godot source copied and unpacked from network drive" @@ -90,7 +91,8 @@ def check_conda_environment_exists() -> bool: "-n", CONDA_ENV_NAME ] - process = subprocess.run(" ".join(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + process = subprocess.run(" ".join(command), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=False) if process.returncode == 1: return False return True @@ -131,6 +133,6 @@ def convert_duration_string_to_seconds(duration: str) -> int: hours=int(duration.split(':')[0]), minutes=int(duration.split(':')[1]), seconds=float(duration.split('.')[0].split(':')[2]), - milliseconds=int(float('0.' + duration.split('.')[1])*1000)) + milliseconds=int(float('0.' + duration.split('.')[1]) * 1000)) return round(time_obj.total_seconds()) diff --git a/gravitymark/gravitymark.py b/gravitymark/gravitymark.py index 6eee1ec..c5b3898 100644 --- a/gravitymark/gravitymark.py +++ b/gravitymark/gravitymark.py @@ -4,7 +4,7 @@ import getpass import subprocess import sys from pathlib import Path -from gravitymark_utils import friendly_test_name, get_args, get_score, create_gravitymark_command +from gravitymark_utils import friendly_test_param, get_args, get_score, create_gravitymark_command PARENT_DIR = str(Path(sys.path[0], "..")) sys.path.append(PARENT_DIR) @@ -19,7 +19,7 @@ GRAVITYMARK_PATH = Path("C:/", "Program Files", "GravityMark", "bin") GRAVITYMARK_EXE = GRAVITYMARK_PATH / "GravityMark.exe" args = get_args() -api = f"-{args.api}" +API = f"-{args.api}" script_dir = Path(__file__).resolve().parent log_dir = script_dir / "run" @@ -36,9 +36,11 @@ formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger("").addHandler(console) -gravitymark_log_path = Path("C:/Users", getpass.getuser(), ".GravityMark", "GravityMark.log") +gravitymark_log_path = Path( + "C:/Users", getpass.getuser(), + ".GravityMark", "GravityMark.log") image_path = log_dir / "result.png" -command = create_gravitymark_command(GRAVITYMARK_EXE, api, image_path) +command = create_gravitymark_command(GRAVITYMARK_EXE, API, image_path) try: logging.info('Starting benchmark!') @@ -47,7 +49,8 @@ try: result = subprocess.run(command, check=True, cwd=GRAVITYMARK_PATH) if result.returncode > 0: - logging.error("GravityMark exited with return code %d", result.returncode) + logging.error("GravityMark exited with return code %d", + result.returncode) sys.exit(1) score = get_score(gravitymark_log_path) @@ -57,12 +60,13 @@ try: sys.exit(1) report = { - "test": friendly_test_name(args.api), + "test": "GravityMark", + "test_parameter": friendly_test_param(args.api), "score": score, "unit": "score" } - write_report_json(log_dir, "report.json", report) + write_report_json(str(log_dir), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/gravitymark/gravitymark_utils.py b/gravitymark/gravitymark_utils.py index 3a0d0b8..899bebf 100644 --- a/gravitymark/gravitymark_utils.py +++ b/gravitymark/gravitymark_utils.py @@ -23,16 +23,16 @@ CLI_OPTIONS = { "-status": "1" } -def friendly_test_name(api: str) -> str: +def friendly_test_param(api: str) -> str: """return a friendlier string given the API harness argument""" if api == "vulkan": - return "GravityMark Vulkan" + return "Vulkan" if api == "opengl": - return "GravityMark OpenGL" + return "OpenGL" if api == "direct3d12": - return "GravityMark DX12" + return "DX12" if api == "direct3d11": - return "GravityMark DX11" + return "DX11" return api def get_args() -> Namespace: diff --git a/grid_legends/grid_legends.py b/grid_legends/grid_legends.py index c8d4b73..558cf7b 100644 --- a/grid_legends/grid_legends.py +++ b/grid_legends/grid_legends.py @@ -33,6 +33,8 @@ CONFIG_PATH = f"C:\\Users\\{username}\\Documents\\My Games\\GRID Legends\\hardwa CONFIG_FILENAME = "hardware_settings_config.xml" CONFIG_FULL_PATH = f"{CONFIG_PATH}\\{CONFIG_FILENAME}" +user.FAILSAFE = False + def get_resolution() -> tuple[int]: """Gets resolution width and height from local xml file created by game.""" resolution = re.compile(r" bool: def copy_handbrake_from_network_drive(): """copy handbrake cli from network drive""" - source = Path("\\\\Labs\\labs\\01_Installers_Utilities\\Handbrake\\X86\\HandBrakeCLI-1.9.1-win-x86_64\\") + source = Path( + "\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\Handbrake\\X86\\HandBrakeCLI-1.9.1-win-x86_64\\") copy_souce = source / HANDBRAKE_EXECUTABLE destination = SCRIPT_DIR / HANDBRAKE_EXECUTABLE shutil.copyfile(copy_souce, destination) @@ -30,7 +31,7 @@ def is_video_source_present() -> bool: def copy_video_source(): """copy big buck bunny source video to local from network drive""" - source = r"\\Labs\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m" + source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, SOURCE_VIDEO_NAME) shutil.copyfile(source, destination) diff --git a/harness_utils/artifacts.py b/harness_utils/artifacts.py index ad26758..e84ffd8 100644 --- a/harness_utils/artifacts.py +++ b/harness_utils/artifacts.py @@ -81,7 +81,7 @@ class ArtifactManager: The newly created artifact's `type` and `description` fields are set to the given `artifact_type` and `description` arguments respectively while the artifact's `filename` is set to the basename of `src`. - + Raises a `ValueError` if `src` points to a directory instead of a file. """ src_path = Path(src) @@ -108,7 +108,7 @@ class ArtifactManager: The newly created artifact's `filename`, `type` and `description` fields are set to the given `filename`, `artifact_type` and `description` arguments respectively. - Raises a `ValueError` if `artifact_type` is not one of the `ArtifactType` values which represents an image. + Raises a `ValueError` if `artifact_type` is not one of the `ArtifactType` values which represents an image. """ if artifact_type not in _IMAGE_ARTIFACT_TYPES: raise ValueError("artifact_type should be a type that represents an image artifact") diff --git a/harness_utils/misc.py b/harness_utils/misc.py index 24b28d8..833cdd1 100644 --- a/harness_utils/misc.py +++ b/harness_utils/misc.py @@ -1,4 +1,5 @@ """Misc utility functions""" +from argparse import ArgumentParser import logging import os from pathlib import Path @@ -10,6 +11,9 @@ import requests import vgamepad as vg import json import re +import sys + +user.FAILSAFE = False class LTTGamePad360(vg.VX360Gamepad): """ @@ -19,7 +23,8 @@ class LTTGamePad360(vg.VX360Gamepad): This class extension provides some useful functions to make your code look a little cleaner when implemented in our harnesses. """ - def single_press(self, button = vg.XUSB_BUTTON.XUSB_GAMEPAD_DPAD_DOWN, pause = 0.1): + + def single_press(self, button=vg.XUSB_BUTTON.XUSB_GAMEPAD_DPAD_DOWN, pause=0.1): """ Custom function to perform a single press of a specified gamepad button @@ -59,6 +64,7 @@ class LTTGamePad360(vg.VX360Gamepad): self.single_press(button) time.sleep(pause) + class LTTGamePadDS4(vg.VDS4Gamepad): """ Class extension for the virtual game pad library @@ -67,7 +73,8 @@ class LTTGamePadDS4(vg.VDS4Gamepad): This class extension provides some useful functions to make your code look a little cleaner when implemented in our harnesses. """ - def single_button_press(self, button = vg.DS4_BUTTONS.DS4_BUTTON_CROSS, fastpause = 0.05): + + def single_button_press(self, button=vg.DS4_BUTTONS.DS4_BUTTON_CROSS, fastpause=0.05): """ Custom function to perform a single press of a specified gamepad digital button @@ -95,7 +102,6 @@ class LTTGamePadDS4(vg.VDS4Gamepad): time.sleep(fastpause) self.release_button(button=button) self.update() - def button_press_n_times(self, button: vg.DS4_BUTTONS, n: int, pause: float): """ @@ -105,7 +111,7 @@ class LTTGamePadDS4(vg.VDS4Gamepad): self.single_button_press(button) time.sleep(pause) - def single_dpad_press(self, direction = vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, pause = 0.1): + def single_dpad_press(self, direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, pause=0.1): """ Custom function to perform a single press of a specified gamepad button @@ -139,6 +145,7 @@ class LTTGamePadDS4(vg.VDS4Gamepad): self.single_dpad_press(direction) time.sleep(pause) + def clickme(x: int, y: int): """Pyautogui's click function sucks, this should do the trick""" gui.moveTo(x, y) @@ -147,10 +154,11 @@ def clickme(x: int, y: int): time.sleep(0.2) gui.mouseUp() + def mouse_scroll_n_times(n: int, scroll_amount: int, pause: float): """ Pyautogui's mouse scroll function often fails to actually scroll in game menus, this functions solves that problem - + n --> the number of times you want to scroll, should be a positive integer scroll_amount --> positive is scroll up, negative is scroll down pause --> the amount of time to pause betwee subsequent scrolls @@ -159,12 +167,19 @@ def mouse_scroll_n_times(n: int, scroll_amount: int, pause: float): gui.vscroll(scroll_amount) time.sleep(pause) -def press_n_times(key: str, n: int, pause: float): + +def int_time() -> int: + """Returns the current time in seconds since epoch as an integer""" + return int(time.time()) + + +def press_n_times(key: str, n: int, pause: float = 0.5): """A helper function press the same button multiple times""" for _ in range(n): user.press(key) time.sleep(pause) + def remove_files(paths: list[str]) -> None: """Removes files specified by provided list of file paths. Does nothing for a path that does not exist. @@ -199,7 +214,7 @@ def extract_file_from_archive(zip_file: Path, member_path: str, destination_dir: def find_eg_game_version(gamefoldername: str) -> str: """Find the version of the specific game (e.g., AlanWake2) from the launcher installed data.""" installerdat = r"C:\ProgramData\Epic\UnrealEngineLauncher\LauncherInstalled.dat" - + try: # Open the file and read its entire content with open(installerdat, encoding="utf-8") as file: @@ -213,7 +228,7 @@ def find_eg_game_version(gamefoldername: str) -> str: # Extract the InstallationList part from the file installation_list_json = installation_list_match.group(1) - + # Load the installation list as JSON installation_list = json.loads(installation_list_json) @@ -228,3 +243,20 @@ def find_eg_game_version(gamefoldername: str) -> str: print(f"Error: {e}") return None + + +def find_word(keras_service, word, msg, timeout=30, interval=1): + """Function to call Keras service to find a word in the screen""" + if keras_service.wait_for_word(word=word, timeout=timeout, interval=interval) is None: + logging.error(msg) + sys.exit(1) + + +def keras_args(): + """helper function to get args for keras""" + parser = ArgumentParser() + parser.add_argument("--kerasHost", dest="keras_host", + help="Host for Keras OCR service", required=True) + parser.add_argument("--kerasPort", dest="keras_port", + help="Port for Keras OCR service", required=True) + return parser.parse_args() diff --git a/harness_utils/output.py b/harness_utils/output.py index 875751d..72d28dd 100644 --- a/harness_utils/output.py +++ b/harness_utils/output.py @@ -1,17 +1,20 @@ """Functions related to logging and formatting output from test harnesses.""" import json import os +import logging DEFAULT_LOGGING_FORMAT = '%(asctime)s %(levelname)-s %(message)s' DEFAULT_DATE_FORMAT = '%m-%d %H:%M' + def setup_log_directory(log_dir: str) -> None: """Creates the log directory for a harness if it does not already exist""" if not os.path.isdir(log_dir): os.mkdir(log_dir) -def write_report_json(log_dir: str, report_name: str, report_json: any) -> None: +def write_report_json( + log_dir: str, report_name: str, report_json: dict) -> None: """Writes the json output of a harness to the log directory""" with open(os.path.join(log_dir, report_name), "w", encoding="utf-8") as file: file.write(json.dumps(report_json)) @@ -25,3 +28,17 @@ def format_resolution(width: int, height: int) -> str: def seconds_to_milliseconds(seconds: float | int) -> int: """Convert seconds to milliseconds""" return round((seconds * 1000)) + + +def setup_logging(log_directory: str) -> None: + """Sets up logging for the harness""" + setup_log_directory(log_directory) + + logging.basicConfig(filename=f'{log_directory}/harness.log', + format=DEFAULT_LOGGING_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + level=logging.DEBUG) + console = logging.StreamHandler() + formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) diff --git a/marvelrivals/README.md b/marvelrivals/README.md index 88b0e29..a6e8a6b 100644 --- a/marvelrivals/README.md +++ b/marvelrivals/README.md @@ -1,12 +1,11 @@ # Marvel Rivals -This benchmark runs a replay of a Season 1 tournament Double Elimination round game between SendHelp and BeerLovers +This benchmark runs a canned benchmark built into the Marvel Rivals settings. ## Prerequisites - Python 3.10+ - Marvel Rivals installed on Steam - Keras OCR service -- Favoriting replay ID 10518740076 ## Options diff --git a/marvelrivals/marvelrivals.py b/marvelrivals/marvelrivals.py index d1244e9..07f4f38 100644 --- a/marvelrivals/marvelrivals.py +++ b/marvelrivals/marvelrivals.py @@ -7,7 +7,7 @@ import time import pyautogui as gui import pydirectinput as user import sys -from marvelrivals_utils import read_resolution +from marvelrivals_utils import read_resolution, find_latest_benchmarkcsv import subprocess sys.path.insert(1, os.path.join(sys.path[0], '..')) @@ -33,7 +33,9 @@ LAUNCHER_NAME = "MarvelRivals_Launcher.exe" APPDATA = os.getenv("LOCALAPPDATA") CONFIG_LOCATION = f"{APPDATA}\\Marvel\\Saved\\Config\\Windows" CONFIG_FILENAME = "GameUserSettings.ini" -cfg = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}" +CFG = f"{CONFIG_LOCATION}\\{CONFIG_FILENAME}" + +user.FAILSAFE = False am = ArtifactManager(LOG_DIR) @@ -49,19 +51,20 @@ def setup_logging(): console.setFormatter(formatter) logging.getLogger('').addHandler(console) + + def start_game(): """Starts the game process""" game_path = get_app_install_location(STEAM_GAME_ID) process_path = os.path.join(game_path, LAUNCHER_NAME) # Full path to the executable - logging.info(f"Starting game: {process_path}") - process = subprocess.Popen([process_path], cwd=game_path) + logging.info("Starting game: %s", process_path) + process = subprocess.Popen([process_path], cwd=game_path) # pylint: disable=R1732 return process def run_benchmark(keras_service): """Run Marvel Rivals benchmark""" setup_start_time = int(time.time()) start_game() - #wait for launcher to launch then click the launch button to launch the launcher into the game that we were launching time.sleep(20) @@ -84,7 +87,13 @@ def run_benchmark(keras_service): gui.mouseDown() time.sleep(0.2) gui.mouseUp() - time.sleep(0.5) + time.sleep(20) + + #checking if a marketing notification has come up + result = keras_service.wait_for_word("view", timeout=15, interval=1) + if result: + user.press("escape") + time.sleep(0.5) #navigating to the video settings and taking screenshots result = keras_service.wait_for_word("play", timeout=30, interval=1) @@ -125,42 +134,14 @@ def run_benchmark(keras_service): time.sleep(1) #navigate to the player profile - user.press("escape") + mouse_scroll_n_times(10, 800, 0.2) time.sleep(1) - result = keras_service.wait_for_word("play", timeout=30, interval=1) + + result = keras_service.wait_for_word("run", timeout=30, interval=1) if not result: - logging.info("Did not find the play menu. Did it press escape?") + logging.info("Did not find the Performance Test. Did it scroll back up properly?") sys.exit(1) - time.sleep(1) - height, width = read_resolution() - location = None - - # We check the resolution so we know which screenshot to use for the locate on screen function - match width: - case "1280": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_720.png", confidence=0.9) - case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_1080.png", confidence=0.9) - case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_1440.png", confidence=0.9) - case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\profile_2160.png", confidence=0.9) - case _: - logging.error("Could not find the profile icon. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) - click_me = gui.center(location) - gui.moveTo(click_me.x, click_me.y) - gui.mouseDown() - time.sleep(0.2) - gui.mouseUp() - time.sleep(0.5) - - #navigate to the replays section - result = keras_service.wait_for_word("favorites", timeout=30, interval=1) - if not result: - logging.info("Did not find the favorites menu. Did it navigate properly to it?") - sys.exit(1) gui.moveTo(result["x"], result["y"]) time.sleep(0.2) gui.mouseDown() @@ -168,69 +149,42 @@ def run_benchmark(keras_service): gui.mouseUp() time.sleep(1) - result = keras_service.wait_for_word("match", timeout=30, interval=1) + result = keras_service.wait_for_word("start", timeout=30, interval=1) if not result: - logging.info("Did not find the match replays menu. Did it click correctly?") + logging.info("Did not find the Start Test button. Keras click correctly?") sys.exit(1) + gui.moveTo(result["x"], result["y"]) time.sleep(0.2) gui.mouseDown() time.sleep(0.2) gui.mouseUp() time.sleep(1) - - #starting the benchmark replay - result = keras_service.wait_for_word("shibuya", timeout=30, interval=1) - if not result: - logging.info("Did not find the replay we were looking for. Is it not saved in the favorites?") - sys.exit(1) - match width: - case "1280": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_720.png", confidence=0.9) - case "1920": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_1080.png", confidence=0.9) - case "2560": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_1440.png", confidence=0.9) - case "3840": - location = gui.locateOnScreen(f"{SCRIPT_DIR}\\screenshots\\play_2160.png", confidence=0.9) - case _: - logging.error("Could not find the play button. The game resolution is currently %s, %s. Are you using a standard resolution?", height, width) - sys.exit(1) - click_me = gui.center(location) - gui.moveTo(click_me.x, click_me.y) - gui.mouseDown() - time.sleep(0.2) - gui.mouseUp() - time.sleep(0.5) - #marking the in-time + #marking the end time setup_end_time = int(time.time()) elapsed_setup_time = round(setup_end_time - setup_start_time, 2) logging.info("Harness setup took %f seconds", elapsed_setup_time) time.sleep(2) - - #looking for the player name to start wait timer till we get into the actual game - result = keras_service.wait_for_word("dluo", timeout=30, interval=1) - if not result: - logging.info("Did not find the player Dluo. Did the replay start?") - sys.exit(1) - time.sleep(90) - #looking for landmark to mark benchmark start time and then wait for first round to finish - if keras_service.wait_for_word(word="defend", timeout=30, interval=1) is None: - logging.info("Didn't see the defend waypoint. Did the game crash?") + #looking for the FPS data graph + result = keras_service.wait_for_word("fps", timeout=30, interval=1) + if not result: + logging.info("Did not find the FPS graph. Did the replay start?") sys.exit(1) - test_start_time = int(time.time()) + 2 - time.sleep(460) + + test_start_time = int(time.time()) + time.sleep(98) #checking that first round has finished - result = keras_service.wait_for_word("complete", timeout=30, interval=1) + result = keras_service.wait_for_word("again", timeout=30, interval=1) if not result: - logging.info("First round doesn't appear to have finished. Did the replay start?") + logging.info("Didn't see the results screen. Did the test crash?") sys.exit(1) test_end_time = int(time.time()) - - am.copy_file(Path(cfg), ArtifactType.CONFIG_TEXT, "Marvel Rivals Video Config") + + am.copy_file(Path(CFG), ArtifactType.CONFIG_TEXT, "Marvel Rivals Video Config") + am.copy_file(Path(find_latest_benchmarkcsv()), ArtifactType.CONFIG_TEXT, "Marvel Rivals Benchmark CSV") logging.info("Run completed. Closing game.") time.sleep(2) @@ -274,4 +228,4 @@ if __name__ == "__main__": except Exception as ex: logging.error("something went wrong running the benchmark!") logging.exception(ex) - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/marvelrivals/marvelrivals_utils.py b/marvelrivals/marvelrivals_utils.py index c50abed..9f93aff 100644 --- a/marvelrivals/marvelrivals_utils.py +++ b/marvelrivals/marvelrivals_utils.py @@ -29,4 +29,13 @@ def read_resolution(): height = height_match.group(1) if width_match is not None: width = width_match.group(1) - return (height, width) \ No newline at end of file + return (height, width) + +def find_latest_benchmarkcsv(): + """find latest log from the benchmark""" + appdata_path = os.getenv('LOCALAPPDATA') + benchmarkcsv_dir = Path(appdata_path) / "Marvel" / "Saved" / "Benchmark" + files = [os.path.join(benchmarkcsv_dir, file) for file in os.listdir( + benchmarkcsv_dir) if os.path.isfile(os.path.join(benchmarkcsv_dir, file))] + latest_file = max(files, key=os.path.getmtime) + return latest_file diff --git a/marvelrivals/screenshots/play_1080.png b/marvelrivals/screenshots/play_1080.png deleted file mode 100644 index 49621b8..0000000 Binary files a/marvelrivals/screenshots/play_1080.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_1440.png b/marvelrivals/screenshots/play_1440.png deleted file mode 100644 index 72ad6e4..0000000 Binary files a/marvelrivals/screenshots/play_1440.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_2160.png b/marvelrivals/screenshots/play_2160.png deleted file mode 100644 index 2c52b1a..0000000 Binary files a/marvelrivals/screenshots/play_2160.png and /dev/null differ diff --git a/marvelrivals/screenshots/play_720.png b/marvelrivals/screenshots/play_720.png deleted file mode 100644 index 0e7d1d9..0000000 Binary files a/marvelrivals/screenshots/play_720.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_1080.png b/marvelrivals/screenshots/profile_1080.png deleted file mode 100644 index 5b42fdc..0000000 Binary files a/marvelrivals/screenshots/profile_1080.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_1440.png b/marvelrivals/screenshots/profile_1440.png deleted file mode 100644 index bc68225..0000000 Binary files a/marvelrivals/screenshots/profile_1440.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_2160.png b/marvelrivals/screenshots/profile_2160.png deleted file mode 100644 index e16adf7..0000000 Binary files a/marvelrivals/screenshots/profile_2160.png and /dev/null differ diff --git a/marvelrivals/screenshots/profile_720.png b/marvelrivals/screenshots/profile_720.png deleted file mode 100644 index 649452b..0000000 Binary files a/marvelrivals/screenshots/profile_720.png and /dev/null differ diff --git a/procyon_ai/ulprocai.py b/procyon_ai/ulprocai.py index 37104f2..467ab2d 100644 --- a/procyon_ai/ulprocai.py +++ b/procyon_ai/ulprocai.py @@ -1,4 +1,5 @@ """UL Procyon Computer Vision test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -32,7 +33,7 @@ from harness_utils.procyoncmd import ( get_cuda_devices, ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -48,74 +49,87 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "AMD_CPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_cpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": "CPU", - "device_id": "CPU", # TODO: Find a good way to report the CPU name here. - "test_name": "WinML CPU (FLOAT32)" + # TODO: Find a good way to report the CPU name here. + "device_id": "CPU", + "test_name": "cpu_float32", + "api": "winml" }, "AMD_GPU0": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[0], - "test_name": "WinML GPU (FLOAT32)" + "test_name": "gpu_float32", + "api": "winml" }, "AMD_GPU1": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"", - "process_name": "WinML.exe", + "process_name": "WinML.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0], - "test_name": "WinML GPU (FLOAT32)" + "test_name": "gpu_float32", + "api": "winml" }, "Intel_CPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_cpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "CPU", "device_name": OPENVINO_DEVICES["CPU"], - "test_name": "Intel OpenVINO CPU (FLOAT32)" + "test_name": "cpu_float32", + "api": "openvino" }, "Intel_GPU0": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO GPU 0 (FLOAT32)" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "gpu_float32", + "api": "openvino" }, "Intel_GPU1": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO GPU 1 (FLOAT32)" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "gpu_float32", + "api": "openvino" }, "Intel_NPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_npu.def\"", - "process_name": "OpenVino.exe", + "process_name": "OpenVino.exe", "device_id": "NPU", "device_name": OPENVINO_DEVICES.get("NPU", "None"), - "test_name": "Intel OpenVINO NPU (FLOAT32)" + "test_name": "npu_float32", + "api": "openvino" }, "NVIDIA_GPU": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_tensorrt.def\"", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "process_name": "TensorRT.exe", - "test_name": "NVIDIA TensorRT (FLOAT32)" + "process_name": "TensorRT.exe", + "test_name": "gpu_float32", + "api": "tensorrt" }, "Qualcomm_HTP": { "config": f"\"{CONFIG_DIR}\\ai_computer_vision_snpe.def\"", "device_id": "CPU", "device_name": "CPU", - "process_name": "SNPE.exe", - "test_name": "Qualcomm SNPE (INTEGER)" + "process_name": "SNPE.exe", + "test_name": "htp_integer", + "api": "snpe" }, } + + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -130,7 +144,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -160,16 +175,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() logging.info("Detected Windows ML Devices: %s", str(WINML_DEVICES)) @@ -203,15 +219,17 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI CV", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], + "api": BENCHMARK_CONFIG[args.engine]["api"], "test_version": find_test_version(), "device_name": BENCHMARK_CONFIG[args.engine]["device_name"], "procyon_version": find_procyon_version(), "unit": "score", - "score": score + "score": score } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/procyon_ai_img_gen/ulprocai_img_gen.py b/procyon_ai_img_gen/ulprocai_img_gen.py index 91ee41f..6a389c2 100644 --- a/procyon_ai_img_gen/ulprocai_img_gen.py +++ b/procyon_ai_img_gen/ulprocai_img_gen.py @@ -1,4 +1,5 @@ """UL Procyon AI Image Generation test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -26,7 +27,7 @@ from harness_utils.output import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -41,102 +42,117 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "AMD_GPU0_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"", - "process_name": "ort-directml.exe", + "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": "0", - "test_name": "ONNX Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "onnx" }, "AMD_GPU1_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"", - "process_name": "ort-directml.exe", + "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": "1" if len(list(WINML_DEVICES.values())) > 1 else "0", - "test_name": "ONNX Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "onnx" }, "AMD_GPU0_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"", "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[0], "device_id": "0", - "test_name": "ONNX Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "onnx" }, "AMD_GPU1_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"", "process_name": "ort-directml.exe", "device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0], "device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0], - "test_name": "ONNX Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "onnx" }, "Intel_GPU0_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion INT8" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_int8", + "api": "openvino" }, "Intel_GPU0_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion FP16" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_fp16", + "api": "openvino" }, "Intel_GPU0_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.0"), - "test_name": "Intel OpenVINO Stable Diffusion FP16 XL" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"), + "test_name": "stable_diffusion_fp16_xl", + "api": "openvino" }, "Intel_GPU1_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion INT8" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_int8", + "api": "openvino" }, "Intel_GPU1_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion FP16" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_fp16", + "api": "openvino" }, "Intel_GPU1_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"", - "process_name": "openvino.exe", + "process_name": "openvino.exe", "device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU", - "device_name": get_openvino_gpu(OPENVINO_DEVICES ,"GPU.1"), - "test_name": "Intel OpenVINO Stable Diffusion FP16 XL" + "device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"), + "test_name": "stable_diffusion_fp16_xl", + "api": "openvino" }, "NVIDIA_GPU_INT8": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion INT8" + "test_name": "stable_diffusion_int8", + "api": "tensorrt" }, "NVIDIA_GPU_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion FP16" + "test_name": "stable_diffusion_fp16", + "api": "tensorrt" }, "NVIDIA_GPU_XL_FP16": { "config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_tensorrt.def\"", - "process_name": "tensorrt.exe", + "process_name": "tensorrt.exe", "device_id": "cuda:0", "device_name": CUDA_DEVICES.get("cuda:0"), - "test_name": "NVIDIA TensorRT Stable Diffusion FP16 XL" + "test_name": "stable_diffusion_fp16_xl", + "api": "tensorrt" } } + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -151,7 +167,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -179,16 +196,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() logging.info("Detected Windows ML Devices: %s", str(WINML_DEVICES)) @@ -197,7 +215,9 @@ try: args = get_arguments() option = BENCHMARK_CONFIG[args.engine]["config"] - cmd = create_procyon_command(option, BENCHMARK_CONFIG[args.engine]["process_name"], BENCHMARK_CONFIG[args.engine]["device_id"]) + cmd = create_procyon_command( + option, BENCHMARK_CONFIG[args.engine]["process_name"], + BENCHMARK_CONFIG[args.engine]["device_id"]) logging.info('Starting benchmark!') logging.info(cmd) start_time = time.time() @@ -221,16 +241,18 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI Image Generation", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], + "api": BENCHMARK_CONFIG[args.engine]["api"], "test_version": find_test_version(), "device_name": BENCHMARK_CONFIG[args.engine]["device_name"], "procyon_version": find_procyon_version(), "unit": "score", "score": score - + } - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/procyon_ai_text_generation/ulprocai_text_gen.py b/procyon_ai_text_generation/ulprocai_text_gen.py index 2655b38..5fe9fb2 100644 --- a/procyon_ai_text_generation/ulprocai_text_gen.py +++ b/procyon_ai_text_generation/ulprocai_text_gen.py @@ -1,4 +1,5 @@ """UL Procyon AI Text Generation test script""" +# pylint: disable=no-name-in-module from argparse import ArgumentParser import logging from pathlib import Path @@ -20,7 +21,7 @@ from harness_utils.output import ( ) ##### -### Globals +# Globals ##### SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" @@ -31,71 +32,83 @@ CONFIG_DIR = SCRIPT_DIR / "config" BENCHMARK_CONFIG = { "All_Models_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_all.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "All LLM Model Text Generation" + "test_name": "all_models", + "api": "onnx" }, "Llama_2_13B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2.def\"", "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "LLama 2 Text Generation" + "test_name": "llama_2_13b", + "api": "onnx" }, - "Llama_3_1_8B_ONNX": { + "Llama_3_1_8B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Llama 3.1 Text Generation" + "test_name": "llama_3_1_8b", + "api": "onnx" }, "Mistral_7B_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Mistral Text Generation" + "test_name": "mistral_7b", + "api": "onnx" }, "Phi_3_5_ONNX": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Phi Text Generation" + "test_name": "phi_3_5", + "api": "onnx" }, "All_Models_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_all_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "All LLM Model Text Generation" + "test_name": "all_models", + "api": "openvino" }, "Llama_2_13B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2_openvino.def\"", "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "LLama 2 Text Generation" + "test_name": "llama_2_13b", + "api": "openvino" }, - "Llama_3_1_8B_OPENVINO": { + "Llama_3_1_8B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Llama 3.1 Text Generation" + "test_name": "llama_3_1_8b", + "api": "openvino" }, "Mistral_7B_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Mistral Text Generation" + "test_name": "mistral_7b", + "api": "openvino" }, "Phi_3_5_OPENVINO": { "config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi_openvino.def\"", - "process_name": "Handler.exe", + "process_name": "Handler.exe", "result_regex": r"(\d+)", - "test_name": "Phi Text Generation" + "test_name": "phi_3_5", + "api": "openvino" } } + RESULTS_FILENAME = "result.xml" REPORT_PATH = LOG_DIR / RESULTS_FILENAME + def setup_logging(): """setup logging""" - setup_log_directory(LOG_DIR) + setup_log_directory(str(LOG_DIR)) logging.basicConfig(filename=LOG_DIR / "harness.log", format=DEFAULT_LOGGING_FORMAT, datefmt=DEFAULT_DATE_FORMAT, @@ -110,7 +123,8 @@ def get_arguments(): """get arguments""" parser = ArgumentParser() parser.add_argument( - "--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys()) + "--engine", dest="engine", help="Engine test type", required=True, + choices=BENCHMARK_CONFIG.keys()) argies = parser.parse_args() return argies @@ -129,16 +143,17 @@ def run_benchmark(process_name, command_to_run): while True: now = time.time() elapsed = now - start_time - if elapsed >= 60: #seconds + if elapsed >= 60: # seconds raise ValueError("BenchMark subprocess did not start in time") process = is_process_running(process_name) if process is not None: process.nice(psutil.HIGH_PRIORITY_CLASS) break time.sleep(0.2) - _, _ = proc.communicate() # blocks until 3dmark exits + _, _ = proc.communicate() # blocks until 3dmark exits return proc + try: setup_logging() args = get_arguments() @@ -165,7 +180,8 @@ try: sys.exit(1) report = { - "test": BENCHMARK_CONFIG[args.engine]["test_name"], + "test": "Procyon AI Text Generation", + "test_parameter": BENCHMARK_CONFIG[args.engine]["test_name"], "unit": "score", "score": score, "start_time": seconds_to_milliseconds(start_time), @@ -175,7 +191,7 @@ try: logging.info("Benchmark took %.2f seconds", elapsed_test_time) logging.info("Score was %s", score) - write_report_json(LOG_DIR, "report.json", report) + write_report_json(str(LOG_DIR), "report.json", report) else: session_report = [] @@ -198,19 +214,19 @@ try: report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": test_type[0], + "test": "Procyon AI Text Generation", + "test_parameter": test_type[1]["test_name"], + "api": test_type[1]["api"], "test_version": find_test_version(), "procyon_version": find_procyon_version(), "unit": "score", "score": score - - } - + } session_report.append(report) - write_report_json(LOG_DIR, "report.json", session_report) + write_report_json(str(LOG_DIR), "report.json", session_report) except Exception as e: logging.error("Something went wrong running the benchmark!") diff --git a/procyon_ai_text_generation/utils.py b/procyon_ai_text_generation/utils.py index c060062..9557076 100644 --- a/procyon_ai_text_generation/utils.py +++ b/procyon_ai_text_generation/utils.py @@ -10,6 +10,7 @@ import logging SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR / "run" + def is_process_running(process_name): """check if given process is running""" for process in psutil.process_iter(['pid', 'name']): @@ -17,6 +18,7 @@ def is_process_running(process_name): return process return None + def regex_find_score_in_xml(result_regex): """Reads score from local game log""" score_pattern = re.compile(result_regex) @@ -30,26 +32,29 @@ def regex_find_score_in_xml(result_regex): score_value = score_match.group(1) return score_value + def get_install_path() -> str: """Gets the path to the Steam installation directory from the SteamPath registry key""" reg_path = r"Software\UL\Procyon" - reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, reg_path, 0, winreg.KEY_READ) - value, _ = winreg.QueryValueEx(reg_key, "InstallDir") + reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + reg_path, 0, winreg.KEY_READ) + value, _ = winreg.QueryValueEx(reg_key, "InstallDir") return value + def find_procyon_version() -> str: """Gets the version of an executable located in the install path.""" install_path = get_install_path() - + if not install_path: logging.info("Installation path not found.") - return None + return "" exe_path = os.path.join(install_path, "ProcyonCmd.exe") if not os.path.exists(exe_path): - logging.info(f"Executable not found at {exe_path}") - return None + logging.info("Executable not found at %s", exe_path) + return "" try: # Get all file version info @@ -61,7 +66,7 @@ def find_procyon_version() -> str: if ms is None or ls is None: logging.info("No FileVersionMS or FileVersionLS found.") - return None + return "" # Convert to human-readable version: major.minor.build.revision major = ms >> 16 @@ -73,29 +78,31 @@ def find_procyon_version() -> str: return version except Exception as e: - logging.info(f"Error retrieving version info from {exe_path}: {e}") - return None # Return None if version info retrieval fails + logging.info("Error retrieving version info from %s: %s", exe_path, e) + return "" # Return empty string if version info retrieval fails + def find_test_version() -> str: """Gets the version of an executable located in the chops path.""" chops_path = "C:\\ProgramData\\UL\\Procyon\\chops\\dlc\\ai-textgeneration-benchmark\\x64" - logging.info(f"The install path for the test is {chops_path}") - + logging.info("The install path for the test is %s", chops_path) + if not chops_path: logging.info("Installation path not found.") - return None + return "" exe_path = os.path.join(chops_path, "Handler.exe") if not os.path.exists(exe_path): - logging.info(f"Executable 'Handler.exe' not found at {exe_path}") - return None + logging.info("Executable 'Handler.exe' not found at %s", exe_path) + return "" try: - lang, codepage = win32api.GetFileVersionInfo(exe_path, "\\VarFileInfo\\Translation")[0] + lang, codepage = win32api.GetFileVersionInfo( + exe_path, "\\VarFileInfo\\Translation")[0] str_info_path = f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion" - return win32api.GetFileVersionInfo(exe_path, str_info_path) + return str(win32api.GetFileVersionInfo(exe_path, str_info_path)) except Exception as e: - logging.info(f"Error retrieving version info from {exe_path}: {e}") - return None # Return None if version info retrieval fails \ No newline at end of file + logging.info("Error retrieving version info from %s: %s", exe_path, e) + return "" # Return empty string if version info retrieval fails diff --git a/pugetbench/manifest.yaml b/pugetbench/manifest.yaml index 1d554ea..c33f0a4 100644 --- a/pugetbench/manifest.yaml +++ b/pugetbench/manifest.yaml @@ -15,6 +15,6 @@ options: - aftereffects - resolve tooltip: Select which test to run - - name: benchmark + - name: benchmark_version type: input tooltip: Version of benchmark to run diff --git a/pugetbench/pugetbench.py b/pugetbench/pugetbench.py index e544cb3..b40bef4 100644 --- a/pugetbench/pugetbench.py +++ b/pugetbench/pugetbench.py @@ -135,19 +135,19 @@ def main(): score = 0 test = "" if args.app == "premierepro": - test = "PugetBench Adobe Premiere Pro" + test = "Adobe Premiere Pro" if version is None: version = get_premierepro_version() elif args.app == "photoshop": - test = "PugetBench Adobe Photoshop" + test = "Adobe Photoshop" if version is None: version = get_photoshop_version() elif args.app == "aftereffects": - test = "PugetBench Adobe After Effects" + test = "Adobe After Effects" if version is None: version = get_aftereffects_version() elif args.app == "resolve": - test = "PugetBench Davinci Resolve Studio" + test = "Davinci Resolve Studio" if version is None: version = get_davinci_version() + "-studio" @@ -162,7 +162,8 @@ def main(): report = { "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "test": test, + "test": "PugetBench", + "test_parameter": test, "app_version": version, "benchmark_version": args.benchmark_version, "pugetbench_version": get_pugetbench_version(), diff --git a/reddeadredemption2/reddeadredemption2.py b/reddeadredemption2/reddeadredemption2.py index 895d1b1..b016c1e 100644 --- a/reddeadredemption2/reddeadredemption2.py +++ b/reddeadredemption2/reddeadredemption2.py @@ -32,6 +32,7 @@ SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run") CONFIG_FULL_PATH = Path("C:/Users/", getpass.getuser(), "Documents", "Rockstar Games", "Red Dead Redemption 2", "Settings", "system.xml") +user.FAILSAFE = False def run_benchmark(): """Starts the benchmark""" @@ -39,8 +40,15 @@ def run_benchmark(): setup_start_time = int(time.time()) exec_steam_run_command(STEAM_GAME_ID) am = ArtifactManager(LOG_DIRECTORY) + time.sleep(80) + # patch to look for seasonal popup + result = kerasService.look_for_word_vulkan("strange", attempts=30, interval=1) + if result: + user.press("enter") + time.sleep(3) + # Press Z to enter settings result = kerasService.look_for_word_vulkan("settings", attempts=30, interval=1) if not result: diff --git a/rocket_league/rocket_league.py b/rocket_league/rocket_league.py index c85d6e2..96532e7 100644 --- a/rocket_league/rocket_league.py +++ b/rocket_league/rocket_league.py @@ -30,7 +30,7 @@ CONFIG_PATH = Path(f"C:\\Users\\{USERNAME}\\Documents\\My Games\\Rocket League\\ PROCESS_NAME = "rocketleague.exe" EXECUTABLE_PATH = find_epic_executable() GAME_ID = "9773aa1aa54f4f7b80e44bef04986cea%3A530145df28a24424923f5828cc9031a1%3ASugar?action=launch&silent=true" -gamefoldername = "rocketleague" +GAMEFOLDERNAME = "rocketleague" am = ArtifactManager(LOG_DIRECTORY) gamepad = LTTGamePadDS4() @@ -63,7 +63,7 @@ def camera_cycle(max_attempts=10): :param check_duration: How long (in seconds) to look for the word before pressing a button. :param button: The gamepad button to press if word is not found. """ - for attempt in range(max_attempts): + for _ in range(max_attempts): # Try finding the word within check_duration seconds found = kerasService.look_for_word(word="player", attempts=2, interval=0.2) @@ -136,6 +136,11 @@ def run_benchmark(): gamepad.single_dpad_press(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH) time.sleep(0.5) + if kerasService.look_for_word(word="club", attempts=5, interval=0.2): + logging.info('Saw Create a Club. Navigating accordingly.') + gamepad.single_dpad_press(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH) + time.sleep(0.5) + gamepad.dpad_press_n_times(direction=vg.DS4_DPAD_DIRECTIONS.DS4_BUTTON_DPAD_SOUTH, n=2, pause=0.8) time.sleep(0.5) gamepad.single_button_press(button=vg.DS4_BUTTONS.DS4_BUTTON_CROSS) @@ -239,7 +244,7 @@ try: "resolution": format_resolution(width, height), "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time), - "game_version": find_eg_game_version(gamefoldername) + "game_version": find_eg_game_version(GAMEFOLDERNAME) } write_report_json(LOG_DIRECTORY, "report.json", report) diff --git a/shadowofthetombraider/shadowofthetombraider.py b/shadowofthetombraider/shadowofthetombraider.py index 1f8717d..85d41c6 100644 --- a/shadowofthetombraider/shadowofthetombraider.py +++ b/shadowofthetombraider/shadowofthetombraider.py @@ -27,6 +27,7 @@ PROCESS_NAME = "SOTTR.exe" SCRIPT_DIR = Path(__file__).resolve().parent LOG_DIR = SCRIPT_DIR.joinpath("run") +user.FAILSAFE = False def setup_logging(): """default logging config""" diff --git a/stellaris/settings/pdx_settings.txt b/stellaris/settings/pdx_settings.txt index ac632d6..b596e26 100644 --- a/stellaris/settings/pdx_settings.txt +++ b/stellaris/settings/pdx_settings.txt @@ -20,7 +20,7 @@ version=0 } "refreshRate"={ - value="50" + value="60" version=0 } } diff --git a/stellaris/settings/settings.txt b/stellaris/settings/settings.txt new file mode 100644 index 0000000..8cd768c --- /dev/null +++ b/stellaris/settings/settings.txt @@ -0,0 +1,73 @@ +force_pow2_textures=no +language="l_english" +graphics= +{ + size= + { + x=1920 + y=1080 + } + + gui_scale=1.000000 + gui_safe_ratio=1.000000 + refreshRate=60 + fullScreen=yes + borderless=no + display_index=0 + renderer=1 + shadowSize=2048 + multi_sampling=4 + multi_sampling_quality=0 + maxanisotropy=16 + gamma=50.000000 + vsync=yes +} +sound_fx_volume=50.000000 +music_volume=50.000000 +scroll_speed=50.000000 +camera_rotation_speed=50.000000 +zoom_speed=50.000000 +mouse_speed=50.000000 +soundgroup="l_english" +master_volume=100.000000 +ambient_volume=50.000000 +dev_master_volume=75.000000 +input_type=0 +precise_mouse_wheel=no +mouse_wheel_acceleration=0.000000 +mouse_wheel_base_speed=1.000000 +input_type=0 +crisis_conversation_speech=yes +voice_volume=50.000000 +tts_volume=75.000000 +camera_look_sensitivity=1.000000 +camera_speed=50.000000 +autosave=0 +tutorial=0 +completed_tutorial_missions=0 +gfx_quality=2 +bloom= +{ + quality=2 + lens_flare=yes +} +cvaa_settings= +{ + mouse_side_button_mode=0 + cvaa_tts_enabled=no + cvaa_stt_enabled=no + general_tts_enabled=no + tooltip_tts_enabled=no + cvaa_chat_visual_alerts=no + cvaa_stt_hotkey_enabled=no + cvaa_chat_large_fonts=no + cvaa_stt_hotkey="" +} +mp_max_ticks_ahead=30 +mapmode_sectors=no +show_startup_game_info="Phoenix v4.0" +hyperlane_opacity=0.300000 +hotkey_activation_delay=0.100000 +hotkey_actualization_delay=0.100000 +hide_unowned_content=no +transfer_speed="fast" diff --git a/stellaris/settings/standard_settings.txt b/stellaris/settings/standard_settings.txt deleted file mode 100644 index 1368e66..0000000 --- a/stellaris/settings/standard_settings.txt +++ /dev/null @@ -1,45 +0,0 @@ -force_pow2_textures=no -language="l_english" -graphics={ - size={ - x=1920 - y=1080 - } - - gui_scale=1.000000 - gui_safe_ratio=1.000000 - refreshRate=50 - fullScreen=yes - borderless=no - display_index=0 - renderer=0 - shadowSize=2048 - multi_sampling=0 - multi_sampling_quality=0 - maxanisotropy=16 - gamma=50.000000 - vsync=yes -} -sound_fx_volume=20.000000 -music_volume=20.000000 -scroll_speed=50.000000 -camera_rotation_speed=50.000000 -zoom_speed=50.000000 -mouse_speed=50.000000 -soundgroup="l_english" -master_volume=52.000000 -ambient_volume=21.000000 -dev_master_volume=75.000000 -input_type=0 -voice_volume=22.000000 -tts_volume=75.000000 -camera_look_sensitivity=1.000000 -camera_speed=50.000000 -autosave=0 -tutorial=0 -completed_tutorial_missions=0 -gfx_quality=0 -bloom={ - quality=0 - lens_flare=no -} \ No newline at end of file diff --git a/stellaris/stellaris.py b/stellaris/stellaris.py index 9d8d135..9b084eb 100644 --- a/stellaris/stellaris.py +++ b/stellaris/stellaris.py @@ -23,6 +23,7 @@ from harness_utils.output import ( ) from harness_utils.steam import get_app_install_location from harness_utils.keras_service import KerasService +from harness_utils.artifacts import ArtifactManager, ArtifactType SCRIPT_DIR = Path(__file__).resolve().parent @@ -67,12 +68,38 @@ def run_benchmark(keras_host, keras_port): start_game() setup_start_time = int(time.time()) time.sleep(5) + am = ArtifactManager(LOG_DIR) + + patchnotes = keras_service.wait_for_word("close", interval=0.5, timeout=100) + if patchnotes: + gui.moveTo(patchnotes["x"], patchnotes["y"]) + time.sleep(0.2) + gui.mouseDown() + time.sleep(0.2) + gui.mouseUp() + time.sleep(0.2) result = keras_service.wait_for_word("credits", interval=0.5, timeout=100) if not result: logging.info("Could not find the paused notification. Unable to mark start time!") sys.exit(1) + result = keras_service.look_for_word("settings", attempts=10, interval=1) + if not result: + logging.info("Did not find the settings button. Is there something wrong on the screen?") + sys.exit(1) + + gui.moveTo(result["x"], result["y"]) + time.sleep(0.2) + gui.mouseDown() + time.sleep(0.2) + gui.mouseUp() + time.sleep(0.5) + am.take_screenshot("settings.png", ArtifactType.CONFIG_IMAGE, "settings") + + time.sleep(0.2) + user.press("esc") + result = keras_service.look_for_word("load", attempts=10, interval=1) if not result: logging.info("Did not find the load save menu. Is there something wrong on the screen?") @@ -102,9 +129,9 @@ def run_benchmark(keras_host, keras_port): logging.info("Could not find the paused notification. Unable to mark start time!") sys.exit(1) - result = keras_service.look_for_word("government", attempts=10, interval=1) + result = keras_service.look_for_word("overview", attempts=10, interval=1) if not result: - logging.info("Did not find the load latest save button. Did keras click correctly?") + logging.info("Did not find the overview in the corner. Did the game load?") sys.exit(1) gui.moveTo(result["x"], result["y"]) @@ -141,6 +168,8 @@ def run_benchmark(keras_host, keras_port): score = find_score_in_log() logging.info("The one year passed in %s seconds", score) terminate_processes(PROCESS_NAME) + am.create_manifest() + return test_start_time, test_end_time, score diff --git a/stellaris/stellaris_utils.py b/stellaris/stellaris_utils.py index 4b88b8b..7a0c00d 100644 --- a/stellaris/stellaris_utils.py +++ b/stellaris/stellaris_utils.py @@ -16,15 +16,16 @@ PROCESS_NAME = "stellaris.exe" STEAM_GAME_ID = 281990 CONFIG_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris") LOG_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\logs") -BENCHMARK_LOCATION = Path(f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\save games\\BENCHMARK") -CONFIG_FILENAME = "standard_settings.txt" +BENCHMARK_LOCATION = Path( + f"C:\\Users\\{USERNAME}\\Documents\\Paradox Interactive\\Stellaris\\save games\\BENCHMARK") +CONFIG_FILENAME = "settings.txt" LOG_FILE = "game.log" benchmark_files = [ "benchmark.ini", "pdx_settings.txt", - "standard_settings.txt" + "settings.txt" ] @@ -76,7 +77,7 @@ def copy_benchmarkfiles() -> None: def copy_save_from_network_drive(file_name, destination): """copy save file from network drive""" - network_dir = Path("\\\\Labs\\labs\\03_ProcessingFiles\\Stellaris") + network_dir = Path("\\\\labs.lmg.gg\\labs\\03_ProcessingFiles\\Stellaris") source_path = network_dir.joinpath(file_name) logging.info("Copying %s from %s", file_name, source_path) shutil.copyfile(source_path, destination) diff --git a/superposition/superposition.py b/superposition/superposition.py index f9af792..12026ad 100644 --- a/superposition/superposition.py +++ b/superposition/superposition.py @@ -43,21 +43,22 @@ formatter = logging.Formatter(LOGGING_FORMAT) console.setFormatter(formatter) logging.getLogger('').addHandler(console) -cmd = f'{INSTALL_DIR}\\{EXECUTABLE}' -argstr = f"-fullscreen 1 -mode default -api {args.api} -quality {args.preset} -iterations 1" -argstr += f" -log_txt {log_dir}\\log.txt" +CMD = f'{INSTALL_DIR}\\{EXECUTABLE}' +ARGSTR = f"-fullscreen 1 -mode default -api {args.api} -quality {args.preset} -iterations 1" +ARGSTR += f" -log_txt {log_dir}\\log.txt" -logging.info(cmd) -logging.info(argstr) -argies = argstr.split(" ") -cmd = cmd.rstrip() -with Popen([cmd, *argies]) as process: +logging.info(CMD) +logging.info(ARGSTR) +argies = ARGSTR.split(" ") +CMD = CMD.rstrip() +with Popen([CMD, *argies]) as process: EXIT_CODE = process.wait() if EXIT_CODE > 0: logging.error("Test failed!") sys.exit(EXIT_CODE) +SCORE = "" pattern = re.compile(r"Score: (\d+)") log_path = os.path.join(log_dir, "log.txt") with open(log_path, encoding="utf-8") as log: @@ -65,11 +66,13 @@ with open(log_path, encoding="utf-8") as log: for line in lines: match = pattern.search(line) if match: - score = match.group(1) + SCORE = match.group(1) report = { - "test": f"Unigine Superposition 2017 {args.preset} ${args.api}", - "score": score, + "test": "Unigine Superposition", + "test_parameter": f"{args.api}", + "test_preset": args.preset, + "score": SCORE, "unit": "score" } diff --git a/the_last_of_us_part_i/the_last_of_us_part_i.py b/the_last_of_us_part_i/the_last_of_us_part_i.py index 0e67136..b760049 100644 --- a/the_last_of_us_part_i/the_last_of_us_part_i.py +++ b/the_last_of_us_part_i/the_last_of_us_part_i.py @@ -20,8 +20,8 @@ from harness_utils.output import ( ) from harness_utils.process import terminate_processes from harness_utils.steam import ( - get_registry_active_user, - exec_steam_run_command, + get_registry_active_user, + exec_steam_run_command, ) from harness_utils.misc import press_n_times @@ -34,19 +34,20 @@ PROCESS_NAME = "tlou" user.FAILSAFE = False + def take_screenshots(am: ArtifactManager) -> None: """Take screenshots of the benchmark settings""" logging.info("Taking screenshots of benchmark settings") - press_n_times("s",2,0.2 ) + press_n_times("s", 2, 0.2) user.press("enter") - press_n_times("s",4,0.2 ) + press_n_times("s", 4, 0.2) user.press("enter") am.take_screenshot("video1.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings1") - press_n_times("s",15,0.2) + press_n_times("s", 15, 0.2) am.take_screenshot("video2.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings2") - press_n_times("s",6, 0.2) + press_n_times("s", 6, 0.2) am.take_screenshot("video3.png", ArtifactType.CONFIG_IMAGE, "screenshot of video settings3") user.press("backspace") @@ -69,6 +70,7 @@ def take_screenshots(am: ArtifactManager) -> None: user.press("backspace") press_n_times("w", 2, 0.2) + def navigate_main_menu(am: ArtifactManager) -> None: """Input to navigate main menu""" logging.info("Navigating main menu") @@ -174,7 +176,7 @@ try: start_time, end_time = run_benchmark() steam_id = get_registry_active_user() config_path = os.path.join( - os.environ["HOMEPATH"], "Saved Games" ,"The Last of Us Part I", + os.environ["HOMEPATH"], "Saved Games", "The Last of Us Part I", "users", str(steam_id), "screeninfo.cfg" ) height, width = get_resolution(config_path) @@ -183,8 +185,8 @@ try: "start_time": seconds_to_milliseconds(start_time), "end_time": seconds_to_milliseconds(end_time) } - write_report_json(LOG_DIRECTORY, "report.json", report) + except Exception as e: logging.error("Something went wrong running the benchmark!") logging.exception(e) diff --git a/the_last_of_us_part_ii/manifest.yaml b/the_last_of_us_part_ii/manifest.yaml new file mode 100644 index 0000000..f530266 --- /dev/null +++ b/the_last_of_us_part_ii/manifest.yaml @@ -0,0 +1,9 @@ +friendly_name: "The Last of Us Part II" +executable: "tlou2.py" +process_name: "tlou-ii.exe" +output_dir: "run" +options: + - name: kerasHost + type: input + - name: kerasPort + type: input diff --git a/the_last_of_us_part_ii/tlou2.py b/the_last_of_us_part_ii/tlou2.py new file mode 100644 index 0000000..6c23d3a --- /dev/null +++ b/the_last_of_us_part_ii/tlou2.py @@ -0,0 +1,286 @@ +"""The Last of Us Part I test script""" +import logging +from pathlib import Path +import time +import sys +import pydirectinput as user +import getpass + +import winreg # for accessing settings, including resolution, in the registry + +import shutil + +sys.path.insert(1, str(Path(sys.path[0]).parent)) + +from harness_utils.keras_service import KerasService +from harness_utils.output import ( + format_resolution, + seconds_to_milliseconds, + write_report_json, + setup_logging, +) +from harness_utils.process import terminate_processes +from harness_utils.steam import ( + exec_steam_run_command, +) + +from harness_utils.artifacts import ArtifactManager, ArtifactType + +from harness_utils.misc import ( + int_time, + find_word, + press_n_times, + keras_args) + +USERNAME = getpass.getuser() +STEAM_GAME_ID = 2531310 +SCRIPT_DIRECTORY = Path(__file__).resolve().parent +LOG_DIRECTORY = SCRIPT_DIRECTORY / "run" +PROCESS_NAME = "tlou-ii.exe" + +user.FAILSAFE = False + + +def reset_savedata(): + """ + Deletes the savegame folder from the local directory and replaces it with a new one from the network drive. + """ + local_savegame_path = Path( + f"C:\\Users\\{USERNAME}\\Documents\\The Last of Us Part II\\76561199405246658\\savedata") # make this global + network_savegame_path = Path( + r"\\labs.lmg.gg\Labs\03_ProcessingFiles\The Last of Us Part II\savedata") + + # Delete the local savedata folder if it exists + if local_savegame_path.exists() and local_savegame_path.is_dir(): + shutil.rmtree(local_savegame_path) + logging.info("Deleted local savedata folder: %s", local_savegame_path) + + # Copy the savedata folder from the network drive + try: + shutil.copytree(network_savegame_path, local_savegame_path) + logging.info("Copied savedata folder from %s to %s", + network_savegame_path, local_savegame_path) + except Exception as e: + logging.error("Failed to copy savedata folder: %s", e) + + # Check if the newly copied directory contains a folder called SAVEFILE0A + + +def delete_autosave(): + """ + Deletes the autosave folder from the local directory if it exists. + """ + local_savegame_path = Path( + f"C:\\Users\\{USERNAME}\\Documents\\The Last of Us Part II\\76561199405246658\\savedata") + savefile_path = local_savegame_path / "SAVEFILE0A" # check for autosaved file, delete if exists + if savefile_path.exists() and savefile_path.is_dir(): + shutil.rmtree(savefile_path) + logging.info("Deleted folder: %s", savefile_path) + + +def get_current_resolution(): + """ + Returns: + tuple: (width, height) + Reads resolutions settings from registry + """ + key_path = r"Software\Naughty Dog\The Last of Us Part II\Graphics" + fullscreen_width = read_registry_value(key_path, "FullscreenWidth") + fullscreen_height = read_registry_value(key_path, "FullscreenHeight") + + return (fullscreen_width, fullscreen_height) + + +def read_registry_value(key_path, value_name): + """ + Reads value from registry + A helper function for get_current_resolution + """ + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_path) as key: + value, _ = winreg.QueryValueEx(key, value_name) + return value + except FileNotFoundError: + logging.error("Registry key not found: %s", value_name) + return None + except OSError as e: + logging.error("Error reading registry value: %s", e) + return None + + +def run_benchmark(keras_service: KerasService) -> tuple: + """Starts Game, Sets Settings, and Runs Benchmark""" + exec_steam_run_command(STEAM_GAME_ID) + setup_start_time = int_time() + am = ArtifactManager(LOG_DIRECTORY) + + if keras_service.wait_for_word(word="sony", timeout=60, interval=0.2) is None: + logging.error("Couldn't find 'sony'") + else: + user.press("escape") + + find_word(keras_service, "story", "Couldn't find main menu : 'story'") + + press_n_times("down", 2) + + # navigate settings + navigate_settings(am, keras_service) + + find_word(keras_service, "story", "Couldn't find main menu the second time : 'story'") + + press_n_times("up", 2) + + user.press("space") + + time.sleep(0.3) + + user.press("space") + + if keras_service.wait_for_word(word="continue", timeout=5, interval=0.2) is None: + user.press("down") + else: + press_n_times("down", 2) + + delete_autosave() + + time.sleep(0.3) + + user.press("space") + + time.sleep(0.3) + + if keras_service.wait_for_word(word="autosave", timeout=5, interval=0.2) is None: + + user.press("space") + + else: + user.press("up") + + time.sleep(0.3) + + user.press("space") + + time.sleep(0.3) + + user.press("left") + + time.sleep(0.3) + + user.press("space") + + setup_end_time = test_start_time = test_end_time = int_time() + + elapsed_setup_time = setup_end_time - setup_start_time + logging.info("Setup took %f seconds", elapsed_setup_time) + + # time of benchmark usually is 4:23 = 263 seconds + + if keras_service.wait_for_word(word="man", timeout=100, interval=0.2) is not None: + + test_start_time = int_time() - 14 + time.sleep(240) + + else: + + logging.error("couldn't find 'man'") + time.sleep(150) + + if keras_service.wait_for_word(word="rush", timeout=100, interval=0.2) is not None: + + time.sleep(3) + test_end_time = int_time() + + else: + + logging.error("couldn't find 'rush', marks end of benchmark") + test_end_time = int_time() + + elapsed_test_time = test_end_time - test_start_time + logging.info("Test took %f seconds", elapsed_test_time) + + terminate_processes(PROCESS_NAME) + + am.create_manifest() + + return test_start_time, test_end_time + + +def navigate_settings(am: ArtifactManager, keras: KerasService) -> None: + """Navigate through settings and take screenshots. + Exits to main menu after taking screenshots. + """ + + user.press("space") + + find_word(keras, "display", "Couldn't find display") + + time.sleep(5) # slow cards may miss the first down + + press_n_times("down", 4) + + user.press("space") + + time.sleep(0.5) + + find_word(keras, "resolution", "Couldn't find resolution") + + am.take_screenshot("display1.png", ArtifactType.CONFIG_IMAGE, "display settings 1") + + user.press("up") + + find_word(keras, "brightness", "Couldn't find brightness") + + am.take_screenshot("display2.png", ArtifactType.CONFIG_IMAGE, "display settings 2") + + user.press("q") # swaps to graphics settings + + time.sleep(0.5) + + find_word(keras, "preset", "Couldn't find preset") + + am.take_screenshot("graphics1.png", ArtifactType.CONFIG_IMAGE, "graphics settings 1") + + user.press("up") + + find_word(keras, "dirt", "Couldn't find dirt") + + am.take_screenshot("graphics3.png", ArtifactType.CONFIG_IMAGE, + "graphics settings 3") # is at the bottom of the menu + + press_n_times("up", 13) + + find_word(keras, "scattering", "Couldn't find scattering") + + am.take_screenshot("graphics2.png", ArtifactType.CONFIG_IMAGE, "graphics settings 2") + + press_n_times("escape", 2) + + +def main(): + """Main function to run the benchmark""" + try: + logging.info("Starting The Last of Us Part II benchmark") + + keras_service = KerasService(keras_args().keras_host, keras_args().keras_port) + + reset_savedata() + + start_time, end_time = run_benchmark(keras_service) + resolution_tuple = get_current_resolution() + report = { + "resolution": format_resolution(resolution_tuple[0], resolution_tuple[1]), + "start_time": seconds_to_milliseconds(start_time), # secconds to miliseconds + "end_time": seconds_to_milliseconds(end_time), + } + write_report_json(LOG_DIRECTORY, "report.json", report) + + except Exception as e: + logging.error("An error occurred: %s", e) + logging.exception(e) + terminate_processes(PROCESS_NAME) + sys.exit(1) + + +if __name__ == "__main__": + setup_logging(LOG_DIRECTORY) + main() diff --git a/tinytinaswonderland/tinytinaswonderland.py b/tinytinaswonderland/tinytinaswonderland.py index 85dd0c2..d40d067 100644 --- a/tinytinaswonderland/tinytinaswonderland.py +++ b/tinytinaswonderland/tinytinaswonderland.py @@ -22,6 +22,8 @@ LOG_DIRECTORY = SCRIPT_DIRECTORY.joinpath("run") STEAM_GAME_ID = 1286680 EXECUTABLE = "Wonderlands.exe" +user.FAILSAFE = False + def setup_logging(): """default logging config""" setup_log_directory(LOG_DIRECTORY) diff --git a/xz/xz_utils.py b/xz/xz_utils.py index ea340e1..e3ba847 100644 --- a/xz/xz_utils.py +++ b/xz/xz_utils.py @@ -16,12 +16,12 @@ def xz_executable_exists() -> bool: def copy_from_network_drive(): """Download xz from network drive""" - source = r"\\Labs\labs\01_Installers_Utilities\xz\xz_5.6.2_x86_64.exe" + source = r"\\labs.lmg.gg\labs\01_Installers_Utilities\xz\xz_5.6.2_x86_64.exe" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, XZ_EXECUTABLE) shutil.copyfile(source, destination) - source = r"\\Labs\labs\03_ProcessingFiles\Compression\tq_dlss_explained_1080p.mp4" + source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Compression\tq_dlss_explained_1080p.mp4" root_dir = os.path.dirname(os.path.realpath(__file__)) destination = os.path.join(root_dir, "tq_dlss_explained_1080p.mp4") shutil.copyfile(source, destination) diff --git a/ycruncher/ycruncher.py b/ycruncher/ycruncher.py index bf775ea..328da42 100644 --- a/ycruncher/ycruncher.py +++ b/ycruncher/ycruncher.py @@ -12,8 +12,8 @@ sys.path.insert(1, os.path.join(sys.path[0], "..")) from harness_utils.output import write_report_json, DEFAULT_LOGGING_FORMAT, DEFAULT_DATE_FORMAT SCRIPT_DIR = Path(__file__).resolve().parent -LOG_DIR = SCRIPT_DIR.joinpath("run") -EXECUTABLE_PATH = SCRIPT_DIR.joinpath(YCRUNCHER_FOLDER_NAME, "y-cruncher.exe") +LOG_DIR = SCRIPT_DIR / "run" +EXECUTABLE_PATH = SCRIPT_DIR / YCRUNCHER_FOLDER_NAME / "y-cruncher.exe" def setup_logging(): @@ -79,7 +79,7 @@ def main(): report = { "start_time": start_time, - "version": "v0.8.5.9543", + "version": "v0.8.5.9545b", "end_time": end_time, "score": avg_score, "unit": "seconds", diff --git a/ycruncher/ycruncher_utils.py b/ycruncher/ycruncher_utils.py index 67eb9b2..fd42828 100644 --- a/ycruncher/ycruncher_utils.py +++ b/ycruncher/ycruncher_utils.py @@ -4,11 +4,11 @@ from zipfile import ZipFile from pathlib import Path import requests -YCRUNCHER_FOLDER_NAME = "y-cruncher v0.8.5.9543" -YCRUNCHER_ZIP_NAME = "y-cruncher.v0.8.5.9543.zip" - +YCRUNCHER_FOLDER_NAME = "y-cruncher v0.8.6.9545" +YCRUNCHER_ZIP_NAME = "y-cruncher.v0.8.6.9545b.zip" SCRIPT_DIR = Path(__file__).resolve().parent + def ycruncher_folder_exists() -> bool: """Check if ycruncher has been downloaded or not""" return SCRIPT_DIR.joinpath(YCRUNCHER_FOLDER_NAME).is_dir() @@ -16,7 +16,7 @@ def ycruncher_folder_exists() -> bool: def download_ycruncher(): """Download and extract Y-Cruncher""" - download_url = "https://github.com/Mysticial/y-cruncher/releases/download/v0.8.5.9543/y-cruncher.v0.8.5.9543.zip" + download_url = "https://github.com/Mysticial/y-cruncher/releases/download/v0.8.6.9545/y-cruncher.v0.8.6.9545b.zip" destination = SCRIPT_DIR / YCRUNCHER_ZIP_NAME response = requests.get(download_url, allow_redirects=True, timeout=180) with open(destination, 'wb') as file: @@ -24,7 +24,7 @@ def download_ycruncher(): with ZipFile(destination, 'r') as zip_object: zip_object.extractall(path=SCRIPT_DIR) + def current_time_ms(): """Get current timestamp in milliseconds since epoch""" return int(time.time() * 1000) - \ No newline at end of file diff --git a/deprecated/F1_22/README.md b/zdeprecated/F1_22/README.md similarity index 100% rename from deprecated/F1_22/README.md rename to zdeprecated/F1_22/README.md diff --git a/deprecated/F1_22/f1.py b/zdeprecated/F1_22/f1.py similarity index 100% rename from deprecated/F1_22/f1.py rename to zdeprecated/F1_22/f1.py diff --git a/deprecated/F1_22/f1_22_utils.py b/zdeprecated/F1_22/f1_22_utils.py similarity index 100% rename from deprecated/F1_22/f1_22_utils.py rename to zdeprecated/F1_22/f1_22_utils.py diff --git a/deprecated/F1_22/manifest.yaml b/zdeprecated/F1_22/manifest.yaml similarity index 100% rename from deprecated/F1_22/manifest.yaml rename to zdeprecated/F1_22/manifest.yaml diff --git a/F1_23/README.md b/zdeprecated/F1_23/README.md similarity index 100% rename from F1_23/README.md rename to zdeprecated/F1_23/README.md diff --git a/F1_23/f1_23.py b/zdeprecated/F1_23/f1_23.py similarity index 100% rename from F1_23/f1_23.py rename to zdeprecated/F1_23/f1_23.py diff --git a/F1_23/f1_23_utils.py b/zdeprecated/F1_23/f1_23_utils.py similarity index 100% rename from F1_23/f1_23_utils.py rename to zdeprecated/F1_23/f1_23_utils.py diff --git a/F1_23/manifest.yaml b/zdeprecated/F1_23/manifest.yaml similarity index 100% rename from F1_23/manifest.yaml rename to zdeprecated/F1_23/manifest.yaml diff --git a/deprecated/README.md b/zdeprecated/README.md similarity index 100% rename from deprecated/README.md rename to zdeprecated/README.md diff --git a/deprecated/aida64gpgpu/README.md b/zdeprecated/aida64gpgpu/README.md similarity index 100% rename from deprecated/aida64gpgpu/README.md rename to zdeprecated/aida64gpgpu/README.md diff --git a/deprecated/aida64gpgpu/aida64_gpgpu_benchmark.png b/zdeprecated/aida64gpgpu/aida64_gpgpu_benchmark.png similarity index 100% rename from deprecated/aida64gpgpu/aida64_gpgpu_benchmark.png rename to zdeprecated/aida64gpgpu/aida64_gpgpu_benchmark.png diff --git a/deprecated/aida64gpgpu/aida64gpgpu.py b/zdeprecated/aida64gpgpu/aida64gpgpu.py similarity index 90% rename from deprecated/aida64gpgpu/aida64gpgpu.py rename to zdeprecated/aida64gpgpu/aida64gpgpu.py index ef5dd8f..f36d370 100644 --- a/deprecated/aida64gpgpu/aida64gpgpu.py +++ b/zdeprecated/aida64gpgpu/aida64gpgpu.py @@ -23,8 +23,8 @@ logging.getLogger('').addHandler(console) executable = os.path.join(INSTALL_DIR, EXECUTABLE) report_dest = os.path.join(log_dir, "report.xml") -argstr = f"/GGBENCH {report_dest}" -result = subprocess.run([executable, "/GGBENCH", report_dest], check=False) +ARGSTR = f"/GGBENCH {report_dest}" +result = subprocess.run([executable, ARGSTR], check=False) if result.returncode > 0: logging.error("Aida failed with exit code {result.returncode}") diff --git a/deprecated/aida64gpgpu/manifest.yaml b/zdeprecated/aida64gpgpu/manifest.yaml similarity index 100% rename from deprecated/aida64gpgpu/manifest.yaml rename to zdeprecated/aida64gpgpu/manifest.yaml diff --git a/deprecated/aida64gpgpu/sample_gpgpu.xml b/zdeprecated/aida64gpgpu/sample_gpgpu.xml similarity index 100% rename from deprecated/aida64gpgpu/sample_gpgpu.xml rename to zdeprecated/aida64gpgpu/sample_gpgpu.xml diff --git a/zz_non_game_harness_template/harness.py b/zz_non_game_harness_template/harness.py new file mode 100644 index 0000000..eaa6f9f --- /dev/null +++ b/zz_non_game_harness_template/harness.py @@ -0,0 +1 @@ +# This is a non-game harness template