harness updates

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 
 Changes are grouped by the date they are merged to the main branch of the repository and are ordered from newest to oldest. Dates use the ISO 8601 extended calendar date format, i.e. YYYY-MM-DD.
 
+## 2024-10-25
+
+- Update test name in y-cruncher harness to not be the tuning used.
+- Update PugetBench harness to use the latest benchmark version available.
+
 ## 2024-10-18
 
 - Updated Cyberpunk 2077 harness to take screenshots of settings and results.

pugetbench/pugetbench.py

@@ -7,7 +7,7 @@ import sys
 from argparse import ArgumentParser
 import time
 from subprocess import Popen
-from utils import find_latest_log, find_score_in_log, get_photoshop_version, get_premierepro_version
+from utils import find_latest_log, find_score_in_log, get_photoshop_version, get_premierepro_version, get_latest_benchmark_by_version
 
 sys.path.insert(1, os.path.join(sys.path[0], ".."))
 from harness_utils.process import terminate_processes
@@ -35,8 +35,10 @@ EXECUTABLE_NAME = "PugetBench for Creators.exe"
 def run_benchmark(application: str, app_version: str) -> Popen:
     """run benchmark"""
     start_time = time.time()
+    benchmark_version = get_latest_benchmark_by_version(application)
+
     executable_path =  Path(f"C:\\Program Files\\PugetBench for Creators\\{EXECUTABLE_NAME}")
-    command_args = ["--run_count" , "1", "--rerun_count", "1", "--benchmark_version", "1.0.0", "--preset", "Standard", "--app_version", f"{app_version}"]
+    command_args = ["--run_count" , "1", "--rerun_count", "1", "--benchmark_version", benchmark_version, "--preset", "Standard", "--app_version", f"{app_version}"]
     photoshop_args = command_args + ["--app", "photoshop"]
     premiere_args = command_args + ["--app", "premierepro"]
     command = None
@@ -45,6 +47,8 @@ def run_benchmark(application: str, app_version: str) -> Popen:
     elif application == "photoshop":
         command =[executable_path] + photoshop_args
 
+    logging.info(command)
+
     with Popen(command) as process:
         exit_code = process.wait()
         end_time = time.time()

pugetbench/utils.py

@@ -5,6 +5,26 @@ from pathlib import Path
 import win32api
 
 
+def get_latest_benchmark_by_version(benchmark_name: str):
+    """get latest benchmark version for photoshop or premiere"""
+    valid_names = ['photoshop', 'premierepro']
+    if benchmark_name not in valid_names:
+        raise ValueError("invalid benchmark name")
+
+    benchmark_json_dir = Path().home() / "AppData/Local/com.puget.benchmark/benchmarks"
+    if not benchmark_json_dir.exists():
+        raise ValueError("could not find benchmark directory in appdata")
+
+    benchmark_files = [f for f in os.listdir(benchmark_json_dir) if benchmark_name in f]
+    # sort assuming the filename format is still premierepro-benchmark-X.X.X.json
+    # convert version to int and order by size
+    sorted_files = sorted(benchmark_files, key=lambda x: tuple(map(int,x.split("-")[-1].split(".")[:-1])))
+    latest_version = sorted_files[len(sorted_files)-1]
+    latest_version = latest_version.split("-")[2]
+    version, _ = latest_version.rsplit(".", 1)
+    return version
+
+
 def find_latest_log():
     """find latest log from pugetbench"""
     appdata_path = os.getenv('LOCALAPPDATA')

ycruncher/ycruncher.py

@@ -83,7 +83,7 @@ def main():
         "end_time": end_time,
         "score": avg_score,
         "unit": "seconds",
-        "test": tunings[0]
+        "test": "y-cruncher"
     }
 
     write_report_json(LOG_DIR, "report.json", report)