Add pugetbench harness (#51)

pugetbench/README.md

@@ -0,0 +1,19 @@
+# PugetBench for Creators
+
+This is a test harness to run the test suite [PugetBench for Creators](https://www.pugetsystems.com/pugetbench/creators/) which contains a test suite for Adobe Photoshop and Premiere Pro.
+
+## Prerequisites
+
+- Python 3.10+
+- PugetBench for Creators installed and activated for CLI features.
+- Adobe Creative Cloud installed with Photoshop and Premiere Pro
+
+## Options
+
+- `--app` : Specifies to run photoshop or premierepro
+
+## Output
+
+report.json
+- `test`: The application that was tested.
+- `score`: The score extraced from PugetBench.

pugetbench/manifest.yaml

@@ -0,0 +1,15 @@
+friendly_name: "PugetBench"
+executable: "pugetbench.py"
+process_name: "PugetBench for Creators.exe"
+disable_presentmon: true
+output_dir: run
+options:
+  - name: app_version
+    type: input
+    tooltip: Version of the selected test
+  - name: app
+    type: select
+    values:
+      - premierepro
+      - photoshop
+    tooltip: Select which test to run

pugetbench/pugetbench.py

@@ -0,0 +1,112 @@
+"""pugetbench for creators test script"""
+import logging
+import os.path
+from pathlib import Path
+import shutil
+import sys
+from argparse import ArgumentParser
+import time
+from subprocess import Popen
+from utils import find_latest_log, find_score_in_log, get_photoshop_version, get_premierepro_version
+
+sys.path.insert(1, os.path.join(sys.path[0], ".."))
+from harness_utils.process import terminate_processes
+from harness_utils.output import (
+    seconds_to_milliseconds,
+    setup_log_directory,
+    write_report_json,
+    DEFAULT_LOGGING_FORMAT
+)
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+log_dir = os.path.join(script_dir, "run")
+setup_log_directory(log_dir)
+logging.basicConfig(filename=f'{log_dir}/harness.log',
+                    format=DEFAULT_LOGGING_FORMAT,
+                    datefmt='%m-%d %H:%M',
+                    level=logging.DEBUG)
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger('').addHandler(console)
+
+EXECUTABLE_NAME = "PugetBench for Creators.exe"
+
+def run_benchmark(application: str, app_version: str) -> Popen:
+    """run benchmark"""
+    start_time = time.time()
+    executable_path =  Path(f"C:\\Program Files\\PugetBench for Creators\\{EXECUTABLE_NAME}")
+    command_args = ["--run_count" , "1", "--rerun_count", "1", "--benchmark_version", "1.0.0", "--preset", "Standard", "--app_version", f"{app_version}"]
+    photoshop_args = command_args + ["--app", "photoshop"]
+    premiere_args = command_args + ["--app", "premierepro"]
+    command = None
+    if application == "premierepro":
+        command = [executable_path] +  premiere_args
+    elif application == "photoshop":
+        command =[executable_path] + photoshop_args
+
+    with Popen(command) as process:
+        exit_code = process.wait()
+        end_time = time.time()
+        return start_time, end_time, exit_code
+
+def main():
+    """main"""
+    start_time = time.time()
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--app", dest="app", help="Application name to test", required=True
+    )
+    parser.add_argument(
+        "--app_version", dest="app_version", help="Application version to test", required=False
+    )
+    args = parser.parse_args()
+    apps = [
+        "premierepro",
+        "photoshop"
+    ]
+
+    if args.app is None or args.app not in apps:
+        logging.info("unrecognized option for program")
+        sys.exit(1)
+
+    version = args.app_version
+    score = 0
+    test = ""
+    if args.app == "premierepro":
+        test = "PugetBench Adobe Premiere Pro"
+        if version is None:
+            version = get_premierepro_version()
+    elif args.app == "photoshop":
+        test = "PugentBench Adobe Photoshop"
+        if version is None:
+            version = get_photoshop_version()
+
+    try:
+        start_time, end_time, exit_code = run_benchmark(args.app, version)
+
+        if exit_code > 0:
+            logging.error("Test failed!")
+            sys.exit(exit_code)
+        log_file = find_latest_log()
+        score = find_score_in_log(log_file)
+        destination = Path(script_dir) / "run" / os.path.split(log_file)[1]
+        shutil.copy(log_file, destination)
+
+        report = {
+            "start_time": seconds_to_milliseconds(start_time),
+            "end_time": seconds_to_milliseconds(end_time),
+            "test": test,
+            "score": score
+        }
+
+        write_report_json(log_dir, "report.json", report)
+    except Exception as e:
+        logging.error("Something went wrong running the benchmark!")
+        logging.exception(e)
+        terminate_processes(EXECUTABLE_NAME)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pugetbench/test_samples/photoshop_log.csv

@@ -0,0 +1,44 @@
+Benchmark Name,PugetBench for Photoshop
+Benchmark Version,1.0.0
+Application Name,Photoshop
+Application Version,25.6
+
+System Specs:
+
+System,ASUS,System Product Name
+OS,Microsoft Corporation,Microsoft Windows 11 Pro,22631
+CPU,GenuineIntel,12th Gen Intel Core i5-12600K,10
+Motherboard,ASUSTeK COMPUTER INC.,ProArt Z690-CREATOR WIFI,1720
+Memory,G Skill Intl,16,GB,5200,MHz
+Memory,G Skill Intl,16,GB,5200,MHz
+GPU,"Parsec Cloud, Inc.",Parsec Virtual Display Adapter,0.41.0.0
+GPU,NVIDIA,NVIDIA GeForce RTX 4070,31.0.15.5176
+Engine
+
+Benchmark Results:
+
+Test,Setting,Result,Units
+Overall Score (Standard),,7097,
+General Score (Standard),General,73,
+Filter Score (Standard),Filter,69,
+File Open - RAW,General,1.53,seconds
+Resize to 150MP - Preserve Details,General,1.98,seconds
+Resize to 150MP - Bicubic Smooth,General,0.63,seconds
+Rotate,General,1.08,seconds
+Select Subject,General,3.18,seconds
+Select and Mask,General,3.78,seconds
+Convert to Smart Object,General,1.79,seconds
+Paint Bucket,General,1.15,seconds
+Smudge Tool,General,12.41,seconds
+Adaptive Wide Angle,Filter,120.3,seconds
+Camera Raw,Filter,3.25,seconds
+Lens Correction,Filter,26.78,seconds
+Content Aware Fill,Filter,8.84,seconds
+Reduce Noise,Filter,11.66,seconds
+Smart Sharpen,Filter,1.31,seconds
+Iris Blur,Filter,7.35,seconds
+Field Blur,Filter,9.38,seconds
+File Save - JPG,General,1.6,seconds
+File Save - PNG,General,61.37,seconds
+File Save - PSD,General,25.77,seconds
+File Open - PSD,General,5.4,seconds

pugetbench/utils.py

@@ -0,0 +1,61 @@
+"""utils file for pugetbench harness"""
+import re
+import os
+from pathlib import Path
+import win32api
+
+
+def find_latest_log():
+    """find latest log from pugetbench"""
+    appdata_path = os.getenv('LOCALAPPDATA')
+    puget_lunch_dir = Path(appdata_path) / "com.puget.benchmark" / "csv"
+    files = [os.path.join(puget_lunch_dir, file) for file in os.listdir(
+        puget_lunch_dir) if os.path.isfile(os.path.join(puget_lunch_dir, file))]
+    latest_file = max(files, key=os.path.getmtime)
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    return Path(script_dir) / latest_file
+
+
+def find_score_in_log(log_path):
+    """find score in pugentbench log file"""
+    with open(log_path, 'r', encoding="utf-8") as file:
+        for line in file:
+            score = is_score_line(line)
+            if score is not None:
+                return score
+    return None
+
+
+def is_score_line(line):
+    """check if string is a score using regex"""
+    regex_pattern = r"^Overall Score.+,+(\d+),+"
+    match = re.search(regex_pattern, line)
+    if match and len(match.groups()) > 0:
+        return match.group(1)
+    return None
+
+
+def get_photoshop_version() -> str:
+    """get current photoshop version string"""
+    path = "C:\\Program Files\\Adobe\\Adobe Photoshop 2024\\Photoshop.exe"
+    try:
+        lang, codepage = win32api.GetFileVersionInfo(
+            path, "\\VarFileInfo\\Translation")[0]
+        str_info_path = f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion"
+        return win32api.GetFileVersionInfo(path, str_info_path)
+    except Exception as e:
+        print(e)
+    return None
+
+
+def get_premierepro_version() -> str:
+    """get current premiere pro version string"""
+    path = "C:\\Program Files\\Adobe\\Adobe Premiere Pro 2024\\Adobe Premiere Pro.exe"
+    try:
+        lang, codepage = win32api.GetFileVersionInfo(
+            path, "\\VarFileInfo\\Translation")[0]
+        str_info_path = f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion"
+        return win32api.GetFileVersionInfo(path, str_info_path)
+    except Exception as e:
+        print(e)
+    return None