GravityMark harness (#77)

* add readme and manifest yaml

* initial harness

* update scripts

* update manifest

* test disable duplicate code

* add comma

* read score from log file

cinebench_2024/cinebench.py

@@ -53,7 +53,7 @@ try:
     setup_start_time = time.time()
 
     with subprocess.Popen(
-        [CINEBENCH_PATH, test_option, DURATION_OPTION,],
+        [CINEBENCH_PATH, test_option, DURATION_OPTION],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         bufsize=1,
@@ -71,7 +71,7 @@ try:
         out, _ = proc.communicate()
 
         if proc.returncode > 0:
-            logging.error("Cinebench did exited with return code %d", proc.returncode)
+            logging.error("Cinebench exited with return code %d", proc.returncode)
             sys.exit(proc.returncode)
 
         score = get_score(out)

cspell.json

@@ -1,29 +1,32 @@
 {
-  "ignorePaths": [".*/"],
+  "ignorePaths": [
+    ".*/"
+  ],
   "allowCompoundWords": true,
   "words": [
-    "flac",
-    "vsix",
-    "csgo",
-    "pycache",
-    "OPTIX",
-    "CUDA",
-    "psutil",
-    "HIWORD",
-    "Kikis",
-    "Jcraft",
-    "Kombustor",
-    "Unigine",
-    "directx",
-    "opengl",
     "argies",
-    "turbopolsa",
-    "RLCS",
+    "csgo",
+    "CUDA",
+    "directx",
     "Dota",
+    "flac",
+    "HIWORD",
+    "Jcraft",
     "Keras",
-    "twwh"
+    "Kikis",
+    "Kombustor",
+    "opengl",
+    "opengles",
+    "OPTIX",
+    "psutil",
+    "pycache",
+    "RLCS",
+    "turbopolsa",
+    "twwh",
+    "Unigine",
+    "vsix"
   ],
   "ignoreRegExpList": [
     "import .*"
   ]
-}
+}

gravitymark/README.md

@@ -0,0 +1,19 @@
+# GravityMark
+
+Runs GravityMark in benchmark mode using the selected graphics API. Stores the results image file.
+
+## Prerequisites
+
+- Python 3.10+
+- GravityMark installed in default location. [Download GravityMark](https://gravitymark.tellusim.com/)
+
+## Options
+
+- `-a` or `--api`: Specifies which graphics API to run for GravityMark to use for the benchmark.
+
+Note that while GravityMark supports OpenGLES and Metal, they are not available on Windows and as such, this test script does not provide the option to use them.
+
+## Output
+
+report.json
+- `api`: The graphics API used for the test

gravitymark/gravitymark.py

@@ -0,0 +1,68 @@
+"""GravityMark test script"""
+import logging
+import getpass
+import subprocess
+import sys
+from pathlib import Path
+from gravitymark_utils import get_args, get_score, create_gravitymark_command
+
+PARENT_DIR = str(Path(sys.path[0], ".."))
+sys.path.append(PARENT_DIR)
+
+from harness_utils.output import (
+    DEFAULT_DATE_FORMAT,
+    DEFAULT_LOGGING_FORMAT,
+    write_report_json
+)
+
+GRAVITYMARK_PATH = Path("C:/", "Program Files", "GravityMark", "bin")
+GRAVITYMARK_EXE = GRAVITYMARK_PATH / "GravityMark.exe"
+
+args = get_args()
+api = f"-{args.api}"
+
+script_dir = Path(__file__).resolve().parent
+log_dir = script_dir / "run"
+log_dir.mkdir(exist_ok=True)
+log_file = log_dir / "harness.log"
+logging.basicConfig(
+    filename=log_file,
+    format=DEFAULT_LOGGING_FORMAT,
+    datefmt=DEFAULT_DATE_FORMAT,
+    level=logging.DEBUG
+)
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger("").addHandler(console)
+
+gravitymark_log_path = Path("C:/Users", getpass.getuser(), ".GravityMark", "GravityMark.log")
+image_path = log_dir / "result.png"
+command = create_gravitymark_command(GRAVITYMARK_EXE, api, image_path)
+
+try:
+    logging.info('Starting benchmark!')
+    # Remove existing log file so we have a fresh file with only the logs of the current run
+    gravitymark_log_path.unlink(missing_ok=True)
+    result = subprocess.run(command, check=True, cwd=GRAVITYMARK_PATH)
+
+    if result.returncode > 0:
+        logging.error("GravityMark exited with return code %d", result.returncode)
+        sys.exit(1)
+
+    score = get_score(gravitymark_log_path)
+
+    if score is None:
+        logging.error("Score not found")
+        sys.exit(1)
+
+    report = {
+        "api": args.api,
+        "score": score
+    }
+
+    write_report_json(log_dir, "report.json", report)
+except Exception as e:
+    logging.error("Something went wrong running the benchmark!")
+    logging.exception(e)
+    sys.exit(1)

gravitymark/gravitymark_utils.py

@@ -0,0 +1,60 @@
+"""Utility functions for GravityMark test script"""
+from argparse import ArgumentParser, Namespace
+import re
+from pathlib import Path
+
+API_OPTIONS = [
+    "vulkan",
+    "opengl",
+    # "opengles",
+    "direct3d12",
+    "direct3d11",
+    # "metal"
+]
+
+CLI_OPTIONS = {
+    # "-image": str(IMAGE_PATH),
+    "-fullscreen": "1",
+    "-fps": "1",
+    "-info": "1",
+    "-sensors": "1",
+    "-benchmark": "1",
+    "-close": "1",
+    "-status": "1"
+}
+
+
+def get_args() -> Namespace:
+    """Get command line arguments for test script"""
+    parser = ArgumentParser()
+    parser.add_argument(
+        "-a", "--api", dest="api", required=True, choices=API_OPTIONS, help="GravityMark graphics API"
+    )
+    return parser.parse_args()
+
+
+def get_score(log_path: Path) -> str | None:
+    """Parses score value from GravityMark log file"""
+    score_pattern = re.compile(r"Score: (\d+)")
+    try:
+        with log_path.open("r", encoding="utf-8") as file:
+            for line in file.readlines():
+                match = score_pattern.search(line)
+                if match:
+                    return match.group(1)
+    except OSError:
+        return None
+
+    return None
+
+
+def create_gravitymark_command(gravitymark_path: Path, api: str, image_path: Path):
+    """Create the command array for subprocess to run GravityMark"""
+    options = [gravitymark_path, api]
+    for arg, value in CLI_OPTIONS.items():
+        options.append(arg)
+        options.append(value)
+    options.append("-image")
+    options.append(str(image_path))
+
+    return options

gravitymark/manifest.yaml

@@ -0,0 +1,16 @@
+friendly_name: "GravityMark"
+executable: "gravitymark.py"
+process_name: "GravityMark.exe"
+disable_presentmon: true
+output_dir: run
+options:
+  - name: api
+    type: select
+    values:
+      - vulkan
+      - opengl
+      # - opengles # Linux only
+      - direct3d12
+      - direct3d11
+      # - metal # macOS only
+    tooltip: Select which Graphics API to use

pylintrc

@@ -8,4 +8,5 @@ disable=
     too-many-statements,
     broad-exception-raised,
     fixme,
-    c-extension-no-member
+    c-extension-no-member,
+    duplicate-code