github/markbench-tests
1
1
Fork 0
mirror of https://github.com/LTTLabsOSS/markbench-tests.git synced 2026-01-08 21:48:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

procyon CV from Jon and first attempt at stable diffusion, needs to be tested on benches

Browse Source
This commit is contained in:
Arty Blue
2024-12-17 09:26:32 -08:00
parent d45c20d7ec
commit 2737c7f767
23 changed files with 857 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
procyon_ai/README.md Normal file
View File

@@ -0,0 +1,21 @@
# 3DMark
Runs one of the 3DMark benchmark scenes and reads the Performance Graphics Score result from the output.
## Prerequisites
- Python 3.10+
- 3DMark Professional Edition installed in default location and activated.
- Desired benchmarks are downloaded,.
## Options
- `--benchmark` Specifies the benchmark to run.
## Output
report.json
- `test`: The name of the selected benchmark
- `score`: 3DMark gpu score
- `start_time`: number representing a timestamp of the test's start time in milliseconds
- `end_time`: number representing a timestamp of the test's end time in milliseconds

28
procyon_ai/config/ai_computer_vision_openvino_cpu.def Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AIOpenVinoBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>CPU</value><!--Options: CPU, GPU, GPU.0, GPU.1, NPU. Run "ProcyonCmd.exe list-openvino-devices" to list the options. Check "ProcyonCmd.exe -h" for the correct parameter syntax. -->
</setting>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
</settings>
</benchmark>

28
procyon_ai/config/ai_computer_vision_openvino_gpu.def Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AIOpenVinoBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>GPU</value><!--Options: CPU, GPU, GPU.0, GPU.1, NPU. Run "ProcyonCmd.exe list-openvino-devices" to list the options. Check "ProcyonCmd.exe -h" for the correct parameter syntax. -->
</setting>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
</settings>
</benchmark>

28
procyon_ai/config/ai_computer_vision_openvino_npu.def Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AIOpenVinoBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>NPU</value><!--Options: CPU, GPU, GPU.0, GPU.1, NPU. Run "ProcyonCmd.exe list-openvino-devices" to list the options. Check "ProcyonCmd.exe -h" for the correct parameter syntax. -->
</setting>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
</settings>
</benchmark>

28
procyon_ai/config/ai_computer_vision_snpe.def Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AISNPEBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>HTP</value>
</setting>
<setting>
<name>ai_inference_precision</name>
<value>integer</value>
</setting>
</settings>
</benchmark>

28
procyon_ai/config/ai_computer_vision_tensorrt.def Normal file
View File

@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AITensorRTBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
<setting>
<name>ai_device_id</name>
<value>cuda:0</value> <!-- Get options from Procyon CLI: "ProcyonCmd.exe list-cuda-devices". Check "ProcyonCmd.exe -h" for correct syntax -->
</setting>
</settings>
</benchmark>

32
procyon_ai/config/ai_computer_vision_winml_cpu.def Normal file
View File

@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AIWinMLBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>CPU</value><!--Options: CPU, GPU -->
</setting>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
<setting>
<name>ai_device_id</name> <!-- Use our CLI e.g. "ProcyonCmd.exe list-winml-devices" to full options. Check "ProcyonCmd.exe -h" for the parameter syntax. -->
<value></value>
</setting>
</settings>
</benchmark>

32
procyon_ai/config/ai_computer_vision_winml_gpu.def Normal file
View File

@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="utf-8"?>
<benchmark>
<test_info>
<benchmark_tests>
<benchmark_test name="AIWinMLBenchmark" test_run_type="EXPLICIT" version="1.0"/>
</benchmark_tests>
</test_info>
<application_info>
<selected_workloads>
<selected_workload name="AIMobileNetV3Default"/>
<selected_workload name="AIInceptionV4Default"/>
<selected_workload name="AIResNet50Default"/>
<selected_workload name="AIDeepLabV3Default"/>
<selected_workload name="AIYOLOV3Default"/>
<selected_workload name="AIESRGANDefault"/>
</selected_workloads>
</application_info>
<settings>
<setting>
<name>ai_device_type</name>
<value>GPU</value><!--Options: CPU, GPU -->
</setting>
<setting>
<name>ai_inference_precision</name>
<value>float32</value><!--Options: float32, float16, integer -->
</setting>
<setting>
<name>ai_device_id</name> <!-- Use our CLI e.g. "ProcyonCmd.exe list-winml-devices" to full options. Check "ProcyonCmd.exe -h" for the parameter syntax. -->
<value></value>
</setting>
</settings>
</benchmark>

17
procyon_ai/manifest.yaml Normal file
View File

@@ -0,0 +1,17 @@
friendly_name: "Procyon AI CV"
executable: "ulprocai.py"
process_name: "ProcyonCmd.exe"
disable_presentmon: true
output_dir: "run"
options:
- name: engine
type: select
values:
- "AMD_CPU"
- "AMD_GPU"
- "Intel_CPU"
- "Intel_GPU"
- "Intel_NPU"
- "NVIDIA_GPU"
- "Qualcomm_HTP"
tooltip: Select which configuration to run for Procyon

154
procyon_ai/ulprocai.py Normal file
View File

@@ -0,0 +1,154 @@
"""3DMark test script"""
from argparse import ArgumentParser
import logging
from pathlib import Path
import subprocess
import sys
import time
import psutil
from utils import find_score_in_xml, is_process_running, get_install_path
PARENT_DIR = str(Path(sys.path[0], ".."))
sys.path.append(PARENT_DIR)
from harness_utils.output import (
DEFAULT_DATE_FORMAT,
DEFAULT_LOGGING_FORMAT,
seconds_to_milliseconds,
setup_log_directory,
write_report_json
)
#####
### Globals
#####
SCRIPT_DIR = Path(__file__).resolve().parent
LOG_DIR = SCRIPT_DIR / "run"
DIR_PROCYON = Path(get_install_path())
EXECUTABLE = "ProcyonCmd.exe"
ABS_EXECUTABLE_PATH = DIR_PROCYON / EXECUTABLE
CONFIG_DIR = SCRIPT_DIR / "config"
BENCHMARK_CONFIG = {
"AMD_CPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_cpu.def\"",
"process_name": "WinML.exe",
"test_name": "WinML CPU (FLOAT32)"
},
"AMD_GPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"",
"process_name": "WinML.exe",
"test_name": "WinML GPU (FLOAT32)"
},
"Intel_CPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_cpu.def\"",
"process_name": "OpenVino.exe",
"test_name": "Intel OpenVINO CPU (FLOAT32)"
},
"Intel_GPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"",
"process_name": "OpenVino.exe",
"test_name": "Intel OpenVINO GPU (FLOAT32)"
},
"Intel_NPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_npu.def\"",
"process_name": "OpenVino.exe",
"test_name": "Intel OpenVINO NPU (FLOAT32)"
},
"NVIDIA_GPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_tensorrt.def\"",
"process_name": "TensorRT.exe",
"test_name": "NVIDIA TensorRT (FLOAT32)"
},
"Qualcomm_HTP": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_snpe.def\"",
"process_name": "SNPE.exe",
"test_name": "Qualcomm SNPE (INTEGER)"
},
}
RESULTS_FILENAME = "result.xml"
REPORT_PATH = LOG_DIR / RESULTS_FILENAME
def setup_logging():
"""setup logging"""
setup_log_directory(LOG_DIR)
logging.basicConfig(filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG)
console = logging.StreamHandler()
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
def get_arguments():
"""get arguments"""
parser = ArgumentParser()
parser.add_argument(
"--engine", dest="engine", help="Engine test type", required=True, choices=BENCHMARK_CONFIG.keys())
argies = parser.parse_args()
return argies
def create_procyon_command(test_option):
"""create command string"""
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\"'
command = command.rstrip()
return command
def run_benchmark(process_name, command_to_run):
"""run the benchmark"""
with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
logging.info("Procyon AI Computer Vision benchmark has started.")
start_time = time.time()
while True:
now = time.time()
elapsed = now - start_time
if elapsed >= 60: #seconds
raise ValueError("BenchMark subprocess did not start in time")
process = is_process_running(process_name)
if process is not None:
process.nice(psutil.HIGH_PRIORITY_CLASS)
break
time.sleep(0.2)
_, _ = proc.communicate() # blocks until 3dmark exits
return proc
try:
setup_logging()
args = get_arguments()
option = BENCHMARK_CONFIG[args.engine]["config"]
cmd = create_procyon_command(option)
logging.info('Starting benchmark!')
logging.info(cmd)
start_time = time.time()
pr = run_benchmark(BENCHMARK_CONFIG[args.engine]["process_name"], cmd)
if pr.returncode > 0:
logging.error("Procyon exited with return code %d", pr.returncode)
sys.exit(pr.returncode)
score = find_score_in_xml()
if score is None:
logging.error("Could not find overall score!")
sys.exit(1)
end_time = time.time()
elapsed_test_time = round(end_time - start_time, 2)
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
logging.info("Score was %s", score)
report = {
"test": BENCHMARK_CONFIG[args.engine]["test_name"],
"unit": "score",
"score": score,
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
}
write_report_json(LOG_DIR, "report.json", report)
except Exception as e:
logging.error("Something went wrong running the benchmark!")
logging.exception(e)
sys.exit(1)

36
procyon_ai/utils.py Normal file
View File

@@ -0,0 +1,36 @@
"""3dmark test utils"""
from pathlib import Path
import psutil
import xml.etree.ElementTree as ET
import winreg
import re
SCRIPT_DIR = Path(__file__).resolve().parent
LOG_DIR = SCRIPT_DIR / "run"
def is_process_running(process_name):
"""check if given process is running"""
for process in psutil.process_iter(['pid', 'name']):
if process.info['name'] == process_name:
return process
return None
def find_score_in_xml():
"""Reads score from local game log"""
score_pattern = re.compile(r"<AIOverallScore>(\d+)")
cfg = f"{LOG_DIR}\\result.xml"
score_value = 0
with open(cfg, encoding="utf-8") as file:
lines = file.readlines()
for line in lines:
score_match = score_pattern.search(line)
if score_match is not None:
score_value = score_match.group(1)
return score_value
def get_install_path() -> str:
"""Gets the path to the Steam installation directory from the SteamPath registry key"""
reg_path = r"Software\UL\Procyon"
reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, reg_path, 0, winreg.KEY_READ)
value, _ = winreg.QueryValueEx(reg_key, "InstallDir")
return value