github/markbench-tests
1
1
Fork 0
mirror of https://github.com/LTTLabsOSS/markbench-tests.git synced 2026-01-08 21:48:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

added results xml to the artifacts

Browse Source
This commit is contained in:
James
2025-12-22 13:45:51 -08:00
parent 6067d6e5cf
commit 0211dcbb8a
3 changed files with 238 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
procyon_ai/ulprocai.py
View File

@@ -1,24 +1,26 @@
"""UL Procyon Computer Vision test script"""
# pylint: disable=no-name-in-module
from argparse import ArgumentParser
import logging
from pathlib import Path
import subprocess
import sys
import time
from argparse import ArgumentParser
from pathlib import Path
import psutil
from utils import (
find_score_in_xml,
is_process_running,
get_install_path,
find_procyon_version,
find_test_version
find_score_in_xml,
find_test_version,
get_install_path,
is_process_running,
)
PARENT_DIR = str(Path(sys.path[0], ".."))
sys.path.append(PARENT_DIR)
from harness_utils.artifacts import ArtifactManager, ArtifactType
from harness_utils.output import (
DEFAULT_DATE_FORMAT,
DEFAULT_LOGGING_FORMAT,
@@ -27,11 +29,12 @@ from harness_utils.output import (
write_report_json,
)
from harness_utils.procyoncmd import (
get_winml_devices,
get_cuda_devices,
get_openvino_devices,
get_openvino_gpu,
get_cuda_devices,
get_winml_devices,
)
#####
# Globals
#####
@@ -48,104 +51,114 @@ CUDA_DEVICES = get_cuda_devices(ABS_EXECUTABLE_PATH)
CONFIG_DIR = SCRIPT_DIR / "config"
BENCHMARK_CONFIG = {
"AMD_CPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_cpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_winml_cpu.def"',
"process_name": "WinML.exe",
"device_name": "CPU",
# TODO: Find a good way to report the CPU name here.
"device_id": "CPU",
"test_name": "cpu_float32",
"api": "winml"
"api": "winml",
},
"AMD_GPU0": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def"',
"process_name": "WinML.exe",
"device_name": list(WINML_DEVICES.keys())[0],
"device_id": list(WINML_DEVICES.values())[0],
"test_name": "gpu_float32",
"api": "winml"
"api": "winml",
},
"AMD_GPU1": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_winml_gpu.def"',
"process_name": "WinML.exe",
"device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0],
"device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0],
"device_name": list(WINML_DEVICES.keys())[1]
if len(list(WINML_DEVICES.keys())) > 1
else list(WINML_DEVICES.keys())[0],
"device_id": list(WINML_DEVICES.values())[1]
if len(list(WINML_DEVICES.values())) > 1
else list(WINML_DEVICES.values())[0],
"test_name": "gpu_float32",
"api": "winml"
"api": "winml",
},
"Intel_CPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_cpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_openvino_cpu.def"',
"process_name": "OpenVino.exe",
"device_id": "CPU",
"device_name": OPENVINO_DEVICES["CPU"],
"test_name": "cpu_float32",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU0": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def"',
"process_name": "OpenVino.exe",
"device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"),
"test_name": "gpu_float32",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU1": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_openvino_gpu.def"',
"process_name": "OpenVino.exe",
"device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"),
"test_name": "gpu_float32",
"api": "openvino"
"api": "openvino",
},
"Intel_NPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_openvino_npu.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_openvino_npu.def"',
"process_name": "OpenVino.exe",
"device_id": "NPU",
"device_name": OPENVINO_DEVICES.get("NPU", "None"),
"test_name": "npu_float32",
"api": "openvino"
"api": "openvino",
},
"NVIDIA_GPU": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_tensorrt.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_tensorrt.def"',
"device_id": "cuda:0",
"device_name": CUDA_DEVICES.get("cuda:0"),
"process_name": "TensorRT.exe",
"test_name": "gpu_float32",
"api": "tensorrt"
"api": "tensorrt",
},
"Qualcomm_HTP": {
"config": f"\"{CONFIG_DIR}\\ai_computer_vision_snpe.def\"",
"config": f'"{CONFIG_DIR}\\ai_computer_vision_snpe.def"',
"device_id": "CPU",
"device_name": "CPU",
"process_name": "SNPE.exe",
"test_name": "htp_integer",
"api": "snpe"
"api": "snpe",
},
}
RESULTS_FILENAME = "result.xml"
REPORT_PATH = LOG_DIR / RESULTS_FILENAME
RESULTS_XML_PATH = LOG_DIR / RESULTS_FILENAME
def setup_logging():
"""setup logging"""
setup_log_directory(str(LOG_DIR))
logging.basicConfig(filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG)
logging.basicConfig(
filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG,
)
console = logging.StreamHandler()
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logging.getLogger("").addHandler(console)
def get_arguments():
"""get arguments"""
parser = ArgumentParser()
parser.add_argument(
"--engine", dest="engine", help="Engine test type", required=True,
choices=BENCHMARK_CONFIG.keys())
"--engine",
dest="engine",
help="Engine test type",
required=True,
choices=BENCHMARK_CONFIG.keys(),
)
argies = parser.parse_args()
return argies
@@ -154,23 +167,28 @@ def create_procyon_command(test_option, process_name, device_id):
"""create command string"""
command = str()
if device_id == 'CPU':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\"'
if device_id == "CPU":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}"'
else:
match process_name:
case 'WinML.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-winml-device {device_id}'
case 'OpenVino.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-openvino-device {device_id}'
case 'TensorRT.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-cuda-device {device_id}'
case "WinML.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-winml-device {device_id}'
case "OpenVino.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-openvino-device {device_id}'
case "TensorRT.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-cuda-device {device_id}'
command = command.rstrip()
return command
def run_benchmark(process_name, command_to_run):
"""run the benchmark"""
with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
with subprocess.Popen(
command_to_run,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
logging.info("Procyon AI Computer Vision benchmark has started.")
while True:
now = time.time()
@@ -191,13 +209,14 @@ try:
logging.info("Detected Windows ML Devices: %s", str(WINML_DEVICES))
logging.info("Detected OpenVino Devices: %s", str(OPENVINO_DEVICES))
logging.info("Detected CUDA Devices: %s", (CUDA_DEVICES))
am = ArtifactManager(LOG_DIR)
args = get_arguments()
option = BENCHMARK_CONFIG[args.engine]["config"]
proc_name = BENCHMARK_CONFIG[args.engine]["process_name"]
dev_id = BENCHMARK_CONFIG[args.engine]["device_id"]
cmd = create_procyon_command(option, proc_name, dev_id)
logging.info('Starting benchmark!')
logging.info("Starting benchmark!")
logging.info(cmd)
start_time = time.time()
pr = run_benchmark(BENCHMARK_CONFIG[args.engine]["process_name"], cmd)
@@ -210,7 +229,8 @@ try:
if score is None:
logging.error("Could not find overall score!")
sys.exit(1)
am.copy_file(RESULTS_XML_PATH, ArtifactType.RESULTS_TEXT, "results xml file")
end_time = time.time()
elapsed_test_time = round(end_time - start_time, 2)
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
@@ -226,9 +246,9 @@ try:
"device_name": BENCHMARK_CONFIG[args.engine]["device_name"],
"procyon_version": find_procyon_version(),
"unit": "score",
"score": score
"score": score,
}
am.create_manifest()
write_report_json(str(LOG_DIR), "report.json", report)
except Exception as e:
logging.error("Something went wrong running the benchmark!")

154
procyon_ai_img_gen/ulprocai_img_gen.py
View File

@@ -1,29 +1,38 @@
"""UL Procyon AI Image Generation test script"""
# pylint: disable=no-name-in-module
from argparse import ArgumentParser
import logging
from pathlib import Path
import subprocess
import sys
import time
from argparse import ArgumentParser
from pathlib import Path
import psutil
from utils import find_score_in_xml, is_process_running, get_install_path, find_procyon_version, find_test_version
from utils import (
find_procyon_version,
find_score_in_xml,
find_test_version,
get_install_path,
is_process_running,
)
PARENT_DIR = str(Path(sys.path[0], ".."))
sys.path.append(PARENT_DIR)
from harness_utils.procyoncmd import (
get_winml_devices,
get_openvino_devices,
get_openvino_gpu,
get_cuda_devices
)
from harness_utils.artifacts import ArtifactManager, ArtifactType
from harness_utils.output import (
DEFAULT_DATE_FORMAT,
DEFAULT_LOGGING_FORMAT,
seconds_to_milliseconds,
setup_log_directory,
write_report_json
write_report_json,
)
from harness_utils.procyoncmd import (
get_cuda_devices,
get_openvino_devices,
get_openvino_gpu,
get_winml_devices,
)
#####
@@ -41,149 +50,163 @@ CUDA_DEVICES = get_cuda_devices(ABS_EXECUTABLE_PATH)
CONFIG_DIR = SCRIPT_DIR / "config"
BENCHMARK_CONFIG = {
"AMD_GPU0_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def"',
"process_name": "ort-directml.exe",
"device_name": list(WINML_DEVICES.keys())[0],
"device_id": "0",
"test_name": "stable_diffusion_fp16",
"api": "onnx"
"api": "onnx",
},
"AMD_GPU1_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_onnxruntime.def"',
"process_name": "ort-directml.exe",
"device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0],
"device_name": list(WINML_DEVICES.keys())[1]
if len(list(WINML_DEVICES.keys())) > 1
else list(WINML_DEVICES.keys())[0],
"device_id": "1" if len(list(WINML_DEVICES.values())) > 1 else "0",
"test_name": "stable_diffusion_fp16",
"api": "onnx"
"api": "onnx",
},
"AMD_GPU0_XL_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def"',
"process_name": "ort-directml.exe",
"device_name": list(WINML_DEVICES.keys())[0],
"device_id": "0",
"test_name": "stable_diffusion_fp16_xl",
"api": "onnx"
"api": "onnx",
},
"AMD_GPU1_XL_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_onnxruntime.def"',
"process_name": "ort-directml.exe",
"device_name": list(WINML_DEVICES.keys())[1] if len(list(WINML_DEVICES.keys())) > 1 else list(WINML_DEVICES.keys())[0],
"device_id": list(WINML_DEVICES.values())[1] if len(list(WINML_DEVICES.values())) > 1 else list(WINML_DEVICES.values())[0],
"device_name": list(WINML_DEVICES.keys())[1]
if len(list(WINML_DEVICES.keys())) > 1
else list(WINML_DEVICES.keys())[0],
"device_id": list(WINML_DEVICES.values())[1]
if len(list(WINML_DEVICES.values())) > 1
else list(WINML_DEVICES.values())[0],
"test_name": "stable_diffusion_fp16_xl",
"api": "onnx"
"api": "onnx",
},
"Intel_GPU0_INT8": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"),
"test_name": "stable_diffusion_int8",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU0_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"),
"test_name": "stable_diffusion_fp16",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU0_XL_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.0" if "GPU.0" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.0"),
"test_name": "stable_diffusion_fp16_xl",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU1_INT8": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"),
"test_name": "stable_diffusion_int8",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU1_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"),
"test_name": "stable_diffusion_fp16",
"api": "openvino"
"api": "openvino",
},
"Intel_GPU1_XL_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_openvino.def"',
"process_name": "openvino.exe",
"device_id": "GPU.1" if "GPU.1" in list(OPENVINO_DEVICES.keys()) else "GPU",
"device_name": get_openvino_gpu(OPENVINO_DEVICES, "GPU.1"),
"test_name": "stable_diffusion_fp16_xl",
"api": "openvino"
"api": "openvino",
},
"NVIDIA_GPU_INT8": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_tensorrt.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15int8_tensorrt.def"',
"process_name": "tensorrt.exe",
"device_id": "cuda:0",
"device_name": CUDA_DEVICES.get("cuda:0"),
"test_name": "stable_diffusion_int8",
"api": "tensorrt"
"api": "tensorrt",
},
"NVIDIA_GPU_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_tensorrt.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sd15fp16_tensorrt.def"',
"process_name": "tensorrt.exe",
"device_id": "cuda:0",
"device_name": CUDA_DEVICES.get("cuda:0"),
"test_name": "stable_diffusion_fp16",
"api": "tensorrt"
"api": "tensorrt",
},
"NVIDIA_GPU_XL_FP16": {
"config": f"\"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_tensorrt.def\"",
"config": f'"{CONFIG_DIR}\\ai_imagegeneration_sdxlfp16_tensorrt.def"',
"process_name": "tensorrt.exe",
"device_id": "cuda:0",
"device_name": CUDA_DEVICES.get("cuda:0"),
"test_name": "stable_diffusion_fp16_xl",
"api": "tensorrt"
}
"api": "tensorrt",
},
}
RESULTS_FILENAME = "result.xml"
REPORT_PATH = LOG_DIR / RESULTS_FILENAME
RESULTS_XML_PATH = LOG_DIR / RESULTS_FILENAME
def setup_logging():
"""setup logging"""
setup_log_directory(str(LOG_DIR))
logging.basicConfig(filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG)
logging.basicConfig(
filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG,
)
console = logging.StreamHandler()
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logging.getLogger("").addHandler(console)
def get_arguments():
"""get arguments"""
parser = ArgumentParser()
parser.add_argument(
"--engine", dest="engine", help="Engine test type", required=True,
choices=BENCHMARK_CONFIG.keys())
"--engine",
dest="engine",
help="Engine test type",
required=True,
choices=BENCHMARK_CONFIG.keys(),
)
argies = parser.parse_args()
return argies
def create_procyon_command(test_option, process_name, device_id):
"""create command string"""
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\"'
command = (
f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}"'
)
match process_name:
case 'ort-directml.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-winml-device {device_id}'
case 'openvino.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-openvino-device {device_id}'
case 'tensorrt.exe':
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\" --select-cuda-device {device_id}'
case "ort-directml.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-winml-device {device_id}'
case "openvino.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-openvino-device {device_id}'
case "tensorrt.exe":
command = f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}" --select-cuda-device {device_id}'
command = command.rstrip()
return command
@@ -191,7 +214,12 @@ def create_procyon_command(test_option, process_name, device_id):
def run_benchmark(process_name, command_to_run):
"""run the benchmark"""
with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
with subprocess.Popen(
command_to_run,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
logging.info("Procyon AI Image Generation benchmark has started.")
while True:
now = time.time()
@@ -216,9 +244,11 @@ try:
args = get_arguments()
option = BENCHMARK_CONFIG[args.engine]["config"]
cmd = create_procyon_command(
option, BENCHMARK_CONFIG[args.engine]["process_name"],
BENCHMARK_CONFIG[args.engine]["device_id"])
logging.info('Starting benchmark!')
option,
BENCHMARK_CONFIG[args.engine]["process_name"],
BENCHMARK_CONFIG[args.engine]["device_id"],
)
logging.info("Starting benchmark!")
logging.info(cmd)
start_time = time.time()
pr = run_benchmark(BENCHMARK_CONFIG[args.engine]["process_name"], cmd)
@@ -235,6 +265,9 @@ try:
end_time = time.time()
elapsed_test_time = round(end_time - start_time, 2)
am = ArtifactManager(LOG_DIR)
am.copy_file(RESULTS_XML_PATH, ArtifactType.RESULTS_TEXT, "results xml file")
am.create_manifest()
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
logging.info("Score was %s", score)
@@ -248,8 +281,7 @@ try:
"device_name": BENCHMARK_CONFIG[args.engine]["device_name"],
"procyon_version": find_procyon_version(),
"unit": "score",
"score": score
"score": score,
}
write_report_json(str(LOG_DIR), "report.json", report)

116
procyon_ai_text_generation/ulprocai_text_gen.py
View File

@@ -1,23 +1,32 @@
"""UL Procyon AI Text Generation test script"""
# pylint: disable=no-name-in-module
from argparse import ArgumentParser
import logging
from pathlib import Path
import subprocess
import sys
import time
from argparse import ArgumentParser
from pathlib import Path
import psutil
from utils import regex_find_score_in_xml, is_process_running, get_install_path, find_procyon_version, find_test_version
from utils import (
find_procyon_version,
find_test_version,
get_install_path,
is_process_running,
regex_find_score_in_xml,
)
PARENT_DIR = str(Path(sys.path[0], ".."))
sys.path.append(PARENT_DIR)
from harness_utils.artifacts import ArtifactManager, ArtifactType
from harness_utils.output import (
DEFAULT_DATE_FORMAT,
DEFAULT_LOGGING_FORMAT,
seconds_to_milliseconds,
setup_log_directory,
write_report_json
write_report_json,
)
#####
@@ -31,114 +40,127 @@ ABS_EXECUTABLE_PATH = DIR_PROCYON / EXECUTABLE
CONFIG_DIR = SCRIPT_DIR / "config"
BENCHMARK_CONFIG = {
"All_Models_ONNX": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_all.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_all.def"',
"process_name": "Handler.exe",
"result_regex": r"<AIImageGenerationOverallScore>(\d+)",
"test_name": "all_models",
"api": "onnx"
"api": "onnx",
},
"Llama_2_13B_ONNX": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_llama2.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationLlama2OverallScore>(\d+)",
"test_name": "llama_2_13b",
"api": "onnx"
"api": "onnx",
},
"Llama_3_1_8B_ONNX": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_llama3.1.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationLlama3OverallScore>(\d+)",
"test_name": "llama_3_1_8b",
"api": "onnx"
"api": "onnx",
},
"Mistral_7B_ONNX": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_mistral.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationMistralOverallScore>(\d+)",
"test_name": "mistral_7b",
"api": "onnx"
"api": "onnx",
},
"Phi_3_5_ONNX": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_phi.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationPhiOverallScore>(\d+)",
"test_name": "phi_3_5",
"api": "onnx"
"api": "onnx",
},
"All_Models_OPENVINO": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_all_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_all_openvino.def"',
"process_name": "Handler.exe",
"result_regex": r"<AIImageGenerationOverallScore>(\d+)",
"test_name": "all_models",
"api": "openvino"
"api": "openvino",
},
"Llama_2_13B_OPENVINO": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_llama2_openvino.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationLlama2OverallScore>(\d+)",
"test_name": "llama_2_13b",
"api": "openvino"
"api": "openvino",
},
"Llama_3_1_8B_OPENVINO": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama3.1_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_llama3.1_openvino.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationLlama3OverallScore>(\d+)",
"test_name": "llama_3_1_8b",
"api": "openvino"
"api": "openvino",
},
"Mistral_7B_OPENVINO": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_mistral_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_mistral_openvino.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationMistralOverallScore>(\d+)",
"test_name": "mistral_7b",
"api": "openvino"
"api": "openvino",
},
"Phi_3_5_OPENVINO": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_phi_openvino.def\"",
"config": f'"{CONFIG_DIR}\\ai_textgeneration_phi_openvino.def"',
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationPhiOverallScore>(\d+)",
"test_name": "phi_3_5",
"api": "openvino"
}
"api": "openvino",
},
}
RESULTS_FILENAME = "result.xml"
REPORT_PATH = LOG_DIR / RESULTS_FILENAME
RESULTS_XML_PATH = LOG_DIR / RESULTS_FILENAME
def setup_logging():
"""setup logging"""
setup_log_directory(str(LOG_DIR))
logging.basicConfig(filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG)
logging.basicConfig(
filename=LOG_DIR / "harness.log",
format=DEFAULT_LOGGING_FORMAT,
datefmt=DEFAULT_DATE_FORMAT,
level=logging.DEBUG,
)
console = logging.StreamHandler()
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
logging.getLogger("").addHandler(console)
def get_arguments():
"""get arguments"""
parser = ArgumentParser()
parser.add_argument(
"--engine", dest="engine", help="Engine test type", required=True,
choices=BENCHMARK_CONFIG.keys())
"--engine",
dest="engine",
help="Engine test type",
required=True,
choices=BENCHMARK_CONFIG.keys(),
)
argies = parser.parse_args()
return argies
def create_procyon_command(test_option):
"""create command string"""
command = f'\"{ABS_EXECUTABLE_PATH}\" --definition={test_option} --export=\"{REPORT_PATH}\"'
command = (
f'"{ABS_EXECUTABLE_PATH}" --definition={test_option} --export="{RESULTS_XML_PATH}"'
)
command = command.rstrip()
return command
def run_benchmark(process_name, command_to_run):
"""run the benchmark"""
with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) as proc:
with subprocess.Popen(
command_to_run,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
logging.info("Procyon AI Text Generation benchmark has started.")
while True:
now = time.time()
@@ -159,7 +181,7 @@ try:
args = get_arguments()
option = BENCHMARK_CONFIG[args.engine]["config"]
cmd = create_procyon_command(option)
logging.info('Starting benchmark!')
logging.info("Starting benchmark!")
logging.info(cmd)
start_time = time.time()
pr = run_benchmark(BENCHMARK_CONFIG[args.engine]["process_name"], cmd)
@@ -171,7 +193,13 @@ try:
end_time = time.time()
elapsed_test_time = round(end_time - start_time, 2)
if not args.engine == "All_Models_OPENVINO" and not args.engine == "All_Models_ONNX":
am = ArtifactManager(LOG_DIR)
am.copy_file(RESULTS_XML_PATH, ArtifactType.RESULTS_TEXT, "results xml file")
am.create_manifest()
if (
not args.engine == "All_Models_OPENVINO"
and not args.engine == "All_Models_ONNX"
):
results_regex = BENCHMARK_CONFIG[args.engine]["result_regex"]
score = regex_find_score_in_xml(results_regex)
@@ -185,7 +213,7 @@ try:
"unit": "score",
"score": score,
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
"end_time": seconds_to_milliseconds(end_time),
}
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
@@ -198,10 +226,15 @@ try:
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
for test_type in BENCHMARK_CONFIG.items():
if test_type[0] == "All_Models_ONNX" or test_type[0] == "All_Models_OPENVINO":
if (
test_type[0] == "All_Models_ONNX"
or test_type[0] == "All_Models_OPENVINO"
):
continue
if ("ONNX" in args.engine and "ONNX" in test_type[0]) or ("OPENVINO" in args.engine and "OPENVINO" in test_type[0]):
if ("ONNX" in args.engine and "ONNX" in test_type[0]) or (
"OPENVINO" in args.engine and "OPENVINO" in test_type[0]
):
results_regex = test_type[1]["result_regex"]
score = regex_find_score_in_xml(results_regex)
@@ -220,8 +253,7 @@ try:
"test_version": find_test_version(),
"procyon_version": find_procyon_version(),
"unit": "score",
"score": score
"score": score,
}
session_report.append(report)