From b229c4f02735be84c3aaaaba0d8dff301f19288e Mon Sep 17 00:00:00 2001 From: j-lin-lmg <167900848+j-lin-lmg@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:20:51 -0800 Subject: [PATCH] Ffmpeg cpu benchmark (#179) Added ffmpeg cpu benchmark based on @nharris-lmg commands 3 options for encode (h264, av1, h265) will report final vmaf score as "score" in report json logs saved to artifacts --- .gitignore | 6 +- ffmpeg_cpu/ffmpeg_cpu.py | 156 ++++++++++++++++++ ffmpeg_cpu/ffmpeg_cpu_utils.py | 43 +++++ ffmpeg_cpu/manifest.yaml | 12 ++ procyon_ai/ulprocai.py | 4 +- .../ulprocai_text_gen.py | 0 6 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 ffmpeg_cpu/ffmpeg_cpu.py create mode 100644 ffmpeg_cpu/ffmpeg_cpu_utils.py create mode 100644 ffmpeg_cpu/manifest.yaml mode change 100644 => 100755 procyon_ai_text_generation/ulprocai_text_gen.py diff --git a/.gitignore b/.gitignore index d91f24a..78a99d6 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,8 @@ godot-4.4.1-stable.zip mingw64/ # python -__pycache__/ \ No newline at end of file +__pycache__/ + +ffmpeg-8.0.1-full_build/ +*.log +*.txt diff --git a/ffmpeg_cpu/ffmpeg_cpu.py b/ffmpeg_cpu/ffmpeg_cpu.py new file mode 100644 index 0000000..a84a249 --- /dev/null +++ b/ffmpeg_cpu/ffmpeg_cpu.py @@ -0,0 +1,156 @@ +"""test script for handbrake encoding tests""" + +import logging +import os +import re +import subprocess +import sys +from argparse import ArgumentParser +from pathlib import Path + +sys.path.insert(1, os.path.join(sys.path[0], "..")) + +from ffmpeg_cpu_utils import ( + copy_ffmpeg_from_network_drive, + copy_video_source, + current_time_ms, + ffmpeg_present, + is_video_source_present, +) + +from harness_utils.artifacts import ArtifactManager, ArtifactType +from harness_utils.output import ( + DEFAULT_DATE_FORMAT, + DEFAULT_LOGGING_FORMAT, + write_report_json, +) + +SCRIPT_DIR = Path(__file__).resolve().parent +LOG_DIR = SCRIPT_DIR.joinpath("run") +LOG_DIR.mkdir(exist_ok=True) + +LOG_FILE = LOG_DIR / "harness.log" +logging.basicConfig( + filename=LOG_FILE, + format=DEFAULT_LOGGING_FORMAT, + datefmt=DEFAULT_DATE_FORMAT, + level=logging.DEBUG, +) + +ENCODERS = ["h264", "av1", "h265"] +FFMPEG_EXE_PATH = SCRIPT_DIR / "ffmpeg-8.0.1-full_build" / "bin" / "ffmpeg.exe" +FFMPEG_VERSION = "ffmpeg-8.0.1-full_build" +VMAF_VERSION = "vmaf_v0.6.1neg" +console = logging.StreamHandler() +formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) +console.setFormatter(formatter) +logging.getLogger("").addHandler(console) + + +def main(): # pylint: disable=too-many-locals + """entrypoint""" + parser = ArgumentParser() + parser.add_argument("--encoder", dest="encoder", required=True) + args = parser.parse_args() + + if args.encoder not in ENCODERS: + logging.error("Invalid encoder selection: %s", args.encoder) + sys.exit(1) + + if not ffmpeg_present(): + logging.info("FFmpeg not found, copying from network drive...") + copy_ffmpeg_from_network_drive() + + if not is_video_source_present(): + logging.info("Video source not found, copying from network drive...") + copy_video_source() + + try: + start_encoding_time = current_time_ms() + logging.info("Starting ffmpeg_cpu benchmark...") + + if args.encoder == "h264": + command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libx264 -preset slow -profile:v high -level:v 5.1 -crf 20 -c:a copy output.mp4" + elif args.encoder == "av1": + command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libsvtav1 -preset 7 -profile:v main -level:v 5.1 -crf 20 -c:a copy output.mp4" + elif args.encoder == "h265": + command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libx265 -preset slow -profile:v main -level:v 5.1 -crf 20 -c:a copy output.mp4" + else: + logging.error("Invalid encoder selection: %s", args.encoder) + sys.exit(1) + + logging.info("Executing command: %s", command) + + with open("encoding.log", "w", encoding="utf-8") as encoding_log: + logging.info("Encoding...") + subprocess.run(command, stderr=encoding_log, check=True) + + end_encoding_time = current_time_ms() + logging.info("Encoding completed") + + logging.info("Beginning VMAF") + + source_path = SCRIPT_DIR / "big_buck_bunny_1080p24.y4m" + encoded_path = SCRIPT_DIR / "output.mp4" + filter_complex = ( + f"libvmaf=model=version={VMAF_VERSION}:n_threads=10:log_path=vmafout.txt" + ) + argument_list = [ + "-i", + str(source_path), + "-i", + str(encoded_path), + "-filter_complex", + filter_complex, + "-f", + "null", + "-", + ] + logging.info("VMAF args: %s", argument_list) + + vmaf_score = None + with open("vmaf.log", "w+", encoding="utf-8") as vmaf_log: + logging.info("Calculating VMAF...") + subprocess.run( + [FFMPEG_EXE_PATH, *argument_list], stderr=vmaf_log, check=True + ) + vmaf_log.flush() + vmaf_log.seek(0) + for line in reversed(vmaf_log.read().splitlines()): + if "VMAF score:" in line: + match = re.search(r"VMAF score:\s*([0-9]+(?:\.[0-9]+)?)", line) + if match: + vmaf_score = float(match.group(1)) + break + end_time = current_time_ms() + logging.info("VMAF score: %s", vmaf_score) + + logging.getLogger("").removeHandler(console) + logging.getLogger("").addHandler(console) + + am = ArtifactManager(LOG_DIR) + am.copy_file("encoding.log", ArtifactType.RESULTS_TEXT, "encoding log file") + am.copy_file("vmaf.log", ArtifactType.RESULTS_TEXT, "vmaf log file") + am.create_manifest() + + report = { + "test": "FFMPEG CPU Encoding", + "test_parameter": str(args.encoder), + "ffmpeg_version": FFMPEG_VERSION, + "vmaf_version": VMAF_VERSION, + "score": vmaf_score, + "unit": "score", + "encoding_duration": end_encoding_time - start_encoding_time, + "vmaf_duration": end_time - end_encoding_time, + "start_time": start_encoding_time, + "end_time": end_time, + } + write_report_json(str(LOG_DIR), "report.json", report) + except Exception as e: + logging.error("Something went wrong running the benchmark!") + logging.exception(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/ffmpeg_cpu/ffmpeg_cpu_utils.py b/ffmpeg_cpu/ffmpeg_cpu_utils.py new file mode 100644 index 0000000..6e10631 --- /dev/null +++ b/ffmpeg_cpu/ffmpeg_cpu_utils.py @@ -0,0 +1,43 @@ +"""utility functions for running ffmpeg tests""" + +import os +import shutil +import time +from pathlib import Path + +L_FFMPEG_FOLDER = Path( + "\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\ffmpeg\\ffmpeg-8.0.1-full_build" +) +SCRIPT_DIR = Path(os.path.dirname(os.path.realpath(__file__))) +FFMPEG_FOLDER_NAME = "ffmpeg-8.0.1-full_build" +FFMPEG_EXE_PATH = SCRIPT_DIR / FFMPEG_FOLDER_NAME / "bin" / "ffmpeg.exe" +SOURCE_VIDEO_NAME = "big_buck_bunny_1080p24.y4m" + + +def ffmpeg_present() -> bool: + """Check if ffmpeg is present on the system""" + return os.path.isfile(FFMPEG_EXE_PATH) + + +def copy_ffmpeg_from_network_drive(): + """copy ffmpeg cli from network drive""" + source = L_FFMPEG_FOLDER + shutil.copytree(source, SCRIPT_DIR / FFMPEG_FOLDER_NAME) + + +def is_video_source_present() -> bool: + """check if big buck bunny video source is present""" + return os.path.isfile(Path(SCRIPT_DIR / SOURCE_VIDEO_NAME)) + + +def copy_video_source(): + """copy big buck bunny source video to local from network drive""" + source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m" + root_dir = os.path.dirname(os.path.realpath(__file__)) + destination = os.path.join(root_dir, SOURCE_VIDEO_NAME) + shutil.copyfile(source, destination) + + +def current_time_ms(): + """Get current timestamp in milliseconds since epoch""" + return int(time.time() * 1000) diff --git a/ffmpeg_cpu/manifest.yaml b/ffmpeg_cpu/manifest.yaml new file mode 100644 index 0000000..f469ab0 --- /dev/null +++ b/ffmpeg_cpu/manifest.yaml @@ -0,0 +1,12 @@ +friendly_name: "FFMPEG CPU VMAF" +executable: "ffmpeg_cpu.py" +process_name: "ffmpeg.exe" +disable_presentmon: true +output_dir: run +options: + - name: encoder + type: select + values: + - h264 + - av1 + - h265 \ No newline at end of file diff --git a/procyon_ai/ulprocai.py b/procyon_ai/ulprocai.py index 88800c6..34b1c92 100644 --- a/procyon_ai/ulprocai.py +++ b/procyon_ai/ulprocai.py @@ -210,7 +210,6 @@ try: logging.info("Detected OpenVino Devices: %s", str(OPENVINO_DEVICES)) logging.info("Detected CUDA Devices: %s", (CUDA_DEVICES)) - am = ArtifactManager(LOG_DIR) args = get_arguments() option = BENCHMARK_CONFIG[args.engine]["config"] proc_name = BENCHMARK_CONFIG[args.engine]["process_name"] @@ -230,7 +229,9 @@ try: logging.error("Could not find overall score!") sys.exit(1) + am = ArtifactManager(LOG_DIR) am.copy_file(RESULTS_XML_PATH, ArtifactType.RESULTS_TEXT, "results xml file") + am.create_manifest() end_time = time.time() elapsed_test_time = round(end_time - start_time, 2) logging.info("Benchmark took %.2f seconds", elapsed_test_time) @@ -248,7 +249,6 @@ try: "unit": "score", "score": score, } - am.create_manifest() write_report_json(str(LOG_DIR), "report.json", report) except Exception as e: logging.error("Something went wrong running the benchmark!") diff --git a/procyon_ai_text_generation/ulprocai_text_gen.py b/procyon_ai_text_generation/ulprocai_text_gen.py old mode 100644 new mode 100755