Ffmpeg cpu benchmark (#179)

Added ffmpeg cpu benchmark based on @nharris-lmg commands

3 options for encode (h264, av1, h265)

will report final vmaf score as "score" in report json

logs saved to artifacts

.gitignore

@@ -38,4 +38,8 @@ godot-4.4.1-stable.zip
 mingw64/
 
 # python
-__pycache__/
+__pycache__/
+
+ffmpeg-8.0.1-full_build/
+*.log
+*.txt

ffmpeg_cpu/ffmpeg_cpu.py

@@ -0,0 +1,156 @@
+"""test script for handbrake encoding tests"""
+
+import logging
+import os
+import re
+import subprocess
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+sys.path.insert(1, os.path.join(sys.path[0], ".."))
+
+from ffmpeg_cpu_utils import (
+    copy_ffmpeg_from_network_drive,
+    copy_video_source,
+    current_time_ms,
+    ffmpeg_present,
+    is_video_source_present,
+)
+
+from harness_utils.artifacts import ArtifactManager, ArtifactType
+from harness_utils.output import (
+    DEFAULT_DATE_FORMAT,
+    DEFAULT_LOGGING_FORMAT,
+    write_report_json,
+)
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = SCRIPT_DIR.joinpath("run")
+LOG_DIR.mkdir(exist_ok=True)
+
+LOG_FILE = LOG_DIR / "harness.log"
+logging.basicConfig(
+    filename=LOG_FILE,
+    format=DEFAULT_LOGGING_FORMAT,
+    datefmt=DEFAULT_DATE_FORMAT,
+    level=logging.DEBUG,
+)
+
+ENCODERS = ["h264", "av1", "h265"]
+FFMPEG_EXE_PATH = SCRIPT_DIR / "ffmpeg-8.0.1-full_build" / "bin" / "ffmpeg.exe"
+FFMPEG_VERSION = "ffmpeg-8.0.1-full_build"
+VMAF_VERSION = "vmaf_v0.6.1neg"
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger("").addHandler(console)
+
+
+def main():  # pylint: disable=too-many-locals
+    """entrypoint"""
+    parser = ArgumentParser()
+    parser.add_argument("--encoder", dest="encoder", required=True)
+    args = parser.parse_args()
+
+    if args.encoder not in ENCODERS:
+        logging.error("Invalid encoder selection: %s", args.encoder)
+        sys.exit(1)
+
+    if not ffmpeg_present():
+        logging.info("FFmpeg not found, copying from network drive...")
+        copy_ffmpeg_from_network_drive()
+
+    if not is_video_source_present():
+        logging.info("Video source not found, copying from network drive...")
+        copy_video_source()
+
+    try:
+        start_encoding_time = current_time_ms()
+        logging.info("Starting ffmpeg_cpu benchmark...")
+
+        if args.encoder == "h264":
+            command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libx264 -preset slow -profile:v high -level:v 5.1 -crf 20 -c:a copy output.mp4"
+        elif args.encoder == "av1":
+            command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libsvtav1 -preset 7 -profile:v main -level:v 5.1 -crf 20 -c:a copy output.mp4"
+        elif args.encoder == "h265":
+            command = f"{FFMPEG_EXE_PATH} -y -i {SCRIPT_DIR}\\big_buck_bunny_1080p24.y4m -c:v libx265 -preset slow -profile:v main -level:v 5.1 -crf 20 -c:a copy output.mp4"
+        else:
+            logging.error("Invalid encoder selection: %s", args.encoder)
+            sys.exit(1)
+
+        logging.info("Executing command: %s", command)
+
+        with open("encoding.log", "w", encoding="utf-8") as encoding_log:
+            logging.info("Encoding...")
+            subprocess.run(command, stderr=encoding_log, check=True)
+
+        end_encoding_time = current_time_ms()
+        logging.info("Encoding completed")
+
+        logging.info("Beginning VMAF")
+
+        source_path = SCRIPT_DIR / "big_buck_bunny_1080p24.y4m"
+        encoded_path = SCRIPT_DIR / "output.mp4"
+        filter_complex = (
+            f"libvmaf=model=version={VMAF_VERSION}:n_threads=10:log_path=vmafout.txt"
+        )
+        argument_list = [
+            "-i",
+            str(source_path),
+            "-i",
+            str(encoded_path),
+            "-filter_complex",
+            filter_complex,
+            "-f",
+            "null",
+            "-",
+        ]
+        logging.info("VMAF args: %s", argument_list)
+
+        vmaf_score = None
+        with open("vmaf.log", "w+", encoding="utf-8") as vmaf_log:
+            logging.info("Calculating VMAF...")
+            subprocess.run(
+                [FFMPEG_EXE_PATH, *argument_list], stderr=vmaf_log, check=True
+            )
+            vmaf_log.flush()
+            vmaf_log.seek(0)
+            for line in reversed(vmaf_log.read().splitlines()):
+                if "VMAF score:" in line:
+                    match = re.search(r"VMAF score:\s*([0-9]+(?:\.[0-9]+)?)", line)
+                    if match:
+                        vmaf_score = float(match.group(1))
+                    break
+        end_time = current_time_ms()
+        logging.info("VMAF score: %s", vmaf_score)
+
+        logging.getLogger("").removeHandler(console)
+        logging.getLogger("").addHandler(console)
+
+        am = ArtifactManager(LOG_DIR)
+        am.copy_file("encoding.log", ArtifactType.RESULTS_TEXT, "encoding log file")
+        am.copy_file("vmaf.log", ArtifactType.RESULTS_TEXT, "vmaf log file")
+        am.create_manifest()
+
+        report = {
+            "test": "FFMPEG CPU Encoding",
+            "test_parameter": str(args.encoder),
+            "ffmpeg_version": FFMPEG_VERSION,
+            "vmaf_version": VMAF_VERSION,
+            "score": vmaf_score,
+            "unit": "score",
+            "encoding_duration": end_encoding_time - start_encoding_time,
+            "vmaf_duration": end_time - end_encoding_time,
+            "start_time": start_encoding_time,
+            "end_time": end_time,
+        }
+        write_report_json(str(LOG_DIR), "report.json", report)
+    except Exception as e:
+        logging.error("Something went wrong running the benchmark!")
+        logging.exception(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

ffmpeg_cpu/ffmpeg_cpu_utils.py

@@ -0,0 +1,43 @@
+"""utility functions for running ffmpeg tests"""
+
+import os
+import shutil
+import time
+from pathlib import Path
+
+L_FFMPEG_FOLDER = Path(
+    "\\\\labs.lmg.gg\\labs\\01_Installers_Utilities\\ffmpeg\\ffmpeg-8.0.1-full_build"
+)
+SCRIPT_DIR = Path(os.path.dirname(os.path.realpath(__file__)))
+FFMPEG_FOLDER_NAME = "ffmpeg-8.0.1-full_build"
+FFMPEG_EXE_PATH = SCRIPT_DIR / FFMPEG_FOLDER_NAME / "bin" / "ffmpeg.exe"
+SOURCE_VIDEO_NAME = "big_buck_bunny_1080p24.y4m"
+
+
+def ffmpeg_present() -> bool:
+    """Check if ffmpeg is present on the system"""
+    return os.path.isfile(FFMPEG_EXE_PATH)
+
+
+def copy_ffmpeg_from_network_drive():
+    """copy ffmpeg cli from network drive"""
+    source = L_FFMPEG_FOLDER
+    shutil.copytree(source, SCRIPT_DIR / FFMPEG_FOLDER_NAME)
+
+
+def is_video_source_present() -> bool:
+    """check if big buck bunny video source is present"""
+    return os.path.isfile(Path(SCRIPT_DIR / SOURCE_VIDEO_NAME))
+
+
+def copy_video_source():
+    """copy big buck bunny source video to local from network drive"""
+    source = r"\\labs.lmg.gg\labs\03_ProcessingFiles\Handbrake Test\big_buck_bunny_1080p24.y4m"
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    destination = os.path.join(root_dir, SOURCE_VIDEO_NAME)
+    shutil.copyfile(source, destination)
+
+
+def current_time_ms():
+    """Get current timestamp in milliseconds since epoch"""
+    return int(time.time() * 1000)

ffmpeg_cpu/manifest.yaml

@@ -0,0 +1,12 @@
+friendly_name: "FFMPEG CPU VMAF"
+executable: "ffmpeg_cpu.py"
+process_name: "ffmpeg.exe"
+disable_presentmon: true
+output_dir: run
+options:
+  - name: encoder
+    type: select
+    values:
+      - h264
+      - av1
+      - h265

procyon_ai/ulprocai.py

@@ -210,7 +210,6 @@ try:
     logging.info("Detected OpenVino Devices: %s", str(OPENVINO_DEVICES))
     logging.info("Detected CUDA Devices: %s", (CUDA_DEVICES))
 
-    am = ArtifactManager(LOG_DIR)
     args = get_arguments()
     option = BENCHMARK_CONFIG[args.engine]["config"]
     proc_name = BENCHMARK_CONFIG[args.engine]["process_name"]
@@ -230,7 +229,9 @@ try:
         logging.error("Could not find overall score!")
         sys.exit(1)
 
+    am = ArtifactManager(LOG_DIR)
     am.copy_file(RESULTS_XML_PATH, ArtifactType.RESULTS_TEXT, "results xml file")
+    am.create_manifest()
     end_time = time.time()
     elapsed_test_time = round(end_time - start_time, 2)
     logging.info("Benchmark took %.2f seconds", elapsed_test_time)
@@ -248,7 +249,6 @@ try:
         "unit": "score",
         "score": score,
     }
-    am.create_manifest()
     write_report_json(str(LOG_DIR), "report.json", report)
 except Exception as e:
     logging.error("Something went wrong running the benchmark!")