github/markbench-tests
1
1
Fork 0
mirror of https://github.com/LTTLabsOSS/markbench-tests.git synced 2026-01-09 14:07:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

update to all models run of benchmark

Browse Source
This commit is contained in:
Arty Blue
2024-12-18 10:28:11 -08:00
parent a38abc5bf0
commit 274a1bfd37
2 changed files with 33 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
procyon_ai_text_generation/manifest.yaml
View File

@@ -1,5 +1,5 @@
friendly_name: "Procyon AI Text Generation"
executable: "ulprocai_img_gen.py"
executable: "ulprocai_text_gen.py"
process_name: "ProcyonCmd.exe"
disable_presentmon: true
output_dir: "run"

41
procyon_ai_text_generation/ulprocai_text_gen.py
View File

@@ -38,7 +38,7 @@ BENCHMARK_CONFIG = {
"Llama_2_13B": {
"config": f"\"{CONFIG_DIR}\\ai_textgeneration_llama2.def\"",
"process_name": "Handler.exe",
"result_regex": r"<AiTextGenerationLlama2OverallScore>(\d+)",
"result_regex": r"<AiTextGenerationLlama2OverallScore >(\d+)",
"test_name": "LLama 2 Text Generation"
},
"Llama_3_1_8B": {
@@ -134,15 +134,38 @@ try:
logging.info("Benchmark took %.2f seconds", elapsed_test_time)
logging.info("Score was %s", score)
report = {
"test": BENCHMARK_CONFIG[args.engine]["test_name"],
"unit": "score",
"score": score,
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
}
if not BENCHMARK_CONFIG[args.engine] == "All_Models":
report = {
"test": BENCHMARK_CONFIG[args.engine]["test_name"],
"unit": "score",
"score": score,
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
}
write_report_json(LOG_DIR, "report.json", report)
write_report_json(LOG_DIR, "report.json", report)
else:
session_report = []
for test_type in BENCHMARK_CONFIG.items():
if test_type[0] == "All_Models":
continue
score = find_score_in_xml(test_type[1]["result_regex"])
if score is None:
logging.error("Could not find overall score!")
sys.exit(1)
report = {
"test": test_type[0],
"unit": "score",
"score": score,
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
}
write_report_json(LOG_DIR, "report.json", session_report)
except Exception as e:
logging.error("Something went wrong running the benchmark!")
logging.exception(e)