Audit SotTR (#27)

* fix some readmes

* remove options from readme

* remove harness specific pyproject and poetry lock

* remove harness presets

* remove outdated info from readme

* remove template tests

* pep8 lint

* Add output to readme

* misc lint
This commit is contained in:
Derek Hirotsu
2023-09-18 15:37:34 -07:00
committed by GitHub
parent 8d1747eb98
commit b750b8625d
27 changed files with 83 additions and 3354 deletions

View File

@@ -13,6 +13,7 @@ sys.path.insert(1, os.path.join(sys.path[0], ".."))
#pylint: disable=wrong-import-position
from harness_utils.keras_service import KerasService
from harness_utils.steam import DEFAULT_EXECUTABLE_PATH as STEAM_PATH
from harness_utils.logging import (
format_resolution,
seconds_to_milliseconds,
@@ -28,8 +29,6 @@ from harness_utils.process import terminate_processes
SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
STEAM_GAME_ID = 1692250
STEAM_PATH = os.path.join(os.environ["ProgramFiles(x86)"], "steam")
STEAM_EXECUTABLE = "steam.exe"
VIDEO_PATH = r"C:\Program Files (x86)\Steam\steamapps\common\F1 22\videos"
skippable = [
@@ -51,9 +50,8 @@ def wait_for_word(word: str, attempts: int = 5, delay_seconds: int = 1) -> bool:
def start_game():
"""Starts the game via steam command."""
steam_run_arg = "steam://rungameid/" + str(STEAM_GAME_ID)
start_cmd = STEAM_PATH + "\\" + STEAM_EXECUTABLE
logging.info("%s %s", start_cmd, steam_run_arg)
return Popen([start_cmd, steam_run_arg])
logging.info("%s %s", STEAM_PATH, steam_run_arg)
return Popen([STEAM_PATH, steam_run_arg])
def navigate_overlay():

View File

@@ -29,8 +29,6 @@ from harness_utils.logging import (
SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
STEAM_GAME_ID = 2108330
STEAM_PATH = os.path.join(os.environ["ProgramFiles(x86)"], "steam")
STEAM_EXECUTABLE = "steam.exe"
PROCESS_NAME = "F1_23"
@@ -47,10 +45,8 @@ def is_word_on_screen(word: str, attempts: int = 5, delay_seconds: int = 1) -> b
def start_game():
"""Starts the game via steam command."""
steam_run_arg = "steam://rungameid/" + str(STEAM_GAME_ID)
start_cmd = STEAM_PATH + "\\" + STEAM_EXECUTABLE
logging.info("%s %s", start_cmd, steam_run_arg)
thread = Popen([start_cmd, steam_run_arg])
return thread
logging.info("%s %s", STEAM_PATH, steam_run_arg)
return Popen([STEAM_PATH, steam_run_arg])
def official() -> any:

View File

@@ -19,8 +19,6 @@ from harness_utils.steam import DEFAULT_EXECUTABLE_PATH as STEAM_PATH
#pylint: enable=wrong-import-position
STEAM_GAME_ID = 1091500
STEAM_PATH = os.path.join(os.environ["ProgramFiles(x86)"], "steam")
STEAM_EXECUTABLE = "steam.exe"
SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
PROCESS_NAME = "cyberpunk2077.exe"
@@ -28,8 +26,7 @@ PROCESS_NAME = "cyberpunk2077.exe"
def start_game():
"""Launch the game with no launcher or start screen"""
cmd = os.path.join(STEAM_PATH, STEAM_EXECUTABLE)
cmd_array = [cmd, "-applaunch",
cmd_array = [STEAM_PATH, "-applaunch",
str(STEAM_GAME_ID), "--launcher-skip", "-skipStartScreen"]
logging.info(" ".join(cmd_array))
return Popen(cmd_array)

View File

@@ -0,0 +1,15 @@
# FarCry 6
This script navigates through in-game menus to the built in benchmark and runs it with the current settings.
## Prerequisites
- Python 3.10+
- FarCry6 installed via Ubisoft Game Launcher
## Output
report.json
- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[heigt]", e.x. `1920x1080`
- `start_time`: number representing a timestamp of the test's start time in milliseconds
- `end_time`: number representing a timestamp of the test's end time in milliseconds

View File

@@ -5,7 +5,7 @@ This script navigates through in-game menus to the built in benchmark and runs i
## Prerequisites
- Python 3.10+
- Hitman 3 installed
- Hitman 3 installed via Steam
## Output

View File

@@ -1,63 +1,21 @@
# Shadow of the Tomb Raider
## TODO's
- Iteration within the same game process.
- Accept resolution as a separate argument to the harness.
## Prerequisites
- Python 3.10+
- Shadow of the Tomb Raider installed.
- Shadow of the Tomb Raider installed via Steam
## Setup
1. Follow the setup instructions for the framework. If you have done so, all required python dependencies *should* be installed.
1. Follow the setup instructions for the framework.
2. Install Shadow of the Tomb Raider from steam.
1. Location does not matter, this harness uses steam to launch the game.
## Configuration
## Output
Below is an example use of this harness as a test in a benchmark configuration.
```yaml
...
...
tests:
- name: shadowofthetombraider
executable: "shadowofthetombraider.py"
process_name: "SOTTR.exe"
output_dir: "run"
args:
- "--preset medium"
- "--resolution 1920,1080
```
__name__ : _(required)_ name of the test. This much match the name of a directory in the harness folder so the framework
can find the executable and any supplementary files.
__executable__ : _(required)_ the entry point to the test harness. In this case a python script.
__process_name__ : _(required)_ The process name that should be the target for FPS recording (ex: PresentMon).
__output_dir__: _(optional)_ Directory containing files to aggregate copies of after a successful test run. If a directory path is
given, the contents are copied.
__args__ : _(optional)_ list of arguments to be appended to the command to execute. All the arguments will be passed to
the executable when invoked by the framework.
### Arguments
|flag|required|what?|notes
|--|--|--|--|
|--preset|No|Graphics preset to load for test|See the `presets` folder to determine options. If none provided, the current settings will be used|
|--resolution|No|Display settings to load for test|If none provided, current display settings will be used|
#### Presets
This harness requires a single argument for the option `preset`. This is the graphics presets that are found in the game. They are represented in YAML in the folder **presets**. To select one you take the prefix of the name of the file, and the harness will find the corresponding YAML file.
For example if I pass in the argument `--preset medium` to the harness. The harness will load the settings in `presets/medium.presets.yaml`. You can also create and supply a custom preset if you wish.
#### Resolution
Resolution is expected to be givin in the format `height,width` so for example `1920,1080`. An error will be thrown if that format is not provided.
report.json
- `resolution`: string representing the resolution the test was run at, formatted as "[width]x[heigt]", e.x. `1920x1080`
- `start_time`: number representing a timestamp of the test's start time in milliseconds
- `end_time`: number representing a timestamp of the test's end time in milliseconds
## Common Issues
1. "Steam cannot sync with cloud"

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.9 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.9 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.3 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 MiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 MiB

View File

@@ -3,12 +3,4 @@ executable: "shadowofthetombraider.py"
process_name: "SOTTR.exe"
# default recording delay to reduce capturing menus during setup, this should be revisited every test bench as loading times may be different
recording_delay: 60
output_dir: "run"
options:
- name: preset
type: select
# presets disabled in favor of manual process for now
# values: [current, low, medium, high, highest]
values: [current]
tooltip: Don't forget to set graphics settings!
output_dir: "run"

File diff suppressed because one or more lines are too long

View File

@@ -1,48 +0,0 @@
name: high
settings:
- name: texture_quality
reg_key: TextureQuality
value: 2
- name: texture_filtering
reg_key: TextureFiltering
value: 2
- name: shadow_quality
reg_key: ShadowQuality
value: 2
- name: ambient_occlusion
reg_key: AmbientOcclusionQuality
value: 1
- name: depth_of_field
reg_key: DOFQuality
value: 1
- name: level_of_detail
reg_key: LevelOfDetail
value: 1
- name: tesselation
reg_key: Tessellation
value: 1
- name: bloom
reg_key: Bloom
value: 1
- name: motion_blur
reg_key: MotionBlur
value: 1
- name: screen_space_reflections
reg_key: ScreenSpaceReflections
value: 1
- name: screen_space_contact_shadows
reg_key: ScreenSpaceContactShadows
value: 0
- name: pure_hair
reg_key: TressFX
value: 1
- name: volumetric_lighting
reg_key: VolumetricLighting
value: 1
- name: lens_flares
reg_key: LensFlares
value: 1
- name: screen_effects
reg_key: ScreenEffects
value: 1

View File

@@ -1,48 +0,0 @@
name: highest
settings:
- name: texture_quality
reg_key: TextureQuality
value: 3
- name: texture_filtering
reg_key: TextureFiltering
value: 3
- name: shadow_quality
reg_key: ShadowQuality
value: 3
- name: ambient_occlusion
reg_key: AmbientOcclusionQuality
value: 2
- name: depth_of_field
reg_key: DOFQuality
value: 2
- name: level_of_detail
reg_key: LevelOfDetail
value: 2
- name: tesselation
reg_key: Tessellation
value: 1
- name: bloom
reg_key: Bloom
value: 1
- name: motion_blur
reg_key: MotionBlur
value: 1
- name: screen_space_reflections
reg_key: ScreenSpaceReflections
value: 1
- name: screen_space_contact_shadows
reg_key: ScreenSpaceContactShadows
value: 1
- name: pure_hair
reg_key: TressFX
value: 1
- name: volumetric_lighting
reg_key: VolumetricLighting
value: 1
- name: lens_flares
reg_key: LensFlares
value: 1
- name: screen_effects
reg_key: ScreenEffects
value: 1

View File

@@ -1,48 +0,0 @@
name: low
settings:
- name: texture_quality
reg_key: TextureQuality
value: 0
- name: texture_filtering
reg_key: TextureFiltering
value: 0
- name: shadow_quality
reg_key: ShadowQuality
value: 1
- name: ambient_occlusion
reg_key: AmbientOcclusionQuality
value: 0
- name: depth_of_field
reg_key: DOFQuality
value: 0
- name: level_of_detail
reg_key: LevelOfDetail
value: 0
- name: tesselation
reg_key: Tessellation
value: 0
- name: bloom
reg_key: Bloom
value: 1
- name: motion_blur
reg_key: MotionBlur
value: 0
- name: screen_space_reflections
reg_key: ScreenSpaceReflections
value: 0
- name: screen_space_contact_shadows
reg_key: ScreenSpaceContactShadows
value: 0
- name: pure_hair
reg_key: TressFX
value: 0
- name: volumetric_lighting
reg_key: VolumetricLighting
value: 1
- name: lens_flares
reg_key: LensFlares
value: 1
- name: screen_effects
reg_key: ScreenEffects
value: 1

View File

@@ -1,48 +0,0 @@
name: medium
settings:
- name: texture_quality
reg_key: TextureQuality
value: 1
- name: texture_filtering
reg_key: TextureFiltering
value: 1
- name: shadow_quality
reg_key: ShadowQuality
value: 1
- name: ambient_occlusion
reg_key: AmbientOcclusionQuality
value: 1
- name: depth_of_field
reg_key: DOFQuality
value: 1
- name: level_of_detail
reg_key: LevelOfDetail
value: 1
- name: tesselation
reg_key: Tessellation
value: 0
- name: bloom
reg_key: Bloom
value: 1
- name: motion_blur
reg_key: MotionBlur
value: 1
- name: screen_space_reflections
reg_key: ScreenSpaceReflections
value: 1
- name: screen_space_contact_shadows
reg_key: ScreenSpaceContactShadows
value: 0
- name: pure_hair
reg_key: TressFX
value: 1
- name: volumetric_lighting
reg_key: VolumetricLighting
value: 1
- name: lens_flares
reg_key: LensFlares
value: 1
- name: screen_effects
reg_key: ScreenEffects
value: 1

View File

@@ -1,21 +0,0 @@
[tool.poetry]
name = "shadowofthetombraider-harness"
version = "0.1.0"
description = ""
authors = ["Nikolas Harris <nikolas@linusmediagroup.com"]
[tool.poetry.dependencies]
python = "^3.10"
PyAutoGUI = "^0.9.53"
PyDirectInput = "^1.0.4"
opencv-python = "^4.5.5"
Pillow = "^9.1.1"
psutil = "^5.9.1"
PyYAML = "^6.0"
imutils = "^0.5.4"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -1,8 +1,10 @@
"""Utility functions for Shadow of the Tomb Raider test script"""
import winreg
import os
import cv2
def get_reg(name) -> any:
"""Get registry key value"""
reg_path = r'SOFTWARE\Eidos Montreal\Shadow of the Tomb Raider\Graphics'
try:
registry_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, reg_path, 0,
@@ -15,6 +17,7 @@ def get_reg(name) -> any:
def get_resolution() -> tuple[int]:
"""Get resolution from registry"""
width = get_reg("FullscreenWidth")
height = get_reg("FullscreenHeight")
return (height, width)

View File

@@ -1,15 +1,25 @@
"""Shadow of the Tomb Raider test script"""
from subprocess import Popen
import sys
import os
import logging
import time
import sys
import pydirectinput as user
import pyautogui as gui
from shadow_of_the_tomb_raider_utils import get_resolution, templates
#pylint: disable=wrong-import-position
sys.path.insert(1, os.path.join(sys.path[0], '..'))
#pylint: disable=wrong-import-position
import deprecated.cv2_utils
from harness_utils.logging import *
from harness_utils.logging import (
seconds_to_milliseconds,
setup_log_directory,
write_report_json,
DEFAULT_LOGGING_FORMAT,
DEFAULT_DATE_FORMAT,
)
from harness_utils.process import terminate_processes
from harness_utils.steam import get_run_game_id_command, DEFAULT_EXECUTABLE_PATH as steam_path
from harness_utils.steam import get_run_game_id_command, DEFAULT_EXECUTABLE_PATH as steam_path
#pylint: enable=wrong-import-position
STEAM_GAME_ID = 750920
@@ -18,12 +28,13 @@ LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
def start_game():
"""Starts the game via steam command."""
steam_run_arg = get_run_game_id_command(STEAM_GAME_ID)
logging.info(steam_path + " " + steam_run_arg)
logging.info("%s %s", steam_path, steam_run_arg)
return Popen([steam_path, steam_run_arg])
def clickOptions(options):
def click_options(options):
"""
If the game is freshly installed the main menu has less options thus the options button
looks different. We also account for if the options button is highlighted if the mouse
@@ -33,60 +44,53 @@ def clickOptions(options):
return
try:
deprecated.cv2_utils.wait_and_click(options[0], "graphics options", deprecated.cv2_utils.ClickType.HARD)
except:
clickOptions(options[1:])
deprecated.cv2_utils.wait_and_click(
options[0], "graphics options", deprecated.cv2_utils.ClickType.HARD)
except deprecated.cv2_utils.ImageNotFoundTimeout:
click_options(options[1:])
def run_benchmark():
"""
Start game via Steam and enter fullscreen mode
"""
t1 = time.time()
game_process = start_game()
"""Start game via Steam and enter fullscreen mode"""
setup_start_time = time.time()
start_game()
try:
deprecated.cv2_utils.wait_and_click('load_menu_play', "play button", timeout=30)
except:
except deprecated.cv2_utils.ImageNotFoundTimeout:
deprecated.cv2_utils.wait_and_click('load_menu_play_orange', "play button", timeout=30)
"""
Wait for game to load and enter graphics submenu
"""
optionImages = [
# Wait for game to load and enter graphics submenu
option_images = [
'menu_options_save_game',
'menu_options_save_game_highlighted',
'menu_options',
'menu_options_highlighted',
]
clickOptions(optionImages)
click_options(option_images)
deprecated.cv2_utils.wait_and_click('menu_graphics', "graphics options", deprecated.cv2_utils.ClickType.HARD)
time.sleep(2) # let the menu transition
screen = gui.screenshot(os.path.join(LOG_DIRECTORY, "display_settings.png"))
gui.screenshot(os.path.join(LOG_DIRECTORY, "display_settings.png"))
deprecated.cv2_utils.wait_and_click('menu_graphics_tab', "graphics tab", deprecated.cv2_utils.ClickType.HARD)
screen = gui.screenshot(os.path.join(LOG_DIRECTORY, "graphics_settings.png"))
gui.screenshot(os.path.join(LOG_DIRECTORY, "graphics_settings.png"))
"""
Start the benchmark!
"""
t2 = time.time()
logging.info(f"Harness setup took {round((t2 - t1), 2)} seconds")
elapsed_setup_time = round(time.time() - setup_start_time, 2)
logging.info("Setup took %f seconds", elapsed_setup_time)
user.press("r")
start_time = time.time()
test_start_time = time.time()
"""
Wait for benchmark to complete
"""
# Wait for benchmark to complete
time.sleep(180)
deprecated.cv2_utils.wait_for_image_on_screen('results_header', interval=2, timeout=60)
end_time = time.time()
logging.info(f"Benchark took {round((end_time - start_time), 2)} seconds")
screen = gui.screenshot(os.path.join(LOG_DIRECTORY, "results.png"))
test_end_time = time.time()
elapsed_test_time = round(test_end_time - test_start_time, 2)
logging.info("Benchmark took %f seconds", elapsed_test_time)
gui.screenshot(os.path.join(LOG_DIRECTORY, "results.png"))
# Exit
terminate_processes("SOTTR")
return start_time, end_time
return test_start_time, test_end_time
setup_log_directory(LOG_DIRECTORY)
@@ -106,14 +110,14 @@ try:
height, width = get_resolution()
result = {
"resolution": f"{width}x{height}",
"graphics_preset": "current",
"start_time": seconds_to_milliseconds(start_time),
"end_time": seconds_to_milliseconds(end_time)
}
write_report_json(LOG_DIRECTORY, "report.json", result)
#pylint: disable=broad-exception-caught
except Exception as e:
logging.error("Something went wrong running the benchmark!")
logging.exception(e)
terminate_processes("SOTTR")
exit(1)
sys.exit(1)

View File

@@ -1,23 +0,0 @@
import cv2
import os
from cv2_utils import *
script_dir = os.path.dirname(os.path.realpath(__file__))
images_dir = os.path.join(script_dir, "images")
test_images_dir = os.path.join(images_dir, "tests")
test_menus = {
"mainmenu_2k": cv2.imread(os.path.join(test_images_dir, "mainmenu_2k.png"), cv2.IMREAD_UNCHANGED),
"graphicsmenu_2k": cv2.imread(os.path.join(test_images_dir, "graphicsmenu_2k.png"), cv2.IMREAD_UNCHANGED),
"mainmenu_4k": cv2.imread(os.path.join(test_images_dir, "mainmenu_4k.png"), cv2.IMREAD_UNCHANGED),
"mainmenu_1": cv2.imread(os.path.join(test_images_dir, "menu1.png"), cv2.IMREAD_UNCHANGED),
"mainmenu_2": cv2.imread(os.path.join(test_images_dir, "menu2.png"), cv2.IMREAD_UNCHANGED),
"mainmenu_3": cv2.imread(os.path.join(test_images_dir, "menu3.png"), cv2.IMREAD_UNCHANGED),
"mainmenu_4": cv2.imread(os.path.join(test_images_dir, "menu4.png"), cv2.IMREAD_UNCHANGED),
"16:10MainMenu": cv2.imread(os.path.join(test_images_dir, "main_menu_1920x1200.png"), cv2.IMREAD_UNCHANGED)
}
found2 = locate_in_image(get_template('menu_options'), test_menus['mainmenu_4k'], threshold=0.8, debug=0)
print(found2)

View File

@@ -1,19 +0,0 @@
import re
import shutil
import tempfile
def sed_inplace(filename, pattern, repl) -> None:
'''
Perform of in-place `sed` substitution: e.g.,
`sed -i -e 's/'${pattern}'/'${repl}' "${filename}"`.
See https://stackoverflow.com/a/31499114
'''
pattern_compiled = re.compile(pattern)
with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
with open(filename) as src_file:
for line in src_file:
tmp_file.write(pattern_compiled.sub(repl, line))
shutil.copystat(filename, tmp_file.name)
shutil.move(tmp_file.name, filename)

View File

@@ -34,8 +34,12 @@ STEAM_GAME_ID = 1649240
SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
PROCESS_NAME = "Returnal"
LOCAL_USER_SETTINGS = f"{os.getenv('LOCALAPPDATA')}\\Returnal\\Steam\\Saved\\Config\\WindowsNoEditor\\GameUserSettings.ini"
VIDEO_PATH = os.path.join(DEFAULT_STEAMAPPS_COMMON_PATH, "Returnal", "Returnal", "Content", "Movies")
LOCAL_USER_SETTINGS = os.path.join(
os.getenv('LOCALAPPDATA'), "Returnal", "Steam",
"Saved", "Config", "WindowsNoEditor", "GameUserSettings.ini"
)
VIDEO_PATH = os.path.join(
DEFAULT_STEAMAPPS_COMMON_PATH, "Returnal", "Returnal", "Content", "Movies")
user.FAILSAFE = False

View File

@@ -36,6 +36,7 @@ def remove_intro_videos(file_paths: list[str]) -> None:
def get_args() -> any:
"""Get command line arguments"""
parser = ArgumentParser()
parser.add_argument(
"--kerasHost", dest="keras_host", help="Host for Keras OCR service", required=True)