Add harnesses from latest MarkBench

tinytinaswonderland/README.md

@@ -0,0 +1,41 @@
+# Tiny Tina's Wonderland
+
+## Prerequisites
+
+- Python 3.10+
+- Tiny Tina's Wonderland installed.
+
+## Setup
+
+  1. Follow the setup instructions for the framework. If you have done so, all required python dependencies *should* be installed.
+  2. Install Tiny Tina's Wonderland from Epic Launcher.
+      1. Location **does** matter, the harness has to be aware of install location.
+
+## Configuration
+
+Below is an example use of this harness as a test in a benchmark configuration.
+
+```yaml
+...
+...
+tests:
+  - name: tinytinaswonderland
+    executable: "tinytinaswonderland.py"
+    process_name: "Wonderlands.exe"
+    asset_paths:
+      - 'harness/tinytinaswonderland/run'
+```
+
+__name__ : _(required)_ name of the test. This much match the name of a directory in the harness folder so the framework
+can find the executable and any supplementary files.
+
+__executable__ : _(required)_ the entry point to the test harness. In this case a python script.
+
+__process_name__ : _(required)_ The process name that should be the target for FPS recording (ex: PresentMon).
+
+__asset_paths__: _(optional)_ list of files to aggregate copies of after a successful test run. If a directory path is
+given, the contents are copied.
+
+## Common Issues
+1. "Login to Microsoft" modal pops up
+    - This game will not let you pass into the menu if you are not signed into Xbox. If you run this game at least once before running you can login then, or pre-login before running the harness.

tinytinaswonderland/cv2_utils.py

@@ -0,0 +1,201 @@
+import logging
+import os
+import time
+from enum import Enum
+
+import cv2
+import imutils
+import numpy as np
+import pyautogui as gui
+import pydirectinput as user
+
+DEFAULT_MATCH_THRESHOLD = 0.80  #
+DEFAULT_INTERVAL = 2  # seconds
+DEFAULT_TIMEOUT = 60  # seconds
+
+# path relative to script
+script_dir = os.path.dirname(os.path.realpath(__file__))
+images_dir = os.path.join(script_dir, "images")
+dir16x9 = os.path.join(images_dir, "16x9")
+dir16x10 = os.path.join(images_dir, "16x10")
+
+templates = {
+    "options": {
+        "16x10": cv2.imread(os.path.join(dir16x10, "options.png"), cv2.IMREAD_UNCHANGED),
+        "16x9": cv2.imread(os.path.join(dir16x9, "options.png"), cv2.IMREAD_UNCHANGED)
+    },
+    "benchmark": {
+        "16x10": cv2.imread(os.path.join(dir16x10, "benchmark.png"), cv2.IMREAD_UNCHANGED),
+        "16x9": cv2.imread(os.path.join(dir16x9, "benchmark.png"), cv2.IMREAD_UNCHANGED)
+    },
+    "start": {
+        "16x10": cv2.imread(os.path.join(dir16x10, "start.png"), cv2.IMREAD_UNCHANGED),
+        "16x9": cv2.imread(os.path.join(dir16x9, "start.png"), cv2.IMREAD_UNCHANGED)
+    }
+}
+
+
+def get_template(name, set="16x9"):
+    if set is None:
+        set = "16x9"
+    return templates[name][set]
+
+
+class ClickType(Enum):
+    SINGLE = 0  # uses .click()
+    DOUBLE = 1  # uses .doubleclick()
+    HARD = 2  # uses mouse.down() and mouse.up()
+    AUTO_GUI = 3  # uses pyautogui instead of pydirectinput
+
+
+def get_middle_of_rect(top_left_corner, height, width):
+    x = top_left_corner[0] + (width / 2)
+    y = top_left_corner[1] + (height / 2)
+    return int(x), int(y)  # round to avoid fractional pixels
+
+
+def click(top_left_corner, img):
+    click_loc = get_middle_of_rect(top_left_corner, img.shape[0], img.shape[1])
+    logging.info(f"Clicking {click_loc}")
+    user.click(click_loc[0], click_loc[1])
+
+
+def double_click(top_left_corner, img):
+    click_loc = get_middle_of_rect(top_left_corner, img.shape[0], img.shape[1])
+    logging.info(f"Double clicking {click_loc}")
+    user.doubleClick(click_loc[0], click_loc[1])
+
+
+def hard_click(top_left_corner, img):
+    click_loc = get_middle_of_rect(top_left_corner, img.shape[0], img.shape[1])
+    logging.info(f"Hard Clicking {click_loc}")
+    user.moveTo(click_loc[0], click_loc[1])
+    user.move(xOffset=5)
+    user.mouseDown()
+    time.sleep(0.2)
+    user.mouseUp()
+
+
+def autogui_click(top_left_corner, img):
+    click_loc = get_middle_of_rect(top_left_corner, img.shape[0], img.shape[1])
+    logging.info(f"Using AutoGui Clicking {click_loc}")
+    user.moveTo(click_loc[0], click_loc[1])
+    gui.click()
+
+
+def wait_and_click(template_name, name, click_type: ClickType = ClickType.SINGLE, timeout=DEFAULT_TIMEOUT,
+                   threshold=DEFAULT_MATCH_THRESHOLD):
+    logging.info(f"Waiting to find and click on {name}")
+    img, img_loc = wait_for_image_on_screen(template_name, timeout=timeout)
+    if click_type == ClickType.SINGLE:
+        click(img_loc, img)
+    elif click_type == ClickType.DOUBLE:
+        double_click(img_loc, img)
+    elif click_type == ClickType.HARD:
+        hard_click(img_loc, img)
+    elif click_type == ClickType.AUTO_GUI:
+        autogui_click(img_loc, img)
+    else:
+        raise ValueError("Unknown click type")
+
+
+class ImageNotFoundTimeout(Exception):
+    pass
+
+
+class ImageNotFound(Exception):
+    pass
+
+
+def gcd(a, b):
+    return a if b == 0 else gcd(b, a % b)
+
+
+def aspect_ratio(w, h):
+    denom = int(gcd(w, h))
+    x = int(w / denom)
+    y = int(h / denom)
+    if x == 8 and y == 5:
+        return "16x10"
+    elif x == 16 and y == 9:
+        return "16x9"
+
+
+def locate_on_screen(template_name, threshold=DEFAULT_MATCH_THRESHOLD, debug=0):
+    screen = gui.screenshot()
+    screen = np.array(
+        screen)  # pyautogui is using Pillow which is giving a format that must be adapted to work with opencv.
+    screen = cv2.cvtColor(screen, cv2.COLOR_RGB2BGR)
+    (h, w) = screen.shape[:2]
+    r = aspect_ratio(w, h)
+    needle = get_template(template_name, r)
+    return needle, locate_in_image(needle, screen, threshold=DEFAULT_MATCH_THRESHOLD, debug=0)
+
+
+# This approach was largely inspired by the article
+# https://pyimagesearch.com/2015/01/26/multi-scale-template-matching-using-python-opencv/
+def locate_in_image(needle, haystack, threshold=DEFAULT_MATCH_THRESHOLD, debug=0):
+    (tH, tW) = needle.shape[:2]
+
+    if debug:
+        cv2.imshow("Looking For", needle)
+        cv2.waitKey(0)
+
+    for scale in np.linspace(0.2, 1.0, 20)[::-1]:
+        # resize the image according to the scale, and keep track
+        # of the ratio of the resizing
+        resized = imutils.resize(haystack, width=int(haystack.shape[1] * scale), inter=cv2.INTER_NEAREST)
+        r = haystack.shape[1] / float(resized.shape[1])
+
+        # if the resized image is smaller than the template, then break
+        # from the loop
+        if resized.shape[0] < tH or resized.shape[1] < tW:
+            break
+
+        result = cv2.matchTemplate(resized, needle, cv2.TM_CCOEFF_NORMED)
+        (_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)
+
+        if debug:
+            # draw a bounding box around the detected region
+            # clone = np.dstack([edged, edged, edged])
+            cv2.rectangle(resized, (maxLoc[0], maxLoc[1]),
+                          (maxLoc[0] + tW, maxLoc[1] + tH), (0, 0, 255), 2)
+            cv2.imshow("Searching", resized)
+            cv2.waitKey(0)
+
+        if maxVal >= threshold:
+            found = (maxVal, maxLoc, r)
+
+            # unpack the bookkeeping variable and compute the (x, y) coordinates
+            # of the bounding box based on the resized ratio
+            (_, maxLoc, r) = found
+            (startX, startY) = (int(maxLoc[0] * r), int(maxLoc[1] * r))
+            (endX, endY) = (int((maxLoc[0] + tW) * r), int((maxLoc[1] + tH) * r))
+
+            if debug:
+                # draw a bounding box around the detected result and display the image
+                cv2.rectangle(haystack, (startX, startY), (endX, endY), (0, 0, 255), 2)
+                cv2.imshow("Found", haystack)
+                cv2.waitKey(0)
+
+            return startX, startY
+    raise ImageNotFound("Image not found on screen")
+
+
+def wait_for_image_on_screen(template_name, match_threshold=DEFAULT_MATCH_THRESHOLD, interval=DEFAULT_INTERVAL,
+                             timeout=DEFAULT_TIMEOUT):
+    """Function that will wait for an image to appear on screen. This function will check every
+     interval for a match that meets is greater than the match threshold. The function will raise
+     an error if the image is not found within the timeout given. Will return the location
+     of the image if found"""
+    t0 = time.time()
+    t1 = t0
+    while not t1 - t0 > timeout:
+        try:
+            img, loc = locate_on_screen(template_name, match_threshold)
+            return img, loc
+        except ImageNotFound:
+            pass
+        time.sleep(interval)
+        t1 = time.time()
+    raise ImageNotFoundTimeout("Could not find image on screen within timeout")

tinytinaswonderland/manifest.yaml

@@ -0,0 +1,13 @@
+friendly_name: "Tiny Tina's Wonderlands"
+executable: "tinytinaswonderland.py"
+process_name: "Wonderlands.exe"
+# default recording delay to reduce capturing menus during setup, this should be revisited every test bench as loading times may be different 
+recording_delay: 75
+asset_paths:
+  - "harness/tinytinaswonderland/run"
+options:
+  - name: preset
+    type: select
+    # disabling presets in favor of manually setting for now
+    # values: [verylow,low,medium,high,ultra,badass] 
+    values: [current] 

tinytinaswonderland/presets/GameUserSettings_badass.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=3
+sg.PostProcessQuality=3
+sg.TextureQuality=3
+sg.EffectsQuality=3
+sg.FoliageQuality=3
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=True
+bEnablePerObjectMotionBlur=True
+bEnableCAS=True
+GraphicsQualityMode=Badass
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=Ultra
+MaterialQuality=Ultra
+AntiAliasingModeIndex=2
+AnisotropicFilteringMode=SixteenX
+ShadowQualityMode=Ultra
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=Ultra
+EnvironmentDetailMode=Ultra
+TerrainMode=Ultra
+FoliageMode=Ultra
+VolumetricFogType=Ultra
+ScreenSpaceReflectionsMode=Ultra
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=Ultra
+AmbientOcclusionQualityMode=Ultra
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.750000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=3000
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/presets/GameUserSettings_high.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=2
+sg.PostProcessQuality=3
+sg.TextureQuality=2
+sg.EffectsQuality=3
+sg.FoliageQuality=2
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=False
+bEnablePerObjectMotionBlur=False
+bEnableCAS=True
+GraphicsQualityMode=High
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=High
+MaterialQuality=High
+AntiAliasingModeIndex=2
+AnisotropicFilteringMode=EightX
+ShadowQualityMode=High
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=High
+EnvironmentDetailMode=High
+TerrainMode=High
+FoliageMode=High
+VolumetricFogType=High
+ScreenSpaceReflectionsMode=High
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=High
+AmbientOcclusionQualityMode=High
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.500000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=2000
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/presets/GameUserSettings_low.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=1
+sg.PostProcessQuality=3
+sg.TextureQuality=1
+sg.EffectsQuality=3
+sg.FoliageQuality=0
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=False
+bEnablePerObjectMotionBlur=False
+bEnableCAS=False
+GraphicsQualityMode=Low
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=Medium
+MaterialQuality=Medium
+AntiAliasingModeIndex=1
+AnisotropicFilteringMode=TwoX
+ShadowQualityMode=Medium
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=Low
+EnvironmentDetailMode=Low
+TerrainMode=Medium
+FoliageMode=Low
+VolumetricFogType=Low
+ScreenSpaceReflectionsMode=Off
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=Low
+AmbientOcclusionQualityMode=Low
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.250000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=1200
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/presets/GameUserSettings_medium.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=1
+sg.PostProcessQuality=3
+sg.TextureQuality=2
+sg.EffectsQuality=3
+sg.FoliageQuality=1
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=False
+bEnablePerObjectMotionBlur=False
+bEnableCAS=False
+GraphicsQualityMode=Medium
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=High
+MaterialQuality=High
+AntiAliasingModeIndex=1
+AnisotropicFilteringMode=FourX
+ShadowQualityMode=Medium
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=Medium
+EnvironmentDetailMode=Medium
+TerrainMode=Medium
+FoliageMode=Medium
+VolumetricFogType=Medium
+ScreenSpaceReflectionsMode=Medium
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=Medium
+AmbientOcclusionQualityMode=Medium
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.500000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=2000
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/presets/GameUserSettings_ultra.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=3
+sg.PostProcessQuality=3
+sg.TextureQuality=3
+sg.EffectsQuality=3
+sg.FoliageQuality=3
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=True
+bEnablePerObjectMotionBlur=True
+bEnableCAS=True
+GraphicsQualityMode=Ultra
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=Ultra
+MaterialQuality=Ultra
+AntiAliasingModeIndex=2
+AnisotropicFilteringMode=SixteenX
+ShadowQualityMode=Ultra
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=Ultra
+EnvironmentDetailMode=Ultra
+TerrainMode=Ultra
+FoliageMode=Ultra
+VolumetricFogType=High
+ScreenSpaceReflectionsMode=High
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=Ultra
+AmbientOcclusionQualityMode=Ultra
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.750000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=3000
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/presets/GameUserSettings_verylow.ini

@@ -0,0 +1,100 @@
+[ScalabilityGroups]
+sg.ResolutionQuality=100.000000
+sg.ViewDistanceQuality=3
+sg.AntiAliasingQuality=3
+sg.ShadowQuality=0
+sg.PostProcessQuality=3
+sg.TextureQuality=0
+sg.EffectsQuality=3
+sg.FoliageQuality=0
+
+[/Script/OakGame.OakGameUserSettings]
+ResolutionScale=Scale_100
+FrameRateLimitMode=Unlimited
+CustomFrameRateLimit=144
+bEnableCameraMotionBlur=False
+bEnablePerObjectMotionBlur=False
+bEnableCAS=False
+GraphicsQualityMode=VeryLow
+RecommendedGraphicsQualityMode=High
+TextureStreamingQuality=Low
+MaterialQuality=Low
+AntiAliasingModeIndex=0
+AnisotropicFilteringMode=Trilinear
+ShadowQualityMode=Low
+PerformanceStatsLevel=0
+EnvironmentTextureDetailMode=Deprecated
+DrawDistanceMode=Low
+EnvironmentDetailMode=Low
+TerrainMode=Low
+FoliageMode=Low
+VolumetricFogType=Low
+ScreenSpaceReflectionsMode=Off
+CharacterTextureDetailMode=Deprecated
+CharacterDetailMode=Low
+AmbientOcclusionQualityMode=Off
+AMD_FSR2_Preset=Off
+AMD_FSR2_Sharpness_Preset=0.000000
+EdgeDetectionTexelOffset=-1.000000
+BenchmarkDataOutputMode=DetailedCSV
+UISafeFrameScaling=1.000000
+HDRBrightness=400.000000
+Brightness=50.000000
+BlackLevel=0.000000
+HdrUIBrightness=4.000000
+LastBenchmarkDateTime=
+LastBenchmarkDataOutputPath=
+LastBenchmarkAvgFPS=0.000000
+LastBenchmarkAvgFrameTime=0.000000
+LastBenchmarkAvgCPU=0.000000
+LastBenchmarkAvgGPU=0.000000
+LastBenchmarkLoadTimeSeconds=0.000000
+AnalyticsTag=
+TextureStreamingViewBoostScale=0.000000
+TextureStreamingViewBoostScaleOverride=-1.000000
+TextureStreamingAdaptiveBoostScale=0.000000
+TextureStreamingAdaptiveBoostScaleOverride=-1.000000
+TextureStreamingPoolSizeMB=800
+TextureStreamingPoolSizeMBOverride=-1
+bHasPerformedDirectXBetaUpgrade=False
+bUseVSync=False
+bUseDynamicResolution=False
+ResolutionSizeX=1920
+ResolutionSizeY=1080
+LastUserConfirmedResolutionSizeX=2560
+LastUserConfirmedResolutionSizeY=1440
+WindowPosX=-1
+WindowPosY=-1
+FullscreenMode=0
+LastConfirmedFullscreenMode=2
+PreferredFullscreenMode=1
+Version=8
+AudioQualityLevel=0
+FrameRateLimit=0.000000
+DesiredScreenWidth=1280
+DesiredScreenHeight=720
+LastUserConfirmedDesiredScreenWidth=1280
+LastUserConfirmedDesiredScreenHeight=720
+LastRecommendedScreenWidth=-1.000000
+LastRecommendedScreenHeight=-1.000000
+PreferredMonitor=SAM7176
+PreferredMonitorDeviceName=
+bPrimaryIsPreferredMonitor=True
+LastCPUBenchmarkResult=1.000000
+LastGPUBenchmarkResult=1.000000
+LastGPUBenchmarkMultiplier=1.000000
+LastBenchmarkResultCPUName=
+LastBenchmarkResultGPUName=
+bUseHDRDisplayOutput=False
+HDRDisplayOutputNits=1000
+bUsePerformanceMode=False
+PreferredGraphicsAPI=DX12
+PreferredRefreshRate=0
+
+[/Script/Engine.GameUserSettings]
+bUseDesiredScreenHeight=False
+
+[ShaderPipelineCache.CacheFile]
+LastOpened=
+
+

tinytinaswonderland/pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "tinytinaswonderland"
+version = "0.1.0"
+description = ""
+authors = ["Nikolas Harris <nikolas@linusmediagroup.com"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+PyAutoGUI = "^0.9.53"
+PyDirectInput = "^1.0.4"
+opencv-python = "^4.5.5"
+Pillow = "^9.1.1"
+psutil = "^5.9.1"
+PyYAML = "^6.0"
+imutils = "^0.5.4"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tinytinaswonderland/template_test.py

@@ -0,0 +1,15 @@
+from cv2_utils import *
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+images_dir = os.path.join(script_dir, "images")
+test_images_dir = os.path.join(images_dir, "tests")
+
+test_menus = {
+    "mainmenu": cv2.imread(os.path.join(test_images_dir, "main_menu_2k.png"), cv2.IMREAD_UNCHANGED),
+    "settings_menu": cv2.imread(os.path.join(test_images_dir, "settings_menu_2k.png"), cv2.IMREAD_UNCHANGED),
+    "benchmark_menu": cv2.imread(os.path.join(test_images_dir, "benchmark_menu_2k.png"), cv2.IMREAD_UNCHANGED),
+    "1080menu": cv2.imread(os.path.join(test_images_dir, "menu_1080p_27inch.png"), cv2.IMREAD_UNCHANGED)
+}
+
+found = locate_in_image(get_template('options'), test_menus['1080menu'], threshold=0.8, debug=1)
+print(found)

tinytinaswonderland/tinytinaswonderland.py

@@ -0,0 +1,86 @@
+import logging
+from subprocess import Popen
+import sys
+
+from cv2_utils import *
+from utils import read_resolution, try_install_paths, get_local_drives
+
+sys.path.insert(1, os.path.join(sys.path[0], '..'))
+
+from harness_utils.logging import *
+from harness_utils.process import terminate_processes
+from harness_utils.steam import get_run_game_id_command, DEFAULT_EXECUTABLE_PATH as steam_path 
+
+SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+LOG_DIRECTORY = os.path.join(SCRIPT_DIRECTORY, "run")
+STEAM_GAME_ID = 1286680
+EXECUTABLE = "Wonderlands.exe"
+
+
+def start_game() -> any:
+    game_process = None
+    try:
+        exec_path = try_install_paths(get_local_drives())
+        game_process = Popen(exec_path)
+    except ValueError:
+        logging.info("Could not find executable, trying a steam launch")
+        pass
+
+    if game_process is None:
+        steam_run_arg = get_run_game_id_command(STEAM_GAME_ID)
+        logging.info(steam_path + " " + steam_run_arg)
+        game_process = Popen([steam_path, steam_run_arg])
+
+
+def run_benchmark():
+    start_game()
+    
+    t1 = time.time()
+
+    # wait for menu to load
+    time.sleep(30)
+
+    wait_and_click("options", "options menu", ClickType.HARD)
+    wait_and_click("benchmark", "benchmark menu", ClickType.HARD)
+    wait_and_click("start", "start benchmark", ClickType.HARD)
+    t2 = time.time()
+    logging.info(f"Harness setup took {round((t2 - t1), 2)} seconds")
+
+    start_time = time.time()
+
+    time.sleep(124)
+    wait_for_image_on_screen("options", 0.8, 2, 60)
+
+    end_time = time.time()
+    logging.info(f"Benchark took {round((end_time - start_time), 2)} seconds")
+    terminate_processes("Wonderlands")
+    return start_time, end_time
+
+
+setup_log_directory(LOG_DIRECTORY)
+
+logging.basicConfig(filename=f'{LOG_DIRECTORY}/harness.log',
+                    format=DEFAULT_LOGGING_FORMAT,
+                    datefmt=DEFAULT_DATE_FORMAT,
+                    level=logging.DEBUG)
+console = logging.StreamHandler()
+formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
+console.setFormatter(formatter)
+logging.getLogger('').addHandler(console)
+
+try:
+    start_time, end_time = run_benchmark()
+    height, width = read_resolution()
+    result = {
+        "resolution": format_resolution(width, height),
+        "graphics_preset": "current",
+        "start_time": seconds_to_milliseconds(start_time),
+        "end_time": seconds_to_milliseconds(end_time)
+    }
+
+    write_report_json(LOG_DIRECTORY, "report.json", result)
+except Exception as e:
+    logging.error("Something went wrong running the benchmark!")
+    logging.exception(e)
+    terminate_processes("Wonderlands")
+    exit(1)

tinytinaswonderland/utils.py

@@ -0,0 +1,72 @@
+import win32api
+import win32file
+import winreg
+import os
+import logging
+import re
+
+EXECUTABLE = "Wonderlands.exe"
+
+
+def get_documents_path() -> str:
+    SHELL_FOLDER_KEY = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders'
+    try:
+        root_handle = winreg.ConnectRegistry(None, winreg.HKEY_CURRENT_USER)
+        shell_folders_handle = winreg.OpenKeyEx(root_handle, SHELL_FOLDER_KEY)
+        personal_path_key = winreg.QueryValueEx(shell_folders_handle, 'Personal')
+        return personal_path_key[0]
+    finally:
+        root_handle.Close()
+
+
+def valid_filepath(path: str) -> bool:
+    """Validate given path is valid and leads to an existing file. A directory
+    will throw an error, path must be a file"""
+    if path is None or len(path.strip()) <= 0:
+        return False
+    if os.path.isdir(path) is True:
+        return False
+    return os.path.isfile(path)
+
+
+def get_local_drives() -> list[str]:
+    """Returns a list containing letters from local drives"""
+    drive_list = win32api.GetLogicalDriveStrings()
+    drive_list = drive_list.split("\x00")[0:-1]  # the last element is ""
+    list_local_drives = []
+    for letter in drive_list:
+        if win32file.GetDriveType(letter) == win32file.DRIVE_FIXED:
+            list_local_drives.append(letter)
+    return list_local_drives 
+
+
+def try_install_paths(drives) -> str:
+    for drive_letter in drives:
+        try_me = f"{drive_letter}Program Files\\Epic Games\\TinyTinasWonderlands\\OakGame\\Binaries\\Win64\\{EXECUTABLE}"
+        try_me2 = f"{drive_letter}Epic Games\\TinyTinasWonderlands\\OakGame\\Binaries\\Win64\\{EXECUTABLE}"
+        if valid_filepath(try_me):
+            return try_me
+        if valid_filepath(try_me2):
+            return try_me2
+    raise ValueError("Cannot find the install path for the game!")
+
+
+def read_resolution() -> tuple[int]:
+    dest = f"{get_documents_path()}\\My Games\\Tiny Tina's Wonderlands\\Saved\\Config\\WindowsNoEditor\\GameUserSettings.ini"
+    hpattern = re.compile(r"ResolutionSizeY=(\d*)")
+    wpattern = re.compile(r"ResolutionSizeX=(\d*)")
+    h = w = 0
+    with open(dest) as fp:
+        lines = fp.readlines()
+        for line in lines:
+            result = hpattern.match(line)
+            if result is not None:
+                h = result.group(1)
+
+            result2 = wpattern.match(line)
+            if result2 is not None:
+                w = result2.group(1)
+            if int(h) > 0 and int(w) > 0:
+                break
+    logging.info(f"Current resolution is {h}x{w}")
+    return (h, w)