diff --git a/invokeai/frontend/web/src/features/controlLayers/components/SimpleSession/context.tsx b/invokeai/frontend/web/src/features/controlLayers/components/SimpleSession/context.tsx
index 3f7a608b7b..f89d76fe8d 100644
--- a/invokeai/frontend/web/src/features/controlLayers/components/SimpleSession/context.tsx
+++ b/invokeai/frontend/web/src/features/controlLayers/components/SimpleSession/context.tsx
@@ -272,8 +272,9 @@ export const CanvasSessionContextProvider = memo(
         if (data.destination !== session.id) {
           return;
         }
+        const isFirstProgressImage = !$progressData.get()[data.item_id]?.progressImage && !!data.image;
         setProgress($progressData, data);
-        if ($autoSwitch.get() === 'first_progress') {
+        if ($autoSwitch.get() === 'first_progress' && isFirstProgressImage) {
           $selectedItemId.set(data.item_id);
         }
       };