diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs
index e39bb9534a..773feea70a 100644
--- a/wgpu-core/src/device/queue.rs
+++ b/wgpu-core/src/device/queue.rs
@@ -559,9 +559,15 @@ impl<G: GlobalIdentityHandlerFactory> Global<G> {
                 }
             }
 
-            let (buffer, transition) = trackers
-                .buffers
-                .use_replace(&*buffer_guard, buffer_id, (), BufferUse::COPY_DST)
+            // Don't do use_replace since the buffer may already no longer have a ref_count.
+            // However, we *know* that it is currently in use, so the tracker must already know about it.
+            let transition = trackers.buffers.change_replace_tracked(
+                id::Valid(buffer_id),
+                (),
+                BufferUse::COPY_DST,
+            );
+            let buffer = buffer_guard
+                .get(buffer_id)
                 .map_err(|_| QueueSubmitError::DestroyedBuffer(buffer_id))?;
             let &(ref buffer_raw, _) = buffer
                 .raw
diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs
index 0a9f9a3158..758a3004a4 100644
--- a/wgpu-core/src/track/mod.rs
+++ b/wgpu-core/src/track/mod.rs
@@ -315,6 +315,18 @@ impl<S: ResourceState> ResourceTracker<S> {
         }
     }
 
+    fn get<'a>(
+        self_backend: wgt::Backend,
+        map: &'a mut FastHashMap<Index, Resource<S>>,
+        id: Valid<S::Id>,
+    ) -> &'a mut Resource<S> {
+        let (index, epoch, backend) = id.0.unzip();
+        debug_assert_eq!(self_backend, backend);
+        let e = map.get_mut(&index).unwrap();
+        assert_eq!(e.epoch, epoch);
+        e
+    }
+
     /// Extend the usage of a specified resource.
     ///
     /// Returns conflicting transition as an error.
@@ -345,6 +357,21 @@ impl<S: ResourceState> ResourceTracker<S> {
         self.temp.drain(..)
     }
 
+    /// Replace the usage of a specified already tracked resource.
+    /// (panics if the resource is not yet tracked)
+    pub(crate) fn change_replace_tracked(
+        &mut self,
+        id: Valid<S::Id>,
+        selector: S::Selector,
+        usage: S::Usage,
+    ) -> Drain<PendingTransition<S>> {
+        let res = Self::get(self.backend, &mut self.map, id);
+        res.state
+            .change(id, selector, usage, Some(&mut self.temp))
+            .ok();
+        self.temp.drain(..)
+    }
+
     /// Turn the tracking from the "expand" mode into the "replace" one,
     /// installing the selected usage as the "first".
     /// This is a special operation only used by the render pass attachments.