From 9e1384dbb0bab76f024fc76849238fdabf5a3ce9 Mon Sep 17 00:00:00 2001 From: Wumpf Date: Mon, 23 Aug 2021 04:46:13 +0200 Subject: [PATCH] Memory init tracker refactor (#1839) * split out init_tracker module with explicit buffer init tracking types * Added shortcut methods for checked init action handling Fixed a few unchecked sites in the process * fix clippy warnings --- wgpu-core/src/binding_model.rs | 4 +- wgpu-core/src/command/bundle.rs | 58 ++++----- wgpu-core/src/command/clear.rs | 20 ++- wgpu-core/src/command/compute.rs | 25 ++-- wgpu-core/src/command/mod.rs | 6 +- wgpu-core/src/command/query.rs | 19 ++- wgpu-core/src/command/render.rs | 85 +++++-------- wgpu-core/src/command/transfer.rs | 74 +++++------ wgpu-core/src/device/mod.rs | 20 +-- wgpu-core/src/init_tracker/buffer.rs | 33 +++++ .../mod.rs} | 115 ++++++++++-------- wgpu-core/src/lib.rs | 2 +- wgpu-core/src/resource.rs | 4 +- 13 files changed, 220 insertions(+), 245 deletions(-) create mode 100644 wgpu-core/src/init_tracker/buffer.rs rename wgpu-core/src/{memory_init_tracker.rs => init_tracker/mod.rs} (67%) diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index 86db068164..e9c85dde46 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -3,7 +3,7 @@ use crate::{ error::{ErrorFormatter, PrettyError}, hub::Resource, id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid}, - memory_init_tracker::MemoryInitTrackerAction, + init_tracker::BufferInitTrackerAction, track::{TrackerSet, UsageConflict, DUMMY_SELECTOR}, validation::{MissingBufferUsageError, MissingTextureUsageError}, FastHashMap, Label, LifeGuard, MultiRefCount, Stored, @@ -671,7 +671,7 @@ pub struct BindGroup { pub(crate) layout_id: Valid, pub(crate) life_guard: LifeGuard, pub(crate) used: TrackerSet, - pub(crate) used_buffer_ranges: Vec>, + pub(crate) used_buffer_ranges: Vec, pub(crate) dynamic_binding_info: Vec, } diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 29ab65eaaf..fd28ca16be 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -46,7 +46,7 @@ use crate::{ error::{ErrorFormatter, PrettyError}, hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Resource, Storage, Token}, id, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::{BufferInitTrackerAction, MemoryInitKind}, pipeline::PipelineFlags, track::{TrackerSet, UsageConflict}, validation::check_buffer_usage, @@ -287,11 +287,11 @@ impl RenderBundleEncoder { Some(s) => offset + s.get(), None => buffer.size, }; - buffer_memory_init_actions.push(MemoryInitTrackerAction { - id: buffer_id, - range: offset..end, - kind: MemoryInitKind::NeedsInitializedMemory, - }); + buffer_memory_init_actions.extend(buffer.initialization_status.create_action( + buffer_id, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); state.index.set_format(index_format); state.index.set_buffer(buffer_id, offset..end); } @@ -314,11 +314,11 @@ impl RenderBundleEncoder { Some(s) => offset + s.get(), None => buffer.size, }; - buffer_memory_init_actions.push(MemoryInitTrackerAction { - id: buffer_id, - range: offset..end, - kind: MemoryInitKind::NeedsInitializedMemory, - }); + buffer_memory_init_actions.extend(buffer.initialization_status.create_action( + buffer_id, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + )); state.vertex[slot as usize].set_buffer(buffer_id, offset..end); } RenderCommand::SetPushConstant { @@ -435,18 +435,11 @@ impl RenderBundleEncoder { check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; - buffer_memory_init_actions.extend( - buffer - .initialization_status - .check( - offset..(offset + mem::size_of::() as u64), - ) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), - ); + buffer_memory_init_actions.extend(buffer.initialization_status.create_action( + buffer_id, + offset..(offset + mem::size_of::() as u64), + MemoryInitKind::NeedsInitializedMemory, + )); commands.extend(state.flush_vertices()); commands.extend(state.flush_binds()); @@ -476,18 +469,11 @@ impl RenderBundleEncoder { check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; - buffer_memory_init_actions.extend( - buffer - .initialization_status - .check( - offset..(offset + mem::size_of::() as u64), - ) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), - ); + buffer_memory_init_actions.extend(buffer.initialization_status.create_action( + buffer_id, + offset..(offset + mem::size_of::() as u64), + MemoryInitKind::NeedsInitializedMemory, + )); commands.extend(state.index.flush()); commands.extend(state.flush_vertices()); @@ -586,7 +572,7 @@ pub struct RenderBundle { pub(super) is_ds_read_only: bool, pub(crate) device_id: Stored, pub(crate) used: TrackerSet, - pub(super) buffer_memory_init_actions: Vec>, + pub(super) buffer_memory_init_actions: Vec, pub(super) context: RenderPassContext, pub(crate) life_guard: LifeGuard, } diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 129f0e0297..e2df916f6a 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -6,7 +6,7 @@ use crate::{ command::CommandBuffer, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Token}, id::{BufferId, CommandEncoderId, TextureId}, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::MemoryInitKind, track::TextureSelector, }; @@ -127,17 +127,13 @@ impl Global { } // Mark dest as initialized. - cmd_buf.buffer_memory_init_actions.extend( - dst_buffer - .initialization_status - .check(offset..end) - .map(|range| MemoryInitTrackerAction { - id: dst, - range, - kind: MemoryInitKind::ImplicitlyInitialized, - }), - ); - + cmd_buf + .buffer_memory_init_actions + .extend(dst_buffer.initialization_status.create_action( + dst, + offset..end, + MemoryInitKind::ImplicitlyInitialized, + )); // actual hal barrier & operation let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); let cmd_buf_raw = cmd_buf.encoder.open(); diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 23f080cc13..970e5d885e 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -9,7 +9,7 @@ use crate::{ error::{ErrorFormatter, PrettyError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::MemoryInitKind, resource::{Buffer, Texture}, track::{StatefulTrackerSubset, TrackerSet, UsageConflict, UseExtendError}, validation::{check_buffer_usage, MissingBufferUsageError}, @@ -366,19 +366,11 @@ impl Global { cmd_buf.buffer_memory_init_actions.extend( bind_group.used_buffer_ranges.iter().filter_map( |action| match buffer_guard.get(action.id) { - Ok(buffer) => buffer - .initialization_status - .check(action.range.clone()) - .map(|range| MemoryInitTrackerAction { - id: action.id, - range, - kind: action.kind, - }), + Ok(buffer) => buffer.initialization_status.check_action(action), Err(_) => None, }, ), ); - let pipeline_layout_id = state.binder.pipeline_layout_id; let entries = state.binder.assign_group( index as usize, @@ -565,14 +557,11 @@ impl Global { let stride = 3 * 4; // 3 integers, x/y/z group size cmd_buf.buffer_memory_init_actions.extend( - indirect_buffer - .initialization_status - .check(offset..(offset + stride)) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + indirect_buffer.initialization_status.create_action( + buffer_id, + offset..(offset + stride), + MemoryInitKind::NeedsInitializedMemory, + ), ); state diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index a5923953e3..df7c64d76a 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -22,7 +22,7 @@ use crate::FastHashMap; use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::{BufferInitTrackerAction, MemoryInitKind}, resource::{Buffer, Texture}, track::{BufferState, ResourceTracker, TextureState, TrackerSet}, Label, Stored, @@ -71,7 +71,7 @@ pub struct BakedCommands { pub(crate) encoder: A::CommandEncoder, pub(crate) list: Vec, pub(crate) trackers: TrackerSet, - buffer_memory_init_actions: Vec>, + buffer_memory_init_actions: Vec, } pub(crate) struct DestroyedBufferError(pub id::BufferId); @@ -158,7 +158,7 @@ pub struct CommandBuffer { status: CommandEncoderStatus, pub(crate) device_id: Stored, pub(crate) trackers: TrackerSet, - buffer_memory_init_actions: Vec>, + buffer_memory_init_actions: Vec, limits: wgt::Limits, support_fill_buffer_texture: bool, #[cfg(feature = "trace")] diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index bdcd93cb1d..ea93322a1c 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -6,7 +6,7 @@ use crate::{ command::{CommandBuffer, CommandEncoderError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id::{self, Id, TypedId}, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::MemoryInitKind, resource::QuerySet, track::UseExtendError, Epoch, FastHashMap, Index, @@ -392,16 +392,13 @@ impl Global { .into()); } - cmd_buf.buffer_memory_init_actions.extend( - dst_buffer - .initialization_status - .check(buffer_start_offset..buffer_end_offset) - .map(|range| MemoryInitTrackerAction { - id: destination, - range, - kind: MemoryInitKind::ImplicitlyInitialized, - }), - ); + cmd_buf + .buffer_memory_init_actions + .extend(dst_buffer.initialization_status.create_action( + destination, + buffer_start_offset..buffer_end_offset, + MemoryInitKind::ImplicitlyInitialized, + )); unsafe { raw_encoder.transition_buffers(dst_barrier); diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 76813896a1..ecafba64a3 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -13,7 +13,7 @@ use crate::{ error::{ErrorFormatter, PrettyError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::MemoryInitKind, pipeline::PipelineFlags, resource::{Texture, TextureView}, track::{StatefulTrackerSubset, TextureSelector, UsageConflict}, @@ -889,14 +889,7 @@ impl Global { cmd_buf.buffer_memory_init_actions.extend( bind_group.used_buffer_ranges.iter().filter_map(|action| { match buffer_guard.get(action.id) { - Ok(buffer) => buffer - .initialization_status - .check(action.range.clone()) - .map(|range| MemoryInitTrackerAction { - id: action.id, - range, - kind: action.kind, - }), + Ok(buffer) => buffer.initialization_status.check_action(action), Err(_) => None, } }), @@ -1067,14 +1060,11 @@ impl Global { state.index.update_limit(); cmd_buf.buffer_memory_init_actions.extend( - buffer - .initialization_status - .check(offset..end) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + buffer.initialization_status.create_action( + buffer_id, + offset..end, + MemoryInitKind::NeedsInitializedMemory, + ), ); let bb = hal::BufferBinding { @@ -1122,14 +1112,11 @@ impl Global { vertex_state.bound = true; cmd_buf.buffer_memory_init_actions.extend( - buffer - .initialization_status - .check(offset..(offset + vertex_state.total_size)) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + buffer.initialization_status.create_action( + buffer_id, + offset..(offset + vertex_state.total_size), + MemoryInitKind::NeedsInitializedMemory, + ), ); let bb = hal::BufferBinding { @@ -1382,14 +1369,11 @@ impl Global { } cmd_buf.buffer_memory_init_actions.extend( - indirect_buffer - .initialization_status - .check(offset..end_offset) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + indirect_buffer.initialization_status.create_action( + buffer_id, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), ); match indexed { @@ -1472,14 +1456,11 @@ impl Global { .map_pass_err(scope); } cmd_buf.buffer_memory_init_actions.extend( - indirect_buffer - .initialization_status - .check(offset..end_offset) - .map(|range| MemoryInitTrackerAction { - id: buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + indirect_buffer.initialization_status.create_action( + buffer_id, + offset..end_offset, + MemoryInitKind::NeedsInitializedMemory, + ), ); let begin_count_offset = count_buffer_offset; @@ -1493,14 +1474,11 @@ impl Global { .map_pass_err(scope); } cmd_buf.buffer_memory_init_actions.extend( - count_buffer - .initialization_status - .check(count_buffer_offset..end_count_offset) - .map(|range| MemoryInitTrackerAction { - id: count_buffer_id, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), + count_buffer.initialization_status.create_action( + count_buffer_id, + count_buffer_offset..end_count_offset, + MemoryInitKind::NeedsInitializedMemory, + ), ); match indexed { @@ -1642,14 +1620,7 @@ impl Global { .buffer_memory_init_actions .iter() .filter_map(|action| match buffer_guard.get(action.id) { - Ok(buffer) => buffer - .initialization_status - .check(action.range.clone()) - .map(|range| MemoryInitTrackerAction { - id: action.id, - range, - kind: action.kind, - }), + Ok(buffer) => buffer.initialization_status.check_action(action), Err(_) => None, }), ); diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index c5db24cf68..1c9980d0c6 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -6,7 +6,7 @@ use crate::{ error::{ErrorFormatter, PrettyError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id::{BufferId, CommandEncoderId, TextureId}, - memory_init_tracker::{MemoryInitKind, MemoryInitTrackerAction}, + init_tracker::MemoryInitKind, resource::{Texture, TextureErrorDimension}, track::TextureSelector, }; @@ -462,26 +462,20 @@ impl Global { } // Make sure source is initialized memory and mark dest as initialized. - cmd_buf.buffer_memory_init_actions.extend( - dst_buffer - .initialization_status - .check(destination_offset..(destination_offset + size)) - .map(|range| MemoryInitTrackerAction { - id: destination, - range, - kind: MemoryInitKind::ImplicitlyInitialized, - }), - ); - cmd_buf.buffer_memory_init_actions.extend( - src_buffer - .initialization_status - .check(source_offset..(source_offset + size)) - .map(|range| MemoryInitTrackerAction { - id: source, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), - ); + cmd_buf + .buffer_memory_init_actions + .extend(dst_buffer.initialization_status.create_action( + destination, + destination_offset..(destination_offset + size), + MemoryInitKind::ImplicitlyInitialized, + )); + cmd_buf + .buffer_memory_init_actions + .extend(src_buffer.initialization_status.create_action( + source, + source_offset..(source_offset + size), + MemoryInitKind::NeedsInitializedMemory, + )); let region = hal::BufferCopy { src_offset: source_offset, @@ -582,16 +576,13 @@ impl Global { true, )?; - cmd_buf.buffer_memory_init_actions.extend( - src_buffer - .initialization_status - .check(source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy)) - .map(|range| MemoryInitTrackerAction { - id: source.buffer, - range, - kind: MemoryInitKind::NeedsInitializedMemory, - }), - ); + cmd_buf + .buffer_memory_init_actions + .extend(src_buffer.initialization_status.create_action( + source.buffer, + source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy), + MemoryInitKind::NeedsInitializedMemory, + )); if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format) { return Err( @@ -712,19 +703,14 @@ impl Global { ); } - cmd_buf.buffer_memory_init_actions.extend( - dst_buffer - .initialization_status - .check( - destination.layout.offset - ..(destination.layout.offset + required_buffer_bytes_in_copy), - ) - .map(|range| MemoryInitTrackerAction { - id: destination.buffer, - range, - kind: MemoryInitKind::ImplicitlyInitialized, - }), - ); + cmd_buf + .buffer_memory_init_actions + .extend(dst_buffer.initialization_status.create_action( + destination.buffer, + destination.layout.offset + ..(destination.layout.offset + required_buffer_bytes_in_copy), + MemoryInitKind::ImplicitlyInitialized, + )); let regions = (0..array_layer_count).map(|rel_array_layer| { let mut texture_base = src_base.clone(); diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 593830f1ce..5b4a004967 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -2,9 +2,9 @@ use crate::{ binding_model, command, conv, device::life::WaitIdleError, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token}, - id, instance, - memory_init_tracker::{MemoryInitKind, MemoryInitTracker, MemoryInitTrackerAction}, - pipeline, present, resource, + id, + init_tracker::{BufferInitTracker, BufferInitTrackerAction, MemoryInitKind}, + instance, pipeline, present, resource, track::{BufferState, TextureSelector, TextureState, TrackerSet, UsageConflict}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, Stored, SubmissionIndex, @@ -527,7 +527,7 @@ impl Device { }, usage: desc.usage, size: desc.size, - initialization_status: MemoryInitTracker::new(desc.size), + initialization_status: BufferInitTracker::new(desc.size), sync_mapped_writes: None, map_state: resource::BufferMapState::Idle, life_guard: LifeGuard::new(desc.label.borrow_or_default()), @@ -1201,7 +1201,7 @@ impl Device { bb: &binding_model::BufferBinding, binding: u32, decl: &wgt::BindGroupLayoutEntry, - used_buffer_ranges: &mut Vec>, + used_buffer_ranges: &mut Vec, dynamic_binding_info: &mut Vec, used: &mut TrackerSet, storage: &'a Storage, id::BufferId>, @@ -1297,11 +1297,11 @@ impl Device { return Err(Error::BindingZeroSize(bb.buffer_id)); } - used_buffer_ranges.push(MemoryInitTrackerAction { - id: bb.buffer_id, - range: bb.offset..(bb.offset + bind_size), - kind: MemoryInitKind::NeedsInitializedMemory, - }); + used_buffer_ranges.extend(buffer.initialization_status.create_action( + bb.buffer_id, + bb.offset..(bb.offset + bind_size), + MemoryInitKind::NeedsInitializedMemory, + )); Ok(hal::BufferBinding { buffer: raw_buffer, diff --git a/wgpu-core/src/init_tracker/buffer.rs b/wgpu-core/src/init_tracker/buffer.rs new file mode 100644 index 0000000000..541d0aefd0 --- /dev/null +++ b/wgpu-core/src/init_tracker/buffer.rs @@ -0,0 +1,33 @@ +use super::{InitTracker, MemoryInitKind}; +use crate::id::BufferId; +use std::ops::Range; + +#[derive(Debug, Clone)] +pub(crate) struct BufferInitTrackerAction { + pub id: BufferId, + pub range: Range, + pub kind: MemoryInitKind, +} + +pub(crate) type BufferInitTracker = InitTracker; + +impl BufferInitTracker { + /// Checks if an action has/requires any effect on the initialization status and shrinks its range if possible. + pub(crate) fn check_action( + &self, + action: &BufferInitTrackerAction, + ) -> Option { + self.create_action(action.id, action.range.clone(), action.kind) + } + + /// Creates an action if it would have any effect on the initialization status and shrinks the range if possible. + pub(crate) fn create_action( + &self, + id: BufferId, + query_range: Range, + kind: MemoryInitKind, + ) -> Option { + self.check(query_range) + .map(|range| BufferInitTrackerAction { id, range, kind }) + } +} diff --git a/wgpu-core/src/memory_init_tracker.rs b/wgpu-core/src/init_tracker/mod.rs similarity index 67% rename from wgpu-core/src/memory_init_tracker.rs rename to wgpu-core/src/init_tracker/mod.rs index 771498f224..dd6a6cc32d 100644 --- a/wgpu-core/src/memory_init_tracker.rs +++ b/wgpu-core/src/init_tracker/mod.rs @@ -1,4 +1,26 @@ -use std::ops::Range; +// WebGPU specification requires all texture & buffer memory to be zero initialized on first read. +// To avoid unnecessary inits, we track the initialization status of every resource and perform inits lazily. +// +// The granularity is different for buffers and textures: +// * Buffer: Byte granularity to support usecases with large, partially bound buffers well. +// * Texture: Mip-level per layer. I.e. a 2D surface is either completely initialized or not, subrects are not tracked. +// +// Every use of a buffer/texture generates a InitTrackerAction which are recorded and later resolved at queue submit by merging them with the current state and each other in execution order. +// It is important to note that from the point of view of the memory init system there are two kind of writes: +// * Full writes: +// Any kind of memcpy operation. These cause a `MemoryInitKind.ImplicitlyInitialized` action. +// * (Potentially) partial writes: +// E.g. write use in a Shader. The system is not able to determine if a resource is fully initialized afterwards but is no longer allowed to perform any clears, +// therefore this leads to a `MemoryInitKind.ImplicitlyInitialized` action, exactly like a read would. + +use smallvec::SmallVec; +use std::{iter, ops::Range}; + +mod buffer; +//mod texture; + +pub(crate) use buffer::{BufferInitTracker, BufferInitTrackerAction}; +//pub(crate) use texture::{TextureInitRange, TextureInitTracker, TextureInitTrackerAction}; #[derive(Debug, Clone, Copy)] pub(crate) enum MemoryInitKind { @@ -6,31 +28,36 @@ pub(crate) enum MemoryInitKind { ImplicitlyInitialized, // The memory range is going to be read, therefore needs to ensure prior initialization. NeedsInitializedMemory, + // The memory going to be discarded and regarded therefore regarded uninitialized. + // TODO: This is tricky to implement: Discards needs to be resolved within AND between command buffers! + // Being able to do this would be quite nice because then we could mark any resource uninitialized at any point in time. + // Practically speaking however, discard can only ever happen for single rendertarget surfaces. + // Considering this, this could potentially be implemented differently since we don't really need to care about ranges of memory. + //DiscardMemory, } -#[derive(Debug, Clone)] -pub(crate) struct MemoryInitTrackerAction { - pub(crate) id: ResourceId, - pub(crate) range: Range, - pub(crate) kind: MemoryInitKind, -} +// Most of the time a resource is either fully uninitialized (one element) or initialized (zero elements). +type UninitializedRangeVec = SmallVec<[Range; 1]>; /// Tracks initialization status of a linear range from 0..size -#[derive(Debug)] -pub(crate) struct MemoryInitTracker { +#[derive(Debug, Clone)] +pub(crate) struct InitTracker { // Ordered, non overlapping list of all uninitialized ranges. - uninitialized_ranges: Vec>, + uninitialized_ranges: UninitializedRangeVec, } -pub(crate) struct MemoryInitTrackerDrain<'a> { - uninitialized_ranges: &'a mut Vec>, - drain_range: Range, +pub(crate) struct InitTrackerDrain<'a, Idx: Ord + Copy> { + uninitialized_ranges: &'a mut UninitializedRangeVec, + drain_range: Range, first_index: usize, next_index: usize, } -impl<'a> Iterator for MemoryInitTrackerDrain<'a> { - type Item = Range; +impl<'a, Idx> Iterator for InitTrackerDrain<'a, Idx> +where + Idx: Ord + Copy, +{ + type Item = Range; fn next(&mut self) -> Option { if let Some(r) = self @@ -89,15 +116,18 @@ impl<'a> Iterator for MemoryInitTrackerDrain<'a> { } } -impl MemoryInitTracker { - pub(crate) fn new(size: wgt::BufferAddress) -> Self { +impl InitTracker +where + Idx: Ord + Copy + Default, +{ + pub(crate) fn new(size: Idx) -> Self { Self { - uninitialized_ranges: vec![0..size], + uninitialized_ranges: iter::once(Idx::default()..size).collect(), } } // Search smallest range.end which is bigger than bound in O(log n) (with n being number of uninitialized ranges) - fn lower_bound(&self, bound: wgt::BufferAddress) -> usize { + fn lower_bound(&self, bound: Idx) -> usize { // This is equivalent to, except that it may return an out of bounds index instead of //self.uninitialized_ranges.iter().position(|r| r.end > bound) @@ -123,10 +153,7 @@ impl MemoryInitTracker { // Checks if there's any uninitialized ranges within a query. // If there are any, the range returned a the subrange of the query_range that contains all these uninitialized regions. // Returned range may be larger than necessary (tradeoff for making this function O(log n)) - pub(crate) fn check( - &self, - query_range: Range, - ) -> Option> { + pub(crate) fn check(&self, query_range: Range) -> Option> { let index = self.lower_bound(query_range.start); self.uninitialized_ranges .get(index) @@ -153,12 +180,9 @@ impl MemoryInitTracker { // Drains uninitialized ranges in a query range. #[must_use] - pub(crate) fn drain( - &mut self, - drain_range: Range, - ) -> MemoryInitTrackerDrain { + pub(crate) fn drain(&mut self, drain_range: Range) -> InitTrackerDrain { let index = self.lower_bound(drain_range.start); - MemoryInitTrackerDrain { + InitTrackerDrain { drain_range, uninitialized_ranges: &mut self.uninitialized_ranges, first_index: index, @@ -167,19 +191,20 @@ impl MemoryInitTracker { } // Clears uninitialized ranges in a query range. - pub(crate) fn clear(&mut self, range: Range) { + pub(crate) fn clear(&mut self, range: Range) { self.drain(range).for_each(drop); } } #[cfg(test)] mod test { - use super::MemoryInitTracker; use std::ops::Range; + type Tracker = super::InitTracker; + #[test] fn check_for_newly_created_tracker() { - let tracker = MemoryInitTracker::new(10); + let tracker = Tracker::new(10); assert_eq!(tracker.check(0..10), Some(0..10)); assert_eq!(tracker.check(0..3), Some(0..3)); assert_eq!(tracker.check(3..4), Some(3..4)); @@ -188,7 +213,7 @@ mod test { #[test] fn check_for_cleared_tracker() { - let mut tracker = MemoryInitTracker::new(10); + let mut tracker = Tracker::new(10); tracker.clear(0..10); assert_eq!(tracker.check(0..10), None); assert_eq!(tracker.check(0..3), None); @@ -198,7 +223,7 @@ mod test { #[test] fn check_for_partially_filled_tracker() { - let mut tracker = MemoryInitTracker::new(25); + let mut tracker = Tracker::new(25); // Two regions of uninitialized memory tracker.clear(0..5); tracker.clear(10..15); @@ -217,7 +242,7 @@ mod test { #[test] fn clear_already_cleared() { - let mut tracker = MemoryInitTracker::new(30); + let mut tracker = Tracker::new(30); tracker.clear(10..20); // Overlapping with non-cleared @@ -233,11 +258,11 @@ mod test { #[test] fn drain_never_returns_ranges_twice_for_same_range() { - let mut tracker = MemoryInitTracker::new(19); + let mut tracker = Tracker::new(19); assert_eq!(tracker.drain(0..19).count(), 1); assert_eq!(tracker.drain(0..19).count(), 0); - let mut tracker = MemoryInitTracker::new(17); + let mut tracker = Tracker::new(17); assert_eq!(tracker.drain(5..8).count(), 1); assert_eq!(tracker.drain(5..8).count(), 0); assert_eq!(tracker.drain(1..3).count(), 1); @@ -248,31 +273,23 @@ mod test { #[test] fn drain_splits_ranges_correctly() { - let mut tracker = MemoryInitTracker::new(1337); + let mut tracker = Tracker::new(1337); assert_eq!( - tracker - .drain(21..42) - .collect::>>(), + tracker.drain(21..42).collect::>>(), vec![21..42] ); assert_eq!( - tracker - .drain(900..1000) - .collect::>>(), + tracker.drain(900..1000).collect::>>(), vec![900..1000] ); // Splitted ranges. assert_eq!( - tracker - .drain(5..1003) - .collect::>>(), + tracker.drain(5..1003).collect::>>(), vec![5..21, 42..900, 1000..1003] ); assert_eq!( - tracker - .drain(0..1337) - .collect::>>(), + tracker.drain(0..1337).collect::>>(), vec![0..5, 1003..1337] ); } diff --git a/wgpu-core/src/lib.rs b/wgpu-core/src/lib.rs index a56840efaa..dae1545754 100644 --- a/wgpu-core/src/lib.rs +++ b/wgpu-core/src/lib.rs @@ -37,8 +37,8 @@ pub mod device; pub mod error; pub mod hub; pub mod id; +mod init_tracker; pub mod instance; -mod memory_init_tracker; pub mod pipeline; pub mod present; pub mod resource; diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 227020ae31..ab6cf120e3 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -2,7 +2,7 @@ use crate::{ device::{DeviceError, HostMap, MissingFeatures}, hub::Resource, id::{DeviceId, SurfaceId, TextureId, Valid}, - memory_init_tracker::MemoryInitTracker, + init_tracker::BufferInitTracker, track::{TextureSelector, DUMMY_SELECTOR}, validation::MissingBufferUsageError, Label, LifeGuard, RefCount, Stored, @@ -120,7 +120,7 @@ pub struct Buffer { pub(crate) device_id: Stored, pub(crate) usage: wgt::BufferUsages, pub(crate) size: wgt::BufferAddress, - pub(crate) initialization_status: MemoryInitTracker, + pub(crate) initialization_status: BufferInitTracker, pub(crate) sync_mapped_writes: Option, pub(crate) life_guard: LifeGuard, pub(crate) map_state: BufferMapState,