1
// SPDX-License-Identifier: GPL-2.0
2

3
//! Direct memory access (DMA).
4
//!
5
//! C header: [`include/linux/dma-mapping.h`](srctree/include/linux/dma-mapping.h)
6

7
use crate::{
8
    bindings, build_assert, device,
9
    device::{Bound, Core},
10
    error::{to_result, Result},
11
    prelude::*,
12
    sync::aref::ARef,
13
    transmute::{AsBytes, FromBytes},
14
};
15

16
/// DMA address type.
17
///
18
/// Represents a bus address used for Direct Memory Access (DMA) operations.
19
///
20
/// This is an alias of the kernel's `dma_addr_t`, which may be `u32` or `u64` depending on
21
/// `CONFIG_ARCH_DMA_ADDR_T_64BIT`.
22
///
23
/// Note that this may be `u64` even on 32-bit architectures.
24
pub type DmaAddress = bindings::dma_addr_t;
25

26
/// Trait to be implemented by DMA capable bus devices.
27
///
28
/// The [`dma::Device`](Device) trait should be implemented by bus specific device representations,
29
/// where the underlying bus is DMA capable, such as [`pci::Device`](::kernel::pci::Device) or
30
/// [`platform::Device`](::kernel::platform::Device).
31
pub trait Device: AsRef<device::Device<Core>> {
32
    /// Set up the device's DMA streaming addressing capabilities.
33
    ///
34
    /// This method is usually called once from `probe()` as soon as the device capabilities are
35
    /// known.
36
    ///
37
    /// # Safety
38
    ///
39
    /// This method must not be called concurrently with any DMA allocation or mapping primitives,
40
    /// such as [`CoherentAllocation::alloc_attrs`].
41
    unsafe fn dma_set_mask(&self, mask: DmaMask) -> Result {
42
        // SAFETY:
43
        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
44
        // - The safety requirement of this function guarantees that there are no concurrent calls
45
        //   to DMA allocation and mapping primitives using this mask.
46
        to_result(unsafe { bindings::dma_set_mask(self.as_ref().as_raw(), mask.value()) })
47
    }
48

49
    /// Set up the device's DMA coherent addressing capabilities.
50
    ///
51
    /// This method is usually called once from `probe()` as soon as the device capabilities are
52
    /// known.
53
    ///
54
    /// # Safety
55
    ///
56
    /// This method must not be called concurrently with any DMA allocation or mapping primitives,
57
    /// such as [`CoherentAllocation::alloc_attrs`].
58
    unsafe fn dma_set_coherent_mask(&self, mask: DmaMask) -> Result {
59
        // SAFETY:
60
        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
61
        // - The safety requirement of this function guarantees that there are no concurrent calls
62
        //   to DMA allocation and mapping primitives using this mask.
63
        to_result(unsafe { bindings::dma_set_coherent_mask(self.as_ref().as_raw(), mask.value()) })
64
    }
65

66
    /// Set up the device's DMA addressing capabilities.
67
    ///
68
    /// This is a combination of [`Device::dma_set_mask`] and [`Device::dma_set_coherent_mask`].
69
    ///
70
    /// This method is usually called once from `probe()` as soon as the device capabilities are
71
    /// known.
72
    ///
73
    /// # Safety
74
    ///
75
    /// This method must not be called concurrently with any DMA allocation or mapping primitives,
76
    /// such as [`CoherentAllocation::alloc_attrs`].
77
    unsafe fn dma_set_mask_and_coherent(&self, mask: DmaMask) -> Result {
78
        // SAFETY:
79
        // - By the type invariant of `device::Device`, `self.as_ref().as_raw()` is valid.
80
        // - The safety requirement of this function guarantees that there are no concurrent calls
81
        //   to DMA allocation and mapping primitives using this mask.
82
        to_result(unsafe {
83
            bindings::dma_set_mask_and_coherent(self.as_ref().as_raw(), mask.value())
84
        })
85
    }
86
}
87

88
/// A DMA mask that holds a bitmask with the lowest `n` bits set.
89
///
90
/// Use [`DmaMask::new`] or [`DmaMask::try_new`] to construct a value. Values
91
/// are guaranteed to never exceed the bit width of `u64`.
92
///
93
/// This is the Rust equivalent of the C macro `DMA_BIT_MASK()`.
94
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
95
pub struct DmaMask(u64);
96

97
impl DmaMask {
98
    /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
99
    ///
100
    /// For `n <= 64`, sets exactly the lowest `n` bits.
101
    /// For `n > 64`, results in a build error.
102
    ///
103
    /// # Examples
104
    ///
105
    /// ```
106
    /// use kernel::dma::DmaMask;
107
    ///
108
    /// let mask0 = DmaMask::new::<0>();
109
    /// assert_eq!(mask0.value(), 0);
110
    ///
111
    /// let mask1 = DmaMask::new::<1>();
112
    /// assert_eq!(mask1.value(), 0b1);
113
    ///
114
    /// let mask64 = DmaMask::new::<64>();
115
    /// assert_eq!(mask64.value(), u64::MAX);
116
    ///
117
    /// // Build failure.
118
    /// // let mask_overflow = DmaMask::new::<100>();
119
    /// ```
120
    #[inline]
121
    pub const fn new<const N: u32>() -> Self {
122
        let Ok(mask) = Self::try_new(N) else {
123
            build_error!("Invalid DMA Mask.");
124
        };
125

126
        mask
127
    }
128

129
    /// Constructs a `DmaMask` with the lowest `n` bits set to `1`.
130
    ///
131
    /// For `n <= 64`, sets exactly the lowest `n` bits.
132
    /// For `n > 64`, returns [`EINVAL`].
133
    ///
134
    /// # Examples
135
    ///
136
    /// ```
137
    /// use kernel::dma::DmaMask;
138
    ///
139
    /// let mask0 = DmaMask::try_new(0)?;
140
    /// assert_eq!(mask0.value(), 0);
141
    ///
142
    /// let mask1 = DmaMask::try_new(1)?;
143
    /// assert_eq!(mask1.value(), 0b1);
144
    ///
145
    /// let mask64 = DmaMask::try_new(64)?;
146
    /// assert_eq!(mask64.value(), u64::MAX);
147
    ///
148
    /// let mask_overflow = DmaMask::try_new(100);
149
    /// assert!(mask_overflow.is_err());
150
    /// # Ok::<(), Error>(())
151
    /// ```
152
    #[inline]
153
    pub const fn try_new(n: u32) -> Result<Self> {
154
        Ok(Self(match n {
155
            0 => 0,
156
            1..=64 => u64::MAX >> (64 - n),
157
            _ => return Err(EINVAL),
158
        }))
159
    }
160

161
    /// Returns the underlying `u64` bitmask value.
162
    #[inline]
163
    pub const fn value(&self) -> u64 {
164
        self.0
165
    }
166
}
167

168
/// Possible attributes associated with a DMA mapping.
169
///
170
/// They can be combined with the operators `|`, `&`, and `!`.
171
///
172
/// Values can be used from the [`attrs`] module.
173
///
174
/// # Examples
175
///
176
/// ```
177
/// # use kernel::device::{Bound, Device};
178
/// use kernel::dma::{attrs::*, CoherentAllocation};
179
///
180
/// # fn test(dev: &Device<Bound>) -> Result {
181
/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
182
/// let c: CoherentAllocation<u64> =
183
///     CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, attribs)?;
184
/// # Ok::<(), Error>(()) }
185
/// ```
186
#[derive(Clone, Copy, PartialEq)]
187
#[repr(transparent)]
188
pub struct Attrs(u32);
189

190
impl Attrs {
191
    /// Get the raw representation of this attribute.
192
    pub(crate) fn as_raw(self) -> crate::ffi::c_ulong {
193
        self.0 as crate::ffi::c_ulong
194
    }
195

196
    /// Check whether `flags` is contained in `self`.
197
    pub fn contains(self, flags: Attrs) -> bool {
198
        (self & flags) == flags
199
    }
200
}
201

202
impl core::ops::BitOr for Attrs {
203
    type Output = Self;
204
    fn bitor(self, rhs: Self) -> Self::Output {
205
        Self(self.0 | rhs.0)
206
    }
207
}
208

209
impl core::ops::BitAnd for Attrs {
210
    type Output = Self;
211
    fn bitand(self, rhs: Self) -> Self::Output {
212
        Self(self.0 & rhs.0)
213
    }
214
}
215

216
impl core::ops::Not for Attrs {
217
    type Output = Self;
218
    fn not(self) -> Self::Output {
219
        Self(!self.0)
220
    }
221
}
222

223
/// DMA mapping attributes.
224
pub mod attrs {
225
    use super::Attrs;
226

227
    /// Specifies that reads and writes to the mapping may be weakly ordered, that is that reads
228
    /// and writes may pass each other.
229
    pub const DMA_ATTR_WEAK_ORDERING: Attrs = Attrs(bindings::DMA_ATTR_WEAK_ORDERING);
230

231
    /// Specifies that writes to the mapping may be buffered to improve performance.
232
    pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE);
233

234
    /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer.
235
    pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING);
236

237
    /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming
238
    /// that it has been already transferred to 'device' domain.
239
    pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC);
240

241
    /// Forces contiguous allocation of the buffer in physical memory.
242
    pub const DMA_ATTR_FORCE_CONTIGUOUS: Attrs = Attrs(bindings::DMA_ATTR_FORCE_CONTIGUOUS);
243

244
    /// Hints DMA-mapping subsystem that it's probably not worth the time to try
245
    /// to allocate memory to in a way that gives better TLB efficiency.
246
    pub const DMA_ATTR_ALLOC_SINGLE_PAGES: Attrs = Attrs(bindings::DMA_ATTR_ALLOC_SINGLE_PAGES);
247

248
    /// This tells the DMA-mapping subsystem to suppress allocation failure reports (similarly to
249
    /// `__GFP_NOWARN`).
250
    pub const DMA_ATTR_NO_WARN: Attrs = Attrs(bindings::DMA_ATTR_NO_WARN);
251

252
    /// Indicates that the buffer is fully accessible at an elevated privilege level (and
253
    /// ideally inaccessible or at least read-only at lesser-privileged levels).
254
    pub const DMA_ATTR_PRIVILEGED: Attrs = Attrs(bindings::DMA_ATTR_PRIVILEGED);
255

256
    /// Indicates that the buffer is MMIO memory.
257
    pub const DMA_ATTR_MMIO: Attrs = Attrs(bindings::DMA_ATTR_MMIO);
258
}
259

260
/// DMA data direction.
261
///
262
/// Corresponds to the C [`enum dma_data_direction`].
263
///
264
/// [`enum dma_data_direction`]: srctree/include/linux/dma-direction.h
265
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
266
#[repr(u32)]
267
pub enum DataDirection {
268
    /// The DMA mapping is for bidirectional data transfer.
269
    ///
270
    /// This is used when the buffer can be both read from and written to by the device.
271
    /// The cache for the corresponding memory region is both flushed and invalidated.
272
    Bidirectional = Self::const_cast(bindings::dma_data_direction_DMA_BIDIRECTIONAL),
273

274
    /// The DMA mapping is for data transfer from memory to the device (write).
275
    ///
276
    /// The CPU has prepared data in the buffer, and the device will read it.
277
    /// The cache for the corresponding memory region is flushed before device access.
278
    ToDevice = Self::const_cast(bindings::dma_data_direction_DMA_TO_DEVICE),
279

280
    /// The DMA mapping is for data transfer from the device to memory (read).
281
    ///
282
    /// The device will write data into the buffer for the CPU to read.
283
    /// The cache for the corresponding memory region is invalidated before CPU access.
284
    FromDevice = Self::const_cast(bindings::dma_data_direction_DMA_FROM_DEVICE),
285

286
    /// The DMA mapping is not for data transfer.
287
    ///
288
    /// This is primarily for debugging purposes. With this direction, the DMA mapping API
289
    /// will not perform any cache coherency operations.
290
    None = Self::const_cast(bindings::dma_data_direction_DMA_NONE),
291
}
292

293
impl DataDirection {
294
    /// Casts the bindgen-generated enum type to a `u32` at compile time.
295
    ///
296
    /// This function will cause a compile-time error if the underlying value of the
297
    /// C enum is out of bounds for `u32`.
298
    const fn const_cast(val: bindings::dma_data_direction) -> u32 {
299
        // CAST: The C standard allows compilers to choose different integer types for enums.
300
        // To safely check the value, we cast it to a wide signed integer type (`i128`)
301
        // which can hold any standard C integer enum type without truncation.
302
        let wide_val = val as i128;
303

304
        // Check if the value is outside the valid range for the target type `u32`.
305
        // CAST: `u32::MAX` is cast to `i128` to match the type of `wide_val` for the comparison.
306
        if wide_val < 0 || wide_val > u32::MAX as i128 {
307
            // Trigger a compile-time error in a const context.
308
            build_error!("C enum value is out of bounds for the target type `u32`.");
309
        }
310

311
        // CAST: This cast is valid because the check above guarantees that `wide_val`
312
        // is within the representable range of `u32`.
313
        wide_val as u32
314
    }
315
}
316

317
impl From<DataDirection> for bindings::dma_data_direction {
318
    /// Returns the raw representation of [`enum dma_data_direction`].
319
    fn from(direction: DataDirection) -> Self {
320
        // CAST: `direction as u32` gets the underlying representation of our `#[repr(u32)]` enum.
321
        // The subsequent cast to `Self` (the bindgen type) assumes the C enum is compatible
322
        // with the enum variants of `DataDirection`, which is a valid assumption given our
323
        // compile-time checks.
324
        direction as u32 as Self
325
    }
326
}
327

328
/// An abstraction of the `dma_alloc_coherent` API.
329
///
330
/// This is an abstraction around the `dma_alloc_coherent` API which is used to allocate and map
331
/// large coherent DMA regions.
332
///
333
/// A [`CoherentAllocation`] instance contains a pointer to the allocated region (in the
334
/// processor's virtual address space) and the device address which can be given to the device
335
/// as the DMA address base of the region. The region is released once [`CoherentAllocation`]
336
/// is dropped.
337
///
338
/// # Invariants
339
///
340
/// - For the lifetime of an instance of [`CoherentAllocation`], the `cpu_addr` is a valid pointer
341
///   to an allocated region of coherent memory and `dma_handle` is the DMA address base of the
342
///   region.
343
/// - The size in bytes of the allocation is equal to `size_of::<T> * count`.
344
/// - `size_of::<T> * count` fits into a `usize`.
345
// TODO
346
//
347
// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
348
// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
349
// that device resources can never survive device unbind.
350
//
351
// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
352
// allocation from surviving device unbind; it would require RCU read side critical sections to
353
// access the memory, which may require subsequent unnecessary copies.
354
//
355
// Hence, find a way to revoke the device resources of a `CoherentAllocation`, but not the
356
// entire `CoherentAllocation` including the allocated memory itself.
357
pub struct CoherentAllocation<T: AsBytes + FromBytes> {
358
    dev: ARef<device::Device>,
359
    dma_handle: DmaAddress,
360
    count: usize,
361
    cpu_addr: *mut T,
362
    dma_attrs: Attrs,
363
}
364

365
impl<T: AsBytes + FromBytes> CoherentAllocation<T> {
366
    /// Allocates a region of `size_of::<T> * count` of coherent memory.
367
    ///
368
    /// # Examples
369
    ///
370
    /// ```
371
    /// # use kernel::device::{Bound, Device};
372
    /// use kernel::dma::{attrs::*, CoherentAllocation};
373
    ///
374
    /// # fn test(dev: &Device<Bound>) -> Result {
375
    /// let c: CoherentAllocation<u64> =
376
    ///     CoherentAllocation::alloc_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
377
    /// # Ok::<(), Error>(()) }
378
    /// ```
379
    pub fn alloc_attrs(
380
        dev: &device::Device<Bound>,
381
        count: usize,
382
        gfp_flags: kernel::alloc::Flags,
383
        dma_attrs: Attrs,
384
    ) -> Result<CoherentAllocation<T>> {
385
        build_assert!(
386
            core::mem::size_of::<T>() > 0,
387
            "It doesn't make sense for the allocated type to be a ZST"
388
        );
389

390
        let size = count
391
            .checked_mul(core::mem::size_of::<T>())
392
            .ok_or(EOVERFLOW)?;
393
        let mut dma_handle = 0;
394
        // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
395
        let ret = unsafe {
396
            bindings::dma_alloc_attrs(
397
                dev.as_raw(),
398
                size,
399
                &mut dma_handle,
400
                gfp_flags.as_raw(),
401
                dma_attrs.as_raw(),
402
            )
403
        };
404
        if ret.is_null() {
405
            return Err(ENOMEM);
406
        }
407
        // INVARIANT:
408
        // - We just successfully allocated a coherent region which is accessible for
409
        //   `count` elements, hence the cpu address is valid. We also hold a refcounted reference
410
        //   to the device.
411
        // - The allocated `size` is equal to `size_of::<T> * count`.
412
        // - The allocated `size` fits into a `usize`.
413
        Ok(Self {
414
            dev: dev.into(),
415
            dma_handle,
416
            count,
417
            cpu_addr: ret.cast::<T>(),
418
            dma_attrs,
419
        })
420
    }
421

422
    /// Performs the same functionality as [`CoherentAllocation::alloc_attrs`], except the
423
    /// `dma_attrs` is 0 by default.
424
    pub fn alloc_coherent(
425
        dev: &device::Device<Bound>,
426
        count: usize,
427
        gfp_flags: kernel::alloc::Flags,
428
    ) -> Result<CoherentAllocation<T>> {
429
        CoherentAllocation::alloc_attrs(dev, count, gfp_flags, Attrs(0))
430
    }
431

432
    /// Returns the number of elements `T` in this allocation.
433
    ///
434
    /// Note that this is not the size of the allocation in bytes, which is provided by
435
    /// [`Self::size`].
436
    pub fn count(&self) -> usize {
437
        self.count
438
    }
439

440
    /// Returns the size in bytes of this allocation.
441
    pub fn size(&self) -> usize {
442
        // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits into
443
        // a `usize`.
444
        self.count * core::mem::size_of::<T>()
445
    }
446

447
    /// Returns the base address to the allocated region in the CPU's virtual address space.
448
    pub fn start_ptr(&self) -> *const T {
449
        self.cpu_addr
450
    }
451

452
    /// Returns the base address to the allocated region in the CPU's virtual address space as
453
    /// a mutable pointer.
454
    pub fn start_ptr_mut(&mut self) -> *mut T {
455
        self.cpu_addr
456
    }
457

458
    /// Returns a DMA handle which may be given to the device as the DMA address base of
459
    /// the region.
460
    pub fn dma_handle(&self) -> DmaAddress {
461
        self.dma_handle
462
    }
463

464
    /// Returns a DMA handle starting at `offset` (in units of `T`) which may be given to the
465
    /// device as the DMA address base of the region.
466
    ///
467
    /// Returns `EINVAL` if `offset` is not within the bounds of the allocation.
468
    pub fn dma_handle_with_offset(&self, offset: usize) -> Result<DmaAddress> {
469
        if offset >= self.count {
470
            Err(EINVAL)
471
        } else {
472
            // INVARIANT: The type invariant of `Self` guarantees that `size_of::<T> * count` fits
473
            // into a `usize`, and `offset` is inferior to `count`.
474
            Ok(self.dma_handle + (offset * core::mem::size_of::<T>()) as DmaAddress)
475
        }
476
    }
477

478
    /// Common helper to validate a range applied from the allocated region in the CPU's virtual
479
    /// address space.
480
    fn validate_range(&self, offset: usize, count: usize) -> Result {
481
        if offset.checked_add(count).ok_or(EOVERFLOW)? > self.count {
482
            return Err(EINVAL);
483
        }
484
        Ok(())
485
    }
486

487
    /// Returns the data from the region starting from `offset` as a slice.
488
    /// `offset` and `count` are in units of `T`, not the number of bytes.
489
    ///
490
    /// For ringbuffer type of r/w access or use-cases where the pointer to the live data is needed,
491
    /// [`CoherentAllocation::start_ptr`] or [`CoherentAllocation::start_ptr_mut`] could be used
492
    /// instead.
493
    ///
494
    /// # Safety
495
    ///
496
    /// * Callers must ensure that the device does not read/write to/from memory while the returned
497
    ///   slice is live.
498
    /// * Callers must ensure that this call does not race with a write to the same region while
499
    ///   the returned slice is live.
500
    pub unsafe fn as_slice(&self, offset: usize, count: usize) -> Result<&[T]> {
501
        self.validate_range(offset, count)?;
502
        // SAFETY:
503
        // - The pointer is valid due to type invariant on `CoherentAllocation`,
504
        //   we've just checked that the range and index is within bounds. The immutability of the
505
        //   data is also guaranteed by the safety requirements of the function.
506
        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
507
        //   that `self.count` won't overflow early in the constructor.
508
        Ok(unsafe { core::slice::from_raw_parts(self.cpu_addr.add(offset), count) })
509
    }
510

511
    /// Performs the same functionality as [`CoherentAllocation::as_slice`], except that a mutable
512
    /// slice is returned.
513
    ///
514
    /// # Safety
515
    ///
516
    /// * Callers must ensure that the device does not read/write to/from memory while the returned
517
    ///   slice is live.
518
    /// * Callers must ensure that this call does not race with a read or write to the same region
519
    ///   while the returned slice is live.
520
    pub unsafe fn as_slice_mut(&mut self, offset: usize, count: usize) -> Result<&mut [T]> {
521
        self.validate_range(offset, count)?;
522
        // SAFETY:
523
        // - The pointer is valid due to type invariant on `CoherentAllocation`,
524
        //   we've just checked that the range and index is within bounds. The immutability of the
525
        //   data is also guaranteed by the safety requirements of the function.
526
        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
527
        //   that `self.count` won't overflow early in the constructor.
528
        Ok(unsafe { core::slice::from_raw_parts_mut(self.cpu_addr.add(offset), count) })
529
    }
530

531
    /// Writes data to the region starting from `offset`. `offset` is in units of `T`, not the
532
    /// number of bytes.
533
    ///
534
    /// # Safety
535
    ///
536
    /// * Callers must ensure that the device does not read/write to/from memory while the returned
537
    ///   slice is live.
538
    /// * Callers must ensure that this call does not race with a read or write to the same region
539
    ///   that overlaps with this write.
540
    ///
541
    /// # Examples
542
    ///
543
    /// ```
544
    /// # fn test(alloc: &mut kernel::dma::CoherentAllocation<u8>) -> Result {
545
    /// let somedata: [u8; 4] = [0xf; 4];
546
    /// let buf: &[u8] = &somedata;
547
    /// // SAFETY: There is no concurrent HW operation on the device and no other R/W access to the
548
    /// // region.
549
    /// unsafe { alloc.write(buf, 0)?; }
550
    /// # Ok::<(), Error>(()) }
551
    /// ```
552
    pub unsafe fn write(&mut self, src: &[T], offset: usize) -> Result {
553
        self.validate_range(offset, src.len())?;
554
        // SAFETY:
555
        // - The pointer is valid due to type invariant on `CoherentAllocation`
556
        //   and we've just checked that the range and index is within bounds.
557
        // - `offset + count` can't overflow since it is smaller than `self.count` and we've checked
558
        //   that `self.count` won't overflow early in the constructor.
559
        unsafe {
560
            core::ptr::copy_nonoverlapping(src.as_ptr(), self.cpu_addr.add(offset), src.len())
561
        };
562
        Ok(())
563
    }
564

565
    /// Returns a pointer to an element from the region with bounds checking. `offset` is in
566
    /// units of `T`, not the number of bytes.
567
    ///
568
    /// Public but hidden since it should only be used from [`dma_read`] and [`dma_write`] macros.
569
    #[doc(hidden)]
570
    pub fn item_from_index(&self, offset: usize) -> Result<*mut T> {
571
        if offset >= self.count {
572
            return Err(EINVAL);
573
        }
574
        // SAFETY:
575
        // - The pointer is valid due to type invariant on `CoherentAllocation`
576
        // and we've just checked that the range and index is within bounds.
577
        // - `offset` can't overflow since it is smaller than `self.count` and we've checked
578
        // that `self.count` won't overflow early in the constructor.
579
        Ok(unsafe { self.cpu_addr.add(offset) })
580
    }
581

582
    /// Reads the value of `field` and ensures that its type is [`FromBytes`].
583
    ///
584
    /// # Safety
585
    ///
586
    /// This must be called from the [`dma_read`] macro which ensures that the `field` pointer is
587
    /// validated beforehand.
588
    ///
589
    /// Public but hidden since it should only be used from [`dma_read`] macro.
590
    #[doc(hidden)]
591
    pub unsafe fn field_read<F: FromBytes>(&self, field: *const F) -> F {
592
        // SAFETY:
593
        // - By the safety requirements field is valid.
594
        // - Using read_volatile() here is not sound as per the usual rules, the usage here is
595
        // a special exception with the following notes in place. When dealing with a potential
596
        // race from a hardware or code outside kernel (e.g. user-space program), we need that
597
        // read on a valid memory is not UB. Currently read_volatile() is used for this, and the
598
        // rationale behind is that it should generate the same code as READ_ONCE() which the
599
        // kernel already relies on to avoid UB on data races. Note that the usage of
600
        // read_volatile() is limited to this particular case, it cannot be used to prevent
601
        // the UB caused by racing between two kernel functions nor do they provide atomicity.
602
        unsafe { field.read_volatile() }
603
    }
604

605
    /// Writes a value to `field` and ensures that its type is [`AsBytes`].
606
    ///
607
    /// # Safety
608
    ///
609
    /// This must be called from the [`dma_write`] macro which ensures that the `field` pointer is
610
    /// validated beforehand.
611
    ///
612
    /// Public but hidden since it should only be used from [`dma_write`] macro.
613
    #[doc(hidden)]
614
    pub unsafe fn field_write<F: AsBytes>(&self, field: *mut F, val: F) {
615
        // SAFETY:
616
        // - By the safety requirements field is valid.
617
        // - Using write_volatile() here is not sound as per the usual rules, the usage here is
618
        // a special exception with the following notes in place. When dealing with a potential
619
        // race from a hardware or code outside kernel (e.g. user-space program), we need that
620
        // write on a valid memory is not UB. Currently write_volatile() is used for this, and the
621
        // rationale behind is that it should generate the same code as WRITE_ONCE() which the
622
        // kernel already relies on to avoid UB on data races. Note that the usage of
623
        // write_volatile() is limited to this particular case, it cannot be used to prevent
624
        // the UB caused by racing between two kernel functions nor do they provide atomicity.
625
        unsafe { field.write_volatile(val) }
626
    }
627
}
628

629
/// Note that the device configured to do DMA must be halted before this object is dropped.
630
impl<T: AsBytes + FromBytes> Drop for CoherentAllocation<T> {
631
    fn drop(&mut self) {
632
        let size = self.count * core::mem::size_of::<T>();
633
        // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
634
        // The cpu address, and the dma handle are valid due to the type invariants on
635
        // `CoherentAllocation`.
636
        unsafe {
637
            bindings::dma_free_attrs(
638
                self.dev.as_raw(),
639
                size,
640
                self.cpu_addr.cast(),
641
                self.dma_handle,
642
                self.dma_attrs.as_raw(),
643
            )
644
        }
645
    }
646
}
647

648
// SAFETY: It is safe to send a `CoherentAllocation` to another thread if `T`
649
// can be sent to another thread.
650
unsafe impl<T: AsBytes + FromBytes + Send> Send for CoherentAllocation<T> {}
651

652
/// Reads a field of an item from an allocated region of structs.
653
///
654
/// # Examples
655
///
656
/// ```
657
/// use kernel::device::Device;
658
/// use kernel::dma::{attrs::*, CoherentAllocation};
659
///
660
/// struct MyStruct { field: u32, }
661
///
662
/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
663
/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
664
/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
665
/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
666
///
667
/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
668
/// let whole = kernel::dma_read!(alloc[2]);
669
/// let field = kernel::dma_read!(alloc[1].field);
670
/// # Ok::<(), Error>(()) }
671
/// ```
672
#[macro_export]
673
macro_rules! dma_read {
674
    ($dma:expr, $idx: expr, $($field:tt)*) => {{
675
        (|| -> ::core::result::Result<_, $crate::error::Error> {
676
            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
677
            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
678
            // dereferenced. The compiler also further validates the expression on whether `field`
679
            // is a member of `item` when expanded by the macro.
680
            unsafe {
681
                let ptr_field = ::core::ptr::addr_of!((*item) $($field)*);
682
                ::core::result::Result::Ok(
683
                    $crate::dma::CoherentAllocation::field_read(&$dma, ptr_field)
684
                )
685
            }
686
        })()
687
    }};
688
    ($dma:ident [ $idx:expr ] $($field:tt)* ) => {
689
        $crate::dma_read!($dma, $idx, $($field)*)
690
    };
691
    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {
692
        $crate::dma_read!($($dma).*, $idx, $($field)*)
693
    };
694
}
695

696
/// Writes to a field of an item from an allocated region of structs.
697
///
698
/// # Examples
699
///
700
/// ```
701
/// use kernel::device::Device;
702
/// use kernel::dma::{attrs::*, CoherentAllocation};
703
///
704
/// struct MyStruct { member: u32, }
705
///
706
/// // SAFETY: All bit patterns are acceptable values for `MyStruct`.
707
/// unsafe impl kernel::transmute::FromBytes for MyStruct{};
708
/// // SAFETY: Instances of `MyStruct` have no uninitialized portions.
709
/// unsafe impl kernel::transmute::AsBytes for MyStruct{};
710
///
711
/// # fn test(alloc: &kernel::dma::CoherentAllocation<MyStruct>) -> Result {
712
/// kernel::dma_write!(alloc[2].member = 0xf);
713
/// kernel::dma_write!(alloc[1] = MyStruct { member: 0xf });
714
/// # Ok::<(), Error>(()) }
715
/// ```
716
#[macro_export]
717
macro_rules! dma_write {
718
    ($dma:ident [ $idx:expr ] $($field:tt)*) => {{
719
        $crate::dma_write!($dma, $idx, $($field)*)
720
    }};
721
    ($($dma:ident).* [ $idx:expr ] $($field:tt)* ) => {{
722
        $crate::dma_write!($($dma).*, $idx, $($field)*)
723
    }};
724
    ($dma:expr, $idx: expr, = $val:expr) => {
725
        (|| -> ::core::result::Result<_, $crate::error::Error> {
726
            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
727
            // SAFETY: `item_from_index` ensures that `item` is always a valid item.
728
            unsafe { $crate::dma::CoherentAllocation::field_write(&$dma, item, $val) }
729
            ::core::result::Result::Ok(())
730
        })()
731
    };
732
    ($dma:expr, $idx: expr, $(.$field:ident)* = $val:expr) => {
733
        (|| -> ::core::result::Result<_, $crate::error::Error> {
734
            let item = $crate::dma::CoherentAllocation::item_from_index(&$dma, $idx)?;
735
            // SAFETY: `item_from_index` ensures that `item` is always a valid pointer and can be
736
            // dereferenced. The compiler also further validates the expression on whether `field`
737
            // is a member of `item` when expanded by the macro.
738
            unsafe {
739
                let ptr_field = ::core::ptr::addr_of_mut!((*item) $(.$field)*);
740
                $crate::dma::CoherentAllocation::field_write(&$dma, ptr_field, $val)
741
            }
742
            ::core::result::Result::Ok(())
743
        })()
744
    };
745
}
746

747