.\" Copyright (c) 2002, 2003 Hiten M. Pandya.
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\"    notice, this list of conditions, and the following disclaimer,
.\"    without modification, immediately at the beginning of the file.
.\" 2. The name of the author may not be used to endorse or promote products
.\"    derived from this software without specific prior written permission.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR, CONTRIBUTORS OR THE
.\" VOICES IN HITEN PANDYA'S HEAD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
.\" TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
.\" PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
.\" LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
.\" NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
.\" SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.\"
.\" Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
.\" All rights reserved.
.\"
.\" This code is derived from software contributed to The NetBSD Foundation
.\" by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
.\" NASA Ames Research Center.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\"    notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\"    notice, this list of conditions and the following disclaimer in the
.\"    documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
.\" PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\" $NetBSD: bus_dma.9,v 1.25 2002/10/14 13:43:16 wiz Exp $
.\"
.Dd April 14, 2020
.Dt BUS_DMA 9
.Os
.Sh NAME
.Nm bus_dma ,
.Nm bus_dma_tag_create ,
.Nm bus_dma_tag_destroy ,
.Nm bus_dma_template_init ,
.Nm bus_dma_template_tag ,
.Nm bus_dma_template_clone ,
.Nm bus_dmamap_create ,
.Nm bus_dmamap_destroy ,
.Nm bus_dmamap_load ,
.Nm bus_dmamap_load_bio ,
.Nm bus_dmamap_load_ccb ,
.Nm bus_dmamap_load_crp ,
.Nm bus_dmamap_load_mbuf ,
.Nm bus_dmamap_load_mbuf_sg ,
.Nm bus_dmamap_load_uio ,
.Nm bus_dmamap_unload ,
.Nm bus_dmamap_sync ,
.Nm bus_dmamem_alloc ,
.Nm bus_dmamem_free
.Nd Bus and Machine Independent DMA Mapping Interface
.Sh SYNOPSIS
.In machine/bus.h
.Ft int
.Fn bus_dma_tag_create "bus_dma_tag_t parent" "bus_size_t alignment" \
"bus_addr_t boundary" "bus_addr_t lowaddr" "bus_addr_t highaddr" \
"bus_dma_filter_t *filtfunc" "void *filtfuncarg" "bus_size_t maxsize" \
"int nsegments" "bus_size_t maxsegsz" "int flags" "bus_dma_lock_t *lockfunc" \
"void *lockfuncarg" "bus_dma_tag_t *dmat"
.Ft int
.Fn bus_dma_tag_destroy "bus_dma_tag_t dmat"
.Ft void
.Fo bus_dma_template_init
.Fa "bus_dma_template_t template"
.Fa "bus_dma_tag_t parent"
.Fc
.Ft int
.Fo bus_dma_template_tag
.Fa "bus_dma_template_t template"
.Fa "bus_dma_tag_t *dmat"
.Fc
.Ft void
.Fo bus_dma_template_clone
.Fa "bus_dma_template_t template"
.Fa "bus_dma_tag_t dmat"
.Fc
.Ft int
.Fn bus_dmamap_create "bus_dma_tag_t dmat" "int flags" "bus_dmamap_t *mapp"
.Ft int
.Fn bus_dmamap_destroy "bus_dma_tag_t dmat" "bus_dmamap_t map"
.Ft int
.Fn bus_dmamap_load "bus_dma_tag_t dmat" "bus_dmamap_t map" "void *buf" \
"bus_size_t buflen" "bus_dmamap_callback_t *callback" "void *callback_arg" \
"int flags"
.Ft int
.Fn bus_dmamap_load_bio "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct bio *bio" "bus_dmamap_callback_t *callback" "void *callback_arg" \
"int flags"
.Ft int
.Fn bus_dmamap_load_ccb "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"union ccb *ccb" "bus_dmamap_callback_t *callback" "void *callback_arg" \
"int flags"
.Ft int
.Fn bus_dmamap_load_crp "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct crypto *crp" "bus_dmamap_callback_t *callback" "void *callback_arg" \
"int flags"
.Ft int
.Fn bus_dmamap_load_mbuf "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct mbuf *mbuf" "bus_dmamap_callback2_t *callback" "void *callback_arg" \
"int flags"
.Ft int
.Fn bus_dmamap_load_mbuf_sg "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct mbuf *mbuf" "bus_dma_segment_t *segs" "int *nsegs" "int flags"
.Ft int
.Fn bus_dmamap_load_uio "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"struct uio *uio" "bus_dmamap_callback2_t *callback" "void *callback_arg" \
"int flags"
.Ft void
.Fn bus_dmamap_unload "bus_dma_tag_t dmat" "bus_dmamap_t map"
.Ft void
.Fn bus_dmamap_sync "bus_dma_tag_t dmat" "bus_dmamap_t map" \
"op"
.Ft int
.Fn bus_dmamem_alloc "bus_dma_tag_t dmat" "void **vaddr" \
"int flags" "bus_dmamap_t *mapp"
.Ft void
.Fn bus_dmamem_free "bus_dma_tag_t dmat" "void *vaddr" \
"bus_dmamap_t map"
.Sh DESCRIPTION
Direct Memory Access (DMA) is a method of transferring data
without involving the CPU, thus providing higher performance.
A DMA transaction can be achieved between device to memory,
device to device, or memory to memory.
.Pp
The
.Nm
API is a bus, device, and machine-independent (MI) interface to
DMA mechanisms.
It provides the client with flexibility and simplicity by
abstracting machine dependent issues like setting up
DMA mappings, handling cache issues, bus specific features
and limitations.
.Sh OVERVIEW
A tag structure
.Vt ( bus_dma_tag_t )
is used to describe the properties of a group of related DMA
transactions.
One way to view this is that a tag describes the limitations of a DMA engine.
For example, if a DMA engine in a device is limited to 32-bit addresses,
that limitation is specified by a parameter when creating the tag
for that device.
Similarly, a tag can be marked as requiring buffers whose addresses are
aligned to a specific boundary.
.Pp
Some devices may require multiple tags to describe DMA
transactions with differing properties.
For example, a device might require 16-byte alignment of its descriptor ring
while permitting arbitrary alignment of I/O buffers.
In this case,
the driver must create one tag for the descriptor ring and a separate tag for
I/O buffers.
If a device has restrictions that are common to all DMA transactions
in addition to restrictions that differ between unrelated groups of
transactions,
the driver can first create a
.Dq parent
tag that decribes the common restrictions.
The per-group tags can then inherit these restrictions from this
.Dq parent
tag rather than having to list them explicitly when creating the per-group tags.
.Pp
A mapping structure
.Vt ( bus_dmamap_t )
represents a mapping of a memory region for DMA.
On systems with I/O MMUs,
the mapping structure tracks any I/O MMU entries used by a request.
For DMA requests that require bounce pages,
the mapping tracks the bounce pages used.
.Pp
To prepare for one or more DMA transactions,
a mapping must be bound to a memory region by calling one of the
.Fn bus_dmamap_load
functions.
These functions configure the mapping which can include programming entries
in an I/O MMU and/or allocating bounce pages.
An output of these functions
(either directly or indirectly by invoking a callback routine)
is the list of scatter/gather address ranges a consumer can pass to a DMA
engine to access the memory region.
When a mapping is no longer needed,
the mapping must be unloaded via
.Fn bus_dmamap_unload .
.Pp
Before and after each DMA transaction,
.Fn bus_dmamap_sync
must be used to ensure that the correct data is used by the DMA engine and
the CPU.
If a mapping uses bounce pages,
the sync operations copy data between the bounce pages and the memory region
bound to the mapping.
Sync operations also handle architecture-specific details such as CPU cache
flushing and CPU memory operation ordering.
.Sh STATIC VS DYNAMIC
.Nm
handles two types of DMA transactions: static and dynamic.
Static transactions are used with a long-lived memory region that is reused
for many transactions such as a descriptor ring.
Dynamic transactions are used for transfers to or from transient buffers
such as I/O buffers holding a network packet or disk block.
Each transaction type uses a different subset of the
.Nm
API.
.Ss Static Transactions
Static transactions use memory regions allocated by
.Nm .
Each static memory region is allocated by calling
.Fn bus_dmamem_alloc .
This function requires a valid tag describing the properties of the
DMA transactions to this region such as alignment or address restrictions.
Multiple regions can share a single tag if they share the same restrictions.
.Pp
.Fn bus_dmamem_alloc
allocates a memory region along with a mapping object.
The associated tag, memory region, and mapping object must then be passed to
.Fn bus_dmamap_load
to bind the mapping to the allocated region and obtain the
scatter/gather list.
.Pp
It is expected that
.Fn bus_dmamem_alloc
will attempt to allocate memory requiring less expensive sync operations
(for example, implementations should not allocate regions requiring bounce
pages),
but sync operations should still be used.
For example, a driver should use
.Fn bus_dmamap_sync
in an interrupt handler before reading descriptor ring entries written by the
device prior to the interrupt.
.Pp
When a consumer is finished with a memory region,
it should unload the mapping via
.Fn bus_dmamap_unload
and then release the memory region and mapping object via
.Fn bus_dmamem_free .
.Ss Dynamic Transactions
Dynamic transactions map memory regions provided by other parts of the system.
A tag must be created via
.Fn bus_dma_tag_create
to describe the DMA transactions to and from these memory regions,
and a pool of mapping objects must be allocated via
.Fn bus_dmamap_create
to track the mappings of any in-flight transactions.
.Pp
When a consumer wishes to schedule a transaction for a memory region,
the consumer must first obtain an unused mapping object from its pool
of mapping objects.
The memory region must be bound to the mapping object via one of the
.Fn bus_dmamap_load
functions.
Before scheduling the transaction,
the consumer should sync the memory region via
.Fn bus_dmamap_sync
with one or more of the
.Dq PRE
flags.
After the transaction has completed,
the consumer should sync the memory region via
.Fn bus_dmamap_sync
with one or more of the
.Dq POST
flags.
The mapping can then be unloaded via
.Fn bus_dmamap_unload ,
and the mapping object can be returned to the pool of unused mapping objects.
.Pp
When a consumer is no longer scheduling DMA transactions,
the mapping objects should be freed via
.Fn bus_dmamap_destroy ,
and the tag should be freed via
.Fn bus_dma_tag_destroy .
.Sh STRUCTURES AND TYPES
.Bl -tag -width indent
.It Vt bus_dma_tag_t
A machine-dependent (MD) opaque type that describes the
characteristics of a group of DMA transactions.
DMA tags are organized into a hierarchy, with each child
tag inheriting the restrictions of its parent.
This allows all devices along the path of DMA transactions
to contribute to the constraints of those transactions.
.It Vt bus_dma_template_t
A template structure for creating a
.Fa bus_dma_tag_t
from a set of defaults.
Once initialized with
.Fn bus_dma_template_init ,
a driver can over-ride individual fields to suit its needs.
The following fields have the indicated values:
.Bd -literal
	alignment	1
	boundary	0
	lowaddr		BUS_SPACE_MAXADDR
	highaddr	BUS_SPACE_MAXADDR
	maxsize		BUS_SPACE_MAXSIZE
	nsegments	BUS_SPACE_UNRESTRICTED
	maxsegsize	BUS_SPACE_MAXSIZE
	flags		0
	lockfunc	NULL
	lockfuncarg	NULL
.Ed
.Pp
Descriptions of each field are documented with
.Fn bus_dma_tag_create .
Note that the
.Fa filtfunc
and
.Fa filtfuncarg
attributes of the DMA tag are not supported with templates.
.It Vt bus_dma_filter_t
Client specified address filter having the format:
.Bl -tag -width indent
.It Ft int
.Fn "client_filter" "void *filtarg" "bus_addr_t testaddr"
.El
.Pp
Address filters can be specified during tag creation to allow
for devices whose DMA address restrictions cannot be specified
by a single window.
The
.Fa filtarg
argument is specified by the client during tag creation to be passed to all
invocations of the callback.
The
.Fa testaddr
argument contains a potential starting address of a DMA mapping.
The filter function operates on the set of addresses from
.Fa testaddr
to
.Ql trunc_page(testaddr) + PAGE_SIZE - 1 ,
inclusive.
The filter function should return zero if any mapping in this range
can be accommodated by the device and non-zero otherwise.
.It Vt bus_dma_segment_t
A machine-dependent type that describes individual
DMA segments.
It contains the following fields:
.Bd -literal
	bus_addr_t	ds_addr;
	bus_size_t	ds_len;
.Ed
.Pp
The
.Fa ds_addr
field contains the device visible address of the DMA segment, and
.Fa ds_len
contains the length of the DMA segment.
Although the DMA segments returned by a mapping call will adhere to
all restrictions necessary for a successful DMA operation, some conversion
(e.g.\& a conversion from host byte order to the device's byte order) is
almost always required when presenting segment information to the device.
.It Vt bus_dmamap_t
A machine-dependent opaque type describing an individual mapping.
One map is used for each memory allocation that will be loaded.
Maps can be reused once they have been unloaded.
Multiple maps can be associated with one DMA tag.
While the value of the map may evaluate to
.Dv NULL
on some platforms under certain conditions,
it should never be assumed that it will be
.Dv NULL
in all cases.
.It Vt bus_dmamap_callback_t
Client specified callback for receiving mapping information resulting from
the load of a
.Vt bus_dmamap_t
via
.Fn bus_dmamap_load ,
.Fn bus_dmamap_load_bio ,
.Fn bus_dmamap_load_ccb ,
or
.Fn bus_dmamap_load_crp .
Callbacks are of the format:
.Bl -tag -width indent
.It Ft void
.Fn "client_callback" "void *callback_arg" "bus_dma_segment_t *segs" \
"int nseg" "int error"
.El
.Pp
The
.Fa callback_arg
is the callback argument passed to dmamap load functions.
The
.Fa segs
and
.Fa nseg
arguments describe an array of
.Vt bus_dma_segment_t
structures that represent the mapping.
This array is only valid within the scope of the callback function.
The success or failure of the mapping is indicated by the
.Fa error
argument.
More information on the use of callbacks can be found in the
description of the individual dmamap load functions.
.It Vt bus_dmamap_callback2_t
Client specified callback for receiving mapping information resulting from
the load of a
.Vt bus_dmamap_t
via
.Fn bus_dmamap_load_uio
or
.Fn bus_dmamap_load_mbuf .
.Pp
Callback2s are of the format:
.Bl -tag -width indent
.It Ft void
.Fn "client_callback2" "void *callback_arg" "bus_dma_segment_t *segs" \
"int nseg" "bus_size_t mapsize" "int error"
.El
.Pp
Callback2's behavior is the same as
.Vt bus_dmamap_callback_t
with the addition that the length of the data mapped is provided via
.Fa mapsize .
.It Vt bus_dmasync_op_t
Memory synchronization operation specifier.
Bus DMA requires explicit synchronization of memory with its device
visible mapping in order to guarantee memory coherency.
The
.Vt bus_dmasync_op_t
allows the type of DMA operation that will be or has been performed
to be communicated to the system so that the correct coherency measures
are taken.
The operations are represented as bitfield flags that can be combined together,
though it only makes sense to combine PRE flags or POST flags, not both.
See the
.Fn bus_dmamap_sync
description below for more details on how to use these operations.
.Pp
All operations specified below are performed from the host memory point of view,
where a read implies data coming from the device to the host memory, and a write
implies data going from the host memory to the device.
Alternatively, the operations can be thought of in terms of driver operations,
where reading a network packet or storage sector corresponds to a read operation
in
.Nm .
.Bl -tag -width ".Dv BUS_DMASYNC_POSTWRITE"
.It Dv BUS_DMASYNC_PREREAD
Perform any synchronization required prior to an update of host memory by the
device.
.It Dv BUS_DMASYNC_PREWRITE
Perform any synchronization required after an update of host memory by the CPU
and prior to device access to host memory.
.It Dv BUS_DMASYNC_POSTREAD
Perform any synchronization required after an update of host memory by the
device and prior to CPU access to host memory.
.It Dv BUS_DMASYNC_POSTWRITE
Perform any synchronization required after device access to host memory.
.El
.It Vt bus_dma_lock_t
Client specified lock/mutex manipulation method.
This will be called from
within busdma whenever a client lock needs to be manipulated.
In its current form, the function will be called immediately before
the callback for a DMA load operation that has been deferred with
.Dv BUS_DMA_LOCK
and immediately after with
.Dv BUS_DMA_UNLOCK .
If the load operation does not need to be deferred, then it
will not be called since the function loading the map should
be holding the appropriate locks.
This method is of the format:
.Bl -tag -width indent
.It Ft void
.Fn "lockfunc" "void *lockfunc_arg" "bus_dma_lock_op_t op"
.El
.Pp
The
.Fa lockfuncarg
argument is specified by the client during tag creation to be passed to all
invocations of the callback.
The
.Fa op
argument specifies the lock operation to perform.
.Pp
Two
.Vt lockfunc
implementations are provided for convenience.
.Fn busdma_lock_mutex
performs standard mutex operations on the sleep mutex provided via
.Fa lockfuncarg .
.Fn dflt_lock
will generate a system panic if it is called.
It is substituted into the tag when
.Fa lockfunc
is passed as
.Dv NULL
to
.Fn bus_dma_tag_create
and is useful for tags that should not be used with deferred load operations.
.It Vt bus_dma_lock_op_t
Operations to be performed by the client-specified
.Fn lockfunc .
.Bl -tag -width ".Dv BUS_DMA_UNLOCK"
.It Dv BUS_DMA_LOCK
Acquires and/or locks the client locking primitive.
.It Dv BUS_DMA_UNLOCK
Releases and/or unlocks the client locking primitive.
.El
.El
.Sh FUNCTIONS
.Bl -tag -width indent
.It Fn bus_dma_tag_create "parent" "alignment" "boundary" "lowaddr" \
"highaddr" "*filtfunc" "*filtfuncarg" "maxsize" "nsegments" "maxsegsz" \
"flags" "lockfunc" "lockfuncarg" "*dmat"
Allocates a DMA tag, and initializes it according to
the arguments provided:
.Bl -tag -width ".Fa filtfuncarg"
.It Fa parent
A parent tag from which to inherit restrictions.
The restrictions passed in other arguments can only further tighten the
restrictions inherited from the parent tag.
.Pp
All tags created by a device driver must inherit from the tag returned by
.Fn bus_get_dma_tag
to honor restrictions between the parent bridge, CPU memory, and the
device.
.It Fa alignment
Alignment constraint, in bytes, of any mappings created using this tag.
The alignment must be a power of 2.
Hardware that can DMA starting at any address would specify
.Em 1
for byte alignment.
Hardware requiring DMA transfers to start on a multiple of 4K
would specify
.Em 4096 .
.It Fa boundary
Boundary constraint, in bytes, of the target DMA memory region.
The boundary indicates the set of addresses, all multiples of the
boundary argument, that cannot be crossed by a single
.Vt bus_dma_segment_t .
The boundary must be a power of 2 and must be no smaller than the
maximum segment size.
.Ql 0
indicates that there are no boundary restrictions.
.It Fa lowaddr , highaddr
Bounds of the window of bus address space that
.Em cannot
be directly accessed by the device.
The window contains all addresses greater than
.Fa lowaddr
and less than or equal to
.Fa highaddr .
For example, a device incapable of DMA above 4GB, would specify a
.Fa highaddr
of
.Dv BUS_SPACE_MAXADDR
and a
.Fa lowaddr
of
.Dv BUS_SPACE_MAXADDR_32BIT .
Similarly a device that can only perform DMA to addresses below
16MB would specify a
.Fa highaddr
of
.Dv BUS_SPACE_MAXADDR
and a
.Fa lowaddr
of
.Dv BUS_SPACE_MAXADDR_24BIT .
Some implementations require that some region of device visible
address space, overlapping available host memory, be outside the
window.
This area of
.Ql safe memory
is used to bounce requests that would otherwise conflict with
the exclusion window.
.It Fa filtfunc
Optional filter function (may be
.Dv NULL )
to be called for any attempt to
map memory into the window described by
.Fa lowaddr
and
.Fa highaddr .
A filter function is only required when the single window described
by
.Fa lowaddr
and
.Fa highaddr
cannot adequately describe the constraints of the device.
The filter function will be called for every machine page
that overlaps the exclusion window.
.It Fa filtfuncarg
Argument passed to all calls to the filter function for this tag.
May be
.Dv NULL .
.It Fa maxsize
Maximum size, in bytes, of the sum of all segment lengths in a given
DMA mapping associated with this tag.
.It Fa nsegments
Number of discontinuities (scatter/gather segments) allowed
in a DMA mapped region.
.It Fa maxsegsz
Maximum size, in bytes, of a segment in any DMA mapped region associated
with
.Fa dmat .
.It Fa flags
Are as follows:
.Bl -tag -width ".Dv BUS_DMA_ALLOCNOW"
.It Dv BUS_DMA_ALLOCNOW
Pre-allocate enough resources to handle at least one map load operation on
this tag.
If sufficient resources are not available,
.Er ENOMEM
is returned.
This should not be used for tags that only describe buffers that will be
allocated with
.Fn bus_dmamem_alloc .
Also, due to resource sharing with other tags, this flag does not guarantee
that resources will be allocated or reserved exclusively for this tag.
It should be treated only as a minor optimization.
.It Dv BUS_DMA_COHERENT
Indicate that the DMA engine and CPU are cache-coherent.
Cached memory may be used to back allocations created by
.Fn bus_dmamem_alloc .
For
.Fn bus_dma_tag_create ,
the
.Dv BUS_DMA_COHERENT
flag is currently implemented on arm64.
.El
.It Fa lockfunc
Optional lock manipulation function (may be
.Dv NULL )
to be called when busdma
needs to manipulate a lock on behalf of the client.
If
.Dv NULL
is specified,
.Fn dflt_lock
is used.
.It Fa lockfuncarg
Optional argument to be passed to the function specified by
.Fa lockfunc .
.It Fa dmat
Pointer to a bus_dma_tag_t where the resulting DMA tag will
be stored.
.El
.Pp
Returns
.Er ENOMEM
if sufficient memory is not available for tag creation
or allocating mapping resources.
.It Fn bus_dma_tag_destroy "dmat"
Deallocate the DMA tag
.Fa dmat
that was created by
.Fn bus_dma_tag_create .
.Pp
Returns
.Er EBUSY
if any DMA maps remain associated with
.Fa dmat
or
.Ql 0
on success.
.It Fn bus_dma_template_init "*template" "parent"
Initializes a
.Fa bus_dma_template_t
structure and associates it with an optional
.Fa parent .
The
.Fa parent
argument may be NULL.
.It Fn bus_dma_template_tag "*template" "*dmat"
Unpacks a template into a tag, and returns the tag via the
.Fa dmat .
All return values are identical to
.Fn bus_dma_tag_create .
.It Fn bus_dma_template_clone "*template" "dmat"
Clones the fields from a tag to a template.
This is useful for cloning tags when paired with
.Fn bus_dma_template_tag .
A template that is filled in as a clone does not need to be initialized
first.
.It Fn bus_dmamap_create "dmat" "flags" "*mapp"
Allocates and initializes a DMA map.
Arguments are as follows:
.Bl -tag -width ".Fa nsegments"
.It Fa dmat
DMA tag.
.It Fa flags
Are as follows:
.Bl -tag -width ".Dv BUS_DMA_COHERENT"
.It Dv BUS_DMA_COHERENT
Attempt to map the memory loaded with this map such that cache sync
operations are as cheap as possible.
This flag is typically set on maps when the memory loaded with these will
be accessed by both a CPU and a DMA engine, frequently such as control data
and as opposed to streamable data such as receive and transmit buffers.
Use of this flag does not remove the requirement of using
.Fn bus_dmamap_sync ,
but it may reduce the cost of performing these operations.
.El
.It Fa mapp
Pointer to a
.Vt bus_dmamap_t
where the resulting DMA map will be stored.
.El
.Pp
Returns
.Er ENOMEM
if sufficient memory is not available for creating the
map or allocating mapping resources.
.It Fn bus_dmamap_destroy "dmat" "map"
Frees all resources associated with a given DMA map.
Arguments are as follows:
.Bl -tag -width ".Fa dmat"
.It Fa dmat
DMA tag used to allocate
.Fa map .
.It Fa map
The DMA map to destroy.
.El
.Pp
Returns
.Er EBUSY
if a mapping is still active for
.Fa map .
.It Fn bus_dmamap_load "dmat" "map" "buf" "buflen" "*callback" \
"callback_arg" "flags"
Creates a mapping in device visible address space of
.Fa buflen
bytes of
.Fa buf ,
associated with the DMA map
.Fa map .
This call will always return immediately and will not block for any reason.
Arguments are as follows:
.Bl -tag -width ".Fa buflen"
.It Fa dmat
DMA tag used to allocate
.Fa map .
.It Fa map
A DMA map without a currently active mapping.
.It Fa buf
A kernel virtual address pointer to a contiguous (in KVA) buffer, to be
mapped into device visible address space.
.It Fa buflen
The size of the buffer.
.It Fa callback Fa callback_arg
The callback function, and its argument.
This function is called once sufficient mapping resources are available for
the DMA operation.
If resources are temporarily unavailable, this function will be deferred until
later, but the load operation will still return immediately to the caller.
Thus, callers should not assume that the callback will be called before the
load returns, and code should be structured appropriately to handle this.
See below for specific flags and error codes that control this behavior.
.It Fa flags
Are as follows:
.Bl -tag -width ".Dv BUS_DMA_NOWAIT"
.It Dv BUS_DMA_NOWAIT
The load should not be deferred in case of insufficient mapping resources,
and instead should return immediately with an appropriate error.
.It Dv BUS_DMA_NOCACHE
The generated transactions to and from the virtual page are non-cacheable.
.El
.El
.Pp
Return values to the caller are as follows:
.Bl -tag -width ".Er EINPROGRESS"
.It 0
The callback has been called and completed.
The status of the mapping has been delivered to the callback.
.It Er EINPROGRESS
The mapping has been deferred for lack of resources.
The callback will be called as soon as resources are available.
Callbacks are serviced in FIFO order.
.Pp
Note that subsequent load operations for the same tag that do not require
extra resources will still succeed.
This may result in out-of-order processing of requests.
If the caller requires the order of requests to be preserved,
then the caller is required to stall subsequent requests until a pending
request's callback is invoked.
.It Er ENOMEM
The load request has failed due to insufficient resources, and the caller
specifically used the
.Dv BUS_DMA_NOWAIT
flag.
.It Er EINVAL
The load request was invalid.
The callback has been called and has been provided the same error.
This error value may indicate that
.Fa dmat ,
.Fa map ,
.Fa buf ,
or
.Fa callback
were invalid, or
.Fa buflen
was larger than the
.Fa maxsize
argument used to create the dma tag
.Fa dmat .
.El
.Pp
When the callback is called, it is presented with an error value
indicating the disposition of the mapping.
Error may be one of the following:
.Bl -tag -width ".Er EINPROGRESS"
.It 0
The mapping was successful and the
.Fa dm_segs
callback argument contains an array of
.Vt bus_dma_segment_t
elements describing the mapping.
This array is only valid during the scope of the callback function.
.It Er EFBIG
A mapping could not be achieved within the segment constraints provided
in the tag even though the requested allocation size was less than maxsize.
.El
.It Fn bus_dmamap_load_bio "dmat" "map" "bio" "callback" "callback_arg" "flags"
This is a variation of
.Fn bus_dmamap_load
which maps buffers pointed to by
.Fa bio
for DMA transfers.
.Fa bio
may point to either a mapped or unmapped buffer.
.It Fn bus_dmamap_load_ccb "dmat" "map" "ccb" "callback" "callback_arg" "flags"
This is a variation of
.Fn bus_dmamap_load
which maps data pointed to by
.Fa ccb
for DMA transfers.
The data for
.Fa ccb
may be any of the following types:
.Bl -tag -width ".Er CAM_DATA_SG_PADDR"
.It CAM_DATA_VADDR
The data is a single KVA buffer.
.It CAM_DATA_PADDR
The data is a single bus address range.
.It CAM_DATA_SG
The data is a scatter/gather list of KVA buffers.
.It CAM_DATA_SG_PADDR
The data is a scatter/gather list of bus address ranges.
.It CAM_DATA_BIO
The data is contained in a
.Vt struct bio
attached to the CCB.
.El
.Pp
.Fn bus_dmamap_load_ccb
supports the following CCB XPT function codes:
.Pp
.Bl -item -offset indent -compact
.It
XPT_ATA_IO
.It
XPT_CONT_TARGET_IO
.It
XPT_SCSI_IO
.El
.It Fn bus_dmamap_load_crp "dmat" "map" "crp" "callback" "callback_arg" "flags"
This is a variation of
.Fn bus_dmamap_load
which maps buffers pointed to by
.Fa crp
for DMA transfers.
The
.Dv BUS_DMA_NOWAIT
flag is implied, thus no callback deferral will happen.
.It Fn bus_dmamap_load_mbuf "dmat" "map" "mbuf" "callback2" "callback_arg" \
"flags"
This is a variation of
.Fn bus_dmamap_load
which maps mbuf chains
for DMA transfers.
A
.Vt bus_size_t
argument is also passed to the callback routine, which
contains the mbuf chain's packet header length.
The
.Dv BUS_DMA_NOWAIT
flag is implied, thus no callback deferral will happen.
.Pp
Mbuf chains are assumed to be in kernel virtual address space.
.Pp
Beside the error values listed for
.Fn bus_dmamap_load ,
.Er EINVAL
will be returned if the size of the mbuf chain exceeds the maximum limit of the
DMA tag.
.It Fn bus_dmamap_load_mbuf_sg "dmat" "map" "mbuf" "segs" "nsegs" "flags"
This is just like
.Fn bus_dmamap_load_mbuf
except that it returns immediately without calling a callback function.
It is provided for efficiency.
The scatter/gather segment array
.Va segs
is provided by the caller and filled in directly by the function.
The
.Va nsegs
argument is returned with the number of segments filled in.
Returns the same errors as
.Fn bus_dmamap_load_mbuf .
.It Fn bus_dmamap_load_uio "dmat" "map" "uio" "callback2" "callback_arg" "flags"
This is a variation of
.Fn bus_dmamap_load
which maps buffers pointed to by
.Fa uio
for DMA transfers.
A
.Vt bus_size_t
argument is also passed to the callback routine, which contains the size of
.Fa uio ,
i.e.
.Fa uio->uio_resid .
The
.Dv BUS_DMA_NOWAIT
flag is implied, thus no callback deferral will happen.
Returns the same errors as
.Fn bus_dmamap_load .
.Pp
If
.Fa uio->uio_segflg
is
.Dv UIO_USERSPACE ,
then it is assumed that the buffer,
.Fa uio
is in
.Fa "uio->uio_td->td_proc" Ns 's
address space.
User space memory must be in-core and wired prior to attempting a map
load operation.
Pages may be locked using
.Xr vslock 9 .
.It Fn bus_dmamap_unload "dmat" "map"
Unloads a DMA map.
Arguments are as follows:
.Bl -tag -width ".Fa dmam"
.It Fa dmat
DMA tag used to allocate
.Fa map .
.It Fa map
The DMA map that is to be unloaded.
.El
.Pp
.Fn bus_dmamap_unload
will not perform any implicit synchronization of DMA buffers.
This must be done explicitly by a call to
.Fn bus_dmamap_sync
prior to unloading the map.
.It Fn bus_dmamap_sync "dmat" "map" "op"
Performs synchronization of a device visible mapping with the CPU visible
memory referenced by that mapping.
Arguments are as follows:
.Bl -tag -width ".Fa dmat"
.It Fa dmat
DMA tag used to allocate
.Fa map .
.It Fa map
The DMA mapping to be synchronized.
.It Fa op
Type of synchronization operation to perform.
See the definition of
.Vt bus_dmasync_op_t
for a description of the acceptable values for
.Fa op .
.El
.Pp
The
.Fn bus_dmamap_sync
function
is the method used to ensure that CPU's and device's direct
memory access (DMA) to shared
memory is coherent.
For example, the CPU might be used to set up the contents of a buffer
that is to be made available to a device.
To ensure that the data are visible via the device's mapping of that
memory, the buffer must be loaded and a DMA sync operation of
.Dv BUS_DMASYNC_PREWRITE
must be performed after the CPU has updated the buffer and before the device
access is initiated.
If the CPU modifies this buffer again later, another
.Dv BUS_DMASYNC_PREWRITE
sync operation must be performed before an additional device
access.
Conversely, suppose a device updates memory that is to be read by a CPU.
In this case, the buffer must be loaded, and a DMA sync operation of
.Dv BUS_DMASYNC_PREREAD
must be performed before the device access is initiated.
The CPU will only be able to see the results of this memory update
once the DMA operation has completed and a
.Dv BUS_DMASYNC_POSTREAD
sync operation has been performed.
.Pp
If read and write operations are not preceded and followed by the
appropriate synchronization operations, behavior is undefined.
.It Fn bus_dmamem_alloc "dmat" "**vaddr" "flags" "*mapp"
Allocates memory that is mapped into KVA at the address returned
in
.Fa vaddr
and that is permanently loaded into the newly created
.Vt bus_dmamap_t
returned via
.Fa mapp .
Arguments are as follows:
.Bl -tag -width ".Fa alignment"
.It Fa dmat
DMA tag describing the constraints of the DMA mapping.
.It Fa vaddr
Pointer to a pointer that will hold the returned KVA mapping of
the allocated region.
.It Fa flags
Flags are defined as follows:
.Bl -tag -width ".Dv BUS_DMA_NOWAIT"
.It Dv BUS_DMA_WAITOK
The routine can safely wait (sleep) for resources.
.It Dv BUS_DMA_NOWAIT
The routine is not allowed to wait for resources.
If resources are not available,
.Dv ENOMEM
is returned.
.It Dv BUS_DMA_COHERENT
Attempt to map this memory in a coherent fashion.
See
.Fn bus_dmamap_create
above for a description of this flag.
For
.Fn bus_dmamem_alloc ,
the
.Dv BUS_DMA_COHERENT
flag is currently implemented on arm and arm64.
.It Dv BUS_DMA_ZERO
Causes the allocated memory to be set to all zeros.
.It Dv BUS_DMA_NOCACHE
The allocated memory will not be cached in the processor caches.
All memory accesses appear on the bus and are executed
without reordering.
For
.Fn bus_dmamem_alloc ,
the
.Dv BUS_DMA_NOCACHE
flag is currently implemented on amd64 and i386 where it results in the
Strong Uncacheable PAT to be set for the allocated virtual address range.
.El
.It Fa mapp
Pointer to a
.Vt bus_dmamap_t
where the resulting DMA map will be stored.
.El
.Pp
The size of memory to be allocated is
.Fa maxsize
as specified in the call to
.Fn bus_dma_tag_create
for
.Fa dmat .
.Pp
The current implementation of
.Fn bus_dmamem_alloc
will allocate all requests as a single segment.
.Pp
An initial load operation is required to obtain the bus address of the allocated
memory, and an unload operation is required before freeing the memory, as
described below in
.Fn bus_dmamem_free .
Maps are automatically handled by this function and should not be explicitly
allocated or destroyed.
.Pp
Although an explicit load is not required for each access to the memory
referenced by the returned map, the synchronization requirements
as described in the
.Fn bus_dmamap_sync
section still apply and should be used to achieve portability on architectures
without coherent buses.
.Pp
Returns
.Er ENOMEM
if sufficient memory is not available for completing
the operation.
.It Fn bus_dmamem_free "dmat" "*vaddr" "map"
Frees memory previously allocated by
.Fn bus_dmamem_alloc .
Any mappings
will be invalidated.
Arguments are as follows:
.Bl -tag -width ".Fa vaddr"
.It Fa dmat
DMA tag.
.It Fa vaddr
Kernel virtual address of the memory.
.It Fa map
DMA map to be invalidated.
.El
.El
.Sh RETURN VALUES
Behavior is undefined if invalid arguments are passed to
any of the above functions.
If sufficient resources cannot be allocated for a given
transaction,
.Er ENOMEM
is returned.
All
routines that are not of type
.Vt void
will return 0 on success or an error
code on failure as discussed above.
.Pp
All
.Vt void
routines will succeed if provided with valid arguments.
.Sh LOCKING
Two locking protocols are used by
.Nm .
The first is a private global lock that is used to synchronize access to the
bounce buffer pool on the architectures that make use of them.
This lock is strictly a leaf lock that is only used internally to
.Nm
and is not exposed to clients of the API.
.Pp
The second protocol involves protecting various resources stored in the tag.
Since almost all
.Nm
operations are done through requests from the driver that created the tag,
the most efficient way to protect the tag resources is through the lock that
the driver uses.
In cases where
.Nm
acts on its own without being called by the driver, the lock primitive
specified in the tag is acquired and released automatically.
An example of this is when the
.Fn bus_dmamap_load
callback function is called from a deferred context instead of the driver
context.
This means that certain
.Nm
functions must always be called with the same lock held that is specified in the
tag.
These functions include:
.Pp
.Bl -item -offset indent -compact
.It
.Fn bus_dmamap_load
.It
.Fn bus_dmamap_load_bio
.It
.Fn bus_dmamap_load_ccb
.It
.Fn bus_dmamap_load_mbuf
.It
.Fn bus_dmamap_load_mbuf_sg
.It
.Fn bus_dmamap_load_uio
.It
.Fn bus_dmamap_unload
.It
.Fn bus_dmamap_sync
.El
.Pp
There is one exception to this rule.
It is common practice to call some of these functions during driver start-up
without any locks held.
So long as there is a guarantee of no possible concurrent use of the tag by
different threads during this operation, it is safe to not hold a lock for
these functions.
.Pp
Certain
.Nm
operations should not be called with the driver lock held, either because
they are already protected by an internal lock, or because they might sleep
due to memory or resource allocation.
The following functions must not be
called with any non-sleepable locks held:
.Pp
.Bl -item -offset indent -compact
.It
.Fn bus_dma_tag_create
.It
.Fn bus_dmamap_create
.It
.Fn bus_dmamem_alloc
.El
.Pp
All other functions do not have a locking protocol and can thus be
called with or without any system or driver locks held.
.Sh SEE ALSO
.Xr devclass 9 ,
.Xr device 9 ,
.Xr driver 9 ,
.Xr rman 9 ,
.Xr vslock 9
.Pp
.Rs
.%A "Jason R. Thorpe"
.%T "A Machine-Independent DMA Framework for NetBSD"
.%J "Proceedings of the Summer 1998 USENIX Technical Conference"
.%Q "USENIX Association"
.%D "June 1998"
.Re
.Sh HISTORY
The
.Nm
interface first appeared in
.Nx 1.3 .
.Pp
The
.Nm
API was adopted from
.Nx
for use in the CAM SCSI subsystem.
The alterations to the original API were aimed to remove the need for
a
.Vt bus_dma_segment_t
array stored in each
.Vt bus_dmamap_t
while allowing callers to queue up on scarce resources.
.Sh AUTHORS
The
.Nm
interface was designed and implemented by
.An Jason R. Thorpe
of the Numerical Aerospace Simulation Facility, NASA Ames Research Center.
Additional input on the
.Nm
design was provided by
.An -nosplit
.An Chris Demetriou ,
.An Charles Hannum ,
.An Ross Harvey ,
.An Matthew Jacob ,
.An Jonathan Stone ,
and
.An Matt Thomas .
.Pp
The
.Nm
interface in
.Fx
benefits from the contributions of
.An Justin T. Gibbs ,
.An Peter Wemm ,
.An Doug Rabson ,
.An Matthew N. Dodd ,
.An Sam Leffler ,
.An Maxime Henrion ,
.An Jake Burkholder ,
.An Takahashi Yoshihiro ,
.An Scott Long
and many others.
.Pp
This manual page was written by
.An Hiten M. Pandya
and
.An Justin T. Gibbs .