1.\" Copyright (c) 2002, 2003 Hiten M. Pandya. 2.\" All rights reserved. 3.\" 4.\" Redistribution and use in source and binary forms, with or without 5.\" modification, are permitted provided that the following conditions 6.\" are met: 7.\" 1. Redistributions of source code must retain the above copyright 8.\" notice, this list of conditions, and the following disclaimer, 9.\" without modification, immediately at the beginning of the file. 10.\" 2. The name of the author may not be used to endorse or promote products 11.\" derived from this software without specific prior written permission. 12.\" 13.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR, CONTRIBUTORS OR THE 17.\" VOICES IN HITEN PANDYA'S HEAD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 19.\" TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20.\" PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 21.\" LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 22.\" NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23.\" SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24.\" 25.\" Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. 26.\" All rights reserved. 27.\" 28.\" This code is derived from software contributed to The NetBSD Foundation 29.\" by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 30.\" NASA Ames Research Center. 31.\" 32.\" Redistribution and use in source and binary forms, with or without 33.\" modification, are permitted provided that the following conditions 34.\" are met: 35.\" 1. Redistributions of source code must retain the above copyright 36.\" notice, this list of conditions and the following disclaimer. 37.\" 2. Redistributions in binary form must reproduce the above copyright 38.\" notice, this list of conditions and the following disclaimer in the 39.\" documentation and/or other materials provided with the distribution. 40.\" 41.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 42.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 43.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 44.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 45.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 46.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 47.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 48.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 49.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 50.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 51.\" POSSIBILITY OF SUCH DAMAGE. 52.\" $NetBSD: bus_dma.9,v 1.25 2002/10/14 13:43:16 wiz Exp $ 53.\" 54.Dd May 25, 2020 55.Dt BUS_DMA 9 56.Os 57.Sh NAME 58.Nm bus_dma , 59.Nm bus_dma_tag_create , 60.Nm bus_dma_tag_destroy , 61.Nm bus_dma_template_init , 62.Nm bus_dma_template_tag , 63.Nm bus_dma_template_clone , 64.Nm bus_dma_template_fill , 65.Nm BUS_DMA_TEMPLATE_FILL , 66.Nm bus_dmamap_create , 67.Nm bus_dmamap_destroy , 68.Nm bus_dmamap_load , 69.Nm bus_dmamap_load_bio , 70.Nm bus_dmamap_load_ccb , 71.Nm bus_dmamap_load_crp , 72.Nm bus_dmamap_load_crp_buffer , 73.Nm bus_dmamap_load_mbuf , 74.Nm bus_dmamap_load_mbuf_sg , 75.Nm bus_dmamap_load_uio , 76.Nm bus_dmamap_unload , 77.Nm bus_dmamap_sync , 78.Nm bus_dmamem_alloc , 79.Nm bus_dmamem_free 80.Nd Bus and Machine Independent DMA Mapping Interface 81.Sh SYNOPSIS 82.In machine/bus.h 83.Ft int 84.Fn bus_dma_tag_create "bus_dma_tag_t parent" "bus_size_t alignment" \ 85"bus_addr_t boundary" "bus_addr_t lowaddr" "bus_addr_t highaddr" \ 86"bus_dma_filter_t *filtfunc" "void *filtfuncarg" "bus_size_t maxsize" \ 87"int nsegments" "bus_size_t maxsegsz" "int flags" "bus_dma_lock_t *lockfunc" \ 88"void *lockfuncarg" "bus_dma_tag_t *dmat" 89.Ft int 90.Fn bus_dma_tag_destroy "bus_dma_tag_t dmat" 91.Ft void 92.Fo bus_dma_template_init 93.Fa "bus_dma_template_t *template" 94.Fa "bus_dma_tag_t parent" 95.Fc 96.Ft int 97.Fo bus_dma_template_tag 98.Fa "bus_dma_template_t *template" 99.Fa "bus_dma_tag_t *dmat" 100.Fc 101.Ft void 102.Fo bus_dma_template_clone 103.Fa "bus_dma_template_t *template" 104.Fa "bus_dma_tag_t dmat" 105.Fc 106.Ft void 107.Fo bus_dma_template_fill 108.Fa "bus_dma_template_t *template" 109.Fa "bus_dma_param_t params[]" 110.Fa "u_int count" 111.Fc 112.Fo BUS_DMA_TEMPLATE_FILL 113.Fa "bus_dma_template_t *template" 114.Fa "bus_dma_param_t param ..." 115.Fc 116.Ft int 117.Fn bus_dmamap_create "bus_dma_tag_t dmat" "int flags" "bus_dmamap_t *mapp" 118.Ft int 119.Fn bus_dmamap_destroy "bus_dma_tag_t dmat" "bus_dmamap_t map" 120.Ft int 121.Fn bus_dmamap_load "bus_dma_tag_t dmat" "bus_dmamap_t map" "void *buf" \ 122"bus_size_t buflen" "bus_dmamap_callback_t *callback" "void *callback_arg" \ 123"int flags" 124.Ft int 125.Fn bus_dmamap_load_bio "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 126"struct bio *bio" "bus_dmamap_callback_t *callback" "void *callback_arg" \ 127"int flags" 128.Ft int 129.Fn bus_dmamap_load_ccb "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 130"union ccb *ccb" "bus_dmamap_callback_t *callback" "void *callback_arg" \ 131"int flags" 132.Ft int 133.Fn bus_dmamap_load_crp "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 134"struct crypto *crp" "bus_dmamap_callback_t *callback" "void *callback_arg" \ 135"int flags" 136.Ft int 137.Fn bus_dmamap_load_crp_buffer "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 138"struct crypto_buffer *cb" "bus_dmamap_callback_t *callback" \ 139"void *callback_arg" "int flags" 140.Ft int 141.Fn bus_dmamap_load_mbuf "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 142"struct mbuf *mbuf" "bus_dmamap_callback2_t *callback" "void *callback_arg" \ 143"int flags" 144.Ft int 145.Fn bus_dmamap_load_mbuf_sg "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 146"struct mbuf *mbuf" "bus_dma_segment_t *segs" "int *nsegs" "int flags" 147.Ft int 148.Fn bus_dmamap_load_uio "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 149"struct uio *uio" "bus_dmamap_callback2_t *callback" "void *callback_arg" \ 150"int flags" 151.Ft void 152.Fn bus_dmamap_unload "bus_dma_tag_t dmat" "bus_dmamap_t map" 153.Ft void 154.Fn bus_dmamap_sync "bus_dma_tag_t dmat" "bus_dmamap_t map" \ 155"op" 156.Ft int 157.Fn bus_dmamem_alloc "bus_dma_tag_t dmat" "void **vaddr" \ 158"int flags" "bus_dmamap_t *mapp" 159.Ft void 160.Fn bus_dmamem_free "bus_dma_tag_t dmat" "void *vaddr" \ 161"bus_dmamap_t map" 162.Sh DESCRIPTION 163Direct Memory Access (DMA) is a method of transferring data 164without involving the CPU, thus providing higher performance. 165A DMA transaction can be achieved between device to memory, 166device to device, or memory to memory. 167.Pp 168The 169.Nm 170API is a bus, device, and machine-independent (MI) interface to 171DMA mechanisms. 172It provides the client with flexibility and simplicity by 173abstracting machine dependent issues like setting up 174DMA mappings, handling cache issues, bus specific features 175and limitations. 176.Sh OVERVIEW 177A tag structure 178.Vt ( bus_dma_tag_t ) 179is used to describe the properties of a group of related DMA 180transactions. 181One way to view this is that a tag describes the limitations of a DMA engine. 182For example, if a DMA engine in a device is limited to 32-bit addresses, 183that limitation is specified by a parameter when creating the tag 184for that device. 185Similarly, a tag can be marked as requiring buffers whose addresses are 186aligned to a specific boundary. 187.Pp 188Some devices may require multiple tags to describe DMA 189transactions with differing properties. 190For example, a device might require 16-byte alignment of its descriptor ring 191while permitting arbitrary alignment of I/O buffers. 192In this case, 193the driver must create one tag for the descriptor ring and a separate tag for 194I/O buffers. 195If a device has restrictions that are common to all DMA transactions 196in addition to restrictions that differ between unrelated groups of 197transactions, 198the driver can first create a 199.Dq parent 200tag that decribes the common restrictions. 201The per-group tags can then inherit these restrictions from this 202.Dq parent 203tag rather than having to list them explicitly when creating the per-group tags. 204.Pp 205A mapping structure 206.Vt ( bus_dmamap_t ) 207represents a mapping of a memory region for DMA. 208On systems with I/O MMUs, 209the mapping structure tracks any I/O MMU entries used by a request. 210For DMA requests that require bounce pages, 211the mapping tracks the bounce pages used. 212.Pp 213To prepare for one or more DMA transactions, 214a mapping must be bound to a memory region by calling one of the 215.Fn bus_dmamap_load 216functions. 217These functions configure the mapping which can include programming entries 218in an I/O MMU and/or allocating bounce pages. 219An output of these functions 220(either directly or indirectly by invoking a callback routine) 221is the list of scatter/gather address ranges a consumer can pass to a DMA 222engine to access the memory region. 223When a mapping is no longer needed, 224the mapping must be unloaded via 225.Fn bus_dmamap_unload . 226.Pp 227Before and after each DMA transaction, 228.Fn bus_dmamap_sync 229must be used to ensure that the correct data is used by the DMA engine and 230the CPU. 231If a mapping uses bounce pages, 232the sync operations copy data between the bounce pages and the memory region 233bound to the mapping. 234Sync operations also handle architecture-specific details such as CPU cache 235flushing and CPU memory operation ordering. 236.Sh STATIC VS DYNAMIC 237.Nm 238handles two types of DMA transactions: static and dynamic. 239Static transactions are used with a long-lived memory region that is reused 240for many transactions such as a descriptor ring. 241Dynamic transactions are used for transfers to or from transient buffers 242such as I/O buffers holding a network packet or disk block. 243Each transaction type uses a different subset of the 244.Nm 245API. 246.Ss Static Transactions 247Static transactions use memory regions allocated by 248.Nm . 249Each static memory region is allocated by calling 250.Fn bus_dmamem_alloc . 251This function requires a valid tag describing the properties of the 252DMA transactions to this region such as alignment or address restrictions. 253Multiple regions can share a single tag if they share the same restrictions. 254.Pp 255.Fn bus_dmamem_alloc 256allocates a memory region along with a mapping object. 257The associated tag, memory region, and mapping object must then be passed to 258.Fn bus_dmamap_load 259to bind the mapping to the allocated region and obtain the 260scatter/gather list. 261.Pp 262It is expected that 263.Fn bus_dmamem_alloc 264will attempt to allocate memory requiring less expensive sync operations 265(for example, implementations should not allocate regions requiring bounce 266pages), 267but sync operations should still be used. 268For example, a driver should use 269.Fn bus_dmamap_sync 270in an interrupt handler before reading descriptor ring entries written by the 271device prior to the interrupt. 272.Pp 273When a consumer is finished with a memory region, 274it should unload the mapping via 275.Fn bus_dmamap_unload 276and then release the memory region and mapping object via 277.Fn bus_dmamem_free . 278.Ss Dynamic Transactions 279Dynamic transactions map memory regions provided by other parts of the system. 280A tag must be created via 281.Fn bus_dma_tag_create 282to describe the DMA transactions to and from these memory regions, 283and a pool of mapping objects must be allocated via 284.Fn bus_dmamap_create 285to track the mappings of any in-flight transactions. 286.Pp 287When a consumer wishes to schedule a transaction for a memory region, 288the consumer must first obtain an unused mapping object from its pool 289of mapping objects. 290The memory region must be bound to the mapping object via one of the 291.Fn bus_dmamap_load 292functions. 293Before scheduling the transaction, 294the consumer should sync the memory region via 295.Fn bus_dmamap_sync 296with one or more of the 297.Dq PRE 298flags. 299After the transaction has completed, 300the consumer should sync the memory region via 301.Fn bus_dmamap_sync 302with one or more of the 303.Dq POST 304flags. 305The mapping can then be unloaded via 306.Fn bus_dmamap_unload , 307and the mapping object can be returned to the pool of unused mapping objects. 308.Pp 309When a consumer is no longer scheduling DMA transactions, 310the mapping objects should be freed via 311.Fn bus_dmamap_destroy , 312and the tag should be freed via 313.Fn bus_dma_tag_destroy . 314.Sh STRUCTURES AND TYPES 315.Bl -tag -width indent 316.It Vt bus_dma_tag_t 317A machine-dependent (MD) opaque type that describes the 318characteristics of a group of DMA transactions. 319DMA tags are organized into a hierarchy, with each child 320tag inheriting the restrictions of its parent. 321This allows all devices along the path of DMA transactions 322to contribute to the constraints of those transactions. 323.It Vt bus_dma_template_t 324A template is a structure for creating a 325.Fa bus_dma_tag_t 326from a set of defaults. 327Once initialized with 328.Fn bus_dma_template_init , 329a driver can over-ride individual fields to suit its needs. 330The following fields start with the indicated default values: 331.Bd -literal 332 alignment 1 333 boundary 0 334 lowaddr BUS_SPACE_MAXADDR 335 highaddr BUS_SPACE_MAXADDR 336 maxsize BUS_SPACE_MAXSIZE 337 nsegments BUS_SPACE_UNRESTRICTED 338 maxsegsize BUS_SPACE_MAXSIZE 339 flags 0 340 lockfunc NULL 341 lockfuncarg NULL 342.Ed 343.Pp 344Descriptions of each field are documented with 345.Fn bus_dma_tag_create . 346Note that the 347.Fa filtfunc 348and 349.Fa filtfuncarg 350attributes of the DMA tag are not supported with templates. 351.It Vt bus_dma_filter_t 352Client specified address filter having the format: 353.Bl -tag -width indent 354.It Ft int 355.Fn "client_filter" "void *filtarg" "bus_addr_t testaddr" 356.El 357.Pp 358Address filters can be specified during tag creation to allow 359for devices whose DMA address restrictions cannot be specified 360by a single window. 361The 362.Fa filtarg 363argument is specified by the client during tag creation to be passed to all 364invocations of the callback. 365The 366.Fa testaddr 367argument contains a potential starting address of a DMA mapping. 368The filter function operates on the set of addresses from 369.Fa testaddr 370to 371.Ql trunc_page(testaddr) + PAGE_SIZE - 1 , 372inclusive. 373The filter function should return zero if any mapping in this range 374can be accommodated by the device and non-zero otherwise. 375.Pp 376.Em Note: The use of filters is no longer supported and will result in an error. 377.It Vt bus_dma_segment_t 378A machine-dependent type that describes individual 379DMA segments. 380It contains the following fields: 381.Bd -literal 382 bus_addr_t ds_addr; 383 bus_size_t ds_len; 384.Ed 385.Pp 386The 387.Fa ds_addr 388field contains the device visible address of the DMA segment, and 389.Fa ds_len 390contains the length of the DMA segment. 391Although the DMA segments returned by a mapping call will adhere to 392all restrictions necessary for a successful DMA operation, some conversion 393(e.g.\& a conversion from host byte order to the device's byte order) is 394almost always required when presenting segment information to the device. 395.It Vt bus_dmamap_t 396A machine-dependent opaque type describing an individual mapping. 397One map is used for each memory allocation that will be loaded. 398Maps can be reused once they have been unloaded. 399Multiple maps can be associated with one DMA tag. 400While the value of the map may evaluate to 401.Dv NULL 402on some platforms under certain conditions, 403it should never be assumed that it will be 404.Dv NULL 405in all cases. 406.It Vt bus_dmamap_callback_t 407Client specified callback for receiving mapping information resulting from 408the load of a 409.Vt bus_dmamap_t 410via 411.Fn bus_dmamap_load , 412.Fn bus_dmamap_load_bio , 413.Fn bus_dmamap_load_ccb , 414.Fn bus_dmamap_load_crp , 415or 416.Fn bus_dmamap_load_crp_buffer . 417Callbacks are of the format: 418.Bl -tag -width indent 419.It Ft void 420.Fn "client_callback" "void *callback_arg" "bus_dma_segment_t *segs" \ 421"int nseg" "int error" 422.El 423.Pp 424The 425.Fa callback_arg 426is the callback argument passed to dmamap load functions. 427The 428.Fa segs 429and 430.Fa nseg 431arguments describe an array of 432.Vt bus_dma_segment_t 433structures that represent the mapping. 434This array is only valid within the scope of the callback function. 435The success or failure of the mapping is indicated by the 436.Fa error 437argument. 438More information on the use of callbacks can be found in the 439description of the individual dmamap load functions. 440.It Vt bus_dmamap_callback2_t 441Client specified callback for receiving mapping information resulting from 442the load of a 443.Vt bus_dmamap_t 444via 445.Fn bus_dmamap_load_uio 446or 447.Fn bus_dmamap_load_mbuf . 448.Pp 449Callback2s are of the format: 450.Bl -tag -width indent 451.It Ft void 452.Fn "client_callback2" "void *callback_arg" "bus_dma_segment_t *segs" \ 453"int nseg" "bus_size_t mapsize" "int error" 454.El 455.Pp 456Callback2's behavior is the same as 457.Vt bus_dmamap_callback_t 458with the addition that the length of the data mapped is provided via 459.Fa mapsize . 460.It Vt bus_dmasync_op_t 461Memory synchronization operation specifier. 462Bus DMA requires explicit synchronization of memory with its device 463visible mapping in order to guarantee memory coherency. 464The 465.Vt bus_dmasync_op_t 466allows the type of DMA operation that will be or has been performed 467to be communicated to the system so that the correct coherency measures 468are taken. 469The operations are represented as bitfield flags that can be combined together, 470though it only makes sense to combine PRE flags or POST flags, not both. 471See the 472.Fn bus_dmamap_sync 473description below for more details on how to use these operations. 474.Pp 475All operations specified below are performed from the host memory point of view, 476where a read implies data coming from the device to the host memory, and a write 477implies data going from the host memory to the device. 478Alternatively, the operations can be thought of in terms of driver operations, 479where reading a network packet or storage sector corresponds to a read operation 480in 481.Nm . 482.Bl -tag -width ".Dv BUS_DMASYNC_POSTWRITE" 483.It Dv BUS_DMASYNC_PREREAD 484Perform any synchronization required prior to an update of host memory by the 485device. 486.It Dv BUS_DMASYNC_PREWRITE 487Perform any synchronization required after an update of host memory by the CPU 488and prior to device access to host memory. 489.It Dv BUS_DMASYNC_POSTREAD 490Perform any synchronization required after an update of host memory by the 491device and prior to CPU access to host memory. 492.It Dv BUS_DMASYNC_POSTWRITE 493Perform any synchronization required after device access to host memory. 494.El 495.It Vt bus_dma_lock_t 496Client specified lock/mutex manipulation method. 497This will be called from 498within busdma whenever a client lock needs to be manipulated. 499In its current form, the function will be called immediately before 500the callback for a DMA load operation that has been deferred with 501.Dv BUS_DMA_LOCK 502and immediately after with 503.Dv BUS_DMA_UNLOCK . 504If the load operation does not need to be deferred, then it 505will not be called since the function loading the map should 506be holding the appropriate locks. 507This method is of the format: 508.Bl -tag -width indent 509.It Ft void 510.Fn "lockfunc" "void *lockfunc_arg" "bus_dma_lock_op_t op" 511.El 512.Pp 513The 514.Fa lockfuncarg 515argument is specified by the client during tag creation to be passed to all 516invocations of the callback. 517The 518.Fa op 519argument specifies the lock operation to perform. 520.Pp 521Two 522.Vt lockfunc 523implementations are provided for convenience. 524.Fn busdma_lock_mutex 525performs standard mutex operations on the sleep mutex provided via 526.Fa lockfuncarg . 527.Fn dflt_lock 528will generate a system panic if it is called. 529It is substituted into the tag when 530.Fa lockfunc 531is passed as 532.Dv NULL 533to 534.Fn bus_dma_tag_create 535and is useful for tags that should not be used with deferred load operations. 536.It Vt bus_dma_lock_op_t 537Operations to be performed by the client-specified 538.Fn lockfunc . 539.Bl -tag -width ".Dv BUS_DMA_UNLOCK" 540.It Dv BUS_DMA_LOCK 541Acquires and/or locks the client locking primitive. 542.It Dv BUS_DMA_UNLOCK 543Releases and/or unlocks the client locking primitive. 544.El 545.El 546.Sh FUNCTIONS 547.Bl -tag -width indent 548.It Fn bus_dma_tag_create "parent" "alignment" "boundary" "lowaddr" \ 549"highaddr" "*filtfunc" "*filtfuncarg" "maxsize" "nsegments" "maxsegsz" \ 550"flags" "lockfunc" "lockfuncarg" "*dmat" 551Allocates a DMA tag, and initializes it according to 552the arguments provided: 553.Bl -tag -width ".Fa filtfuncarg" 554.It Fa parent 555A parent tag from which to inherit restrictions. 556The restrictions passed in other arguments can only further tighten the 557restrictions inherited from the parent tag. 558.Pp 559All tags created by a device driver must inherit from the tag returned by 560.Fn bus_get_dma_tag 561to honor restrictions between the parent bridge, CPU memory, and the 562device. 563.It Fa alignment 564Alignment constraint, in bytes, of any mappings created using this tag. 565The alignment must be a power of 2. 566Hardware that can DMA starting at any address would specify 567.Em 1 568for byte alignment. 569Hardware requiring DMA transfers to start on a multiple of 4K 570would specify 571.Em 4096 . 572.It Fa boundary 573Boundary constraint, in bytes, of the target DMA memory region. 574The boundary indicates the set of addresses, all multiples of the 575boundary argument, that cannot be crossed by a single 576.Vt bus_dma_segment_t . 577The boundary must be a power of 2 and must be no smaller than the 578maximum segment size. 579.Ql 0 580indicates that there are no boundary restrictions. 581.It Fa lowaddr , highaddr 582Bounds of the window of bus address space that 583.Em cannot 584be directly accessed by the device. 585The window contains all addresses greater than 586.Fa lowaddr 587and less than or equal to 588.Fa highaddr . 589For example, a device incapable of DMA above 4GB, would specify a 590.Fa highaddr 591of 592.Dv BUS_SPACE_MAXADDR 593and a 594.Fa lowaddr 595of 596.Dv BUS_SPACE_MAXADDR_32BIT . 597Similarly a device that can only perform DMA to addresses below 59816MB would specify a 599.Fa highaddr 600of 601.Dv BUS_SPACE_MAXADDR 602and a 603.Fa lowaddr 604of 605.Dv BUS_SPACE_MAXADDR_24BIT . 606Some implementations require that some region of device visible 607address space, overlapping available host memory, be outside the 608window. 609This area of 610.Ql safe memory 611is used to bounce requests that would otherwise conflict with 612the exclusion window. 613.It Fa filtfunc 614Formerly the optional filter function; must be 615.Dv NULL . 616.It Fa filtfuncarg 617Must be 618.Dv NULL . 619.It Fa maxsize 620Maximum size, in bytes, of the sum of all segment lengths in a given 621DMA mapping associated with this tag. 622.It Fa nsegments 623Number of discontinuities (scatter/gather segments) allowed 624in a DMA mapped region. 625.It Fa maxsegsz 626Maximum size, in bytes, of a segment in any DMA mapped region associated 627with 628.Fa dmat . 629.It Fa flags 630Are as follows: 631.Bl -tag -width ".Dv BUS_DMA_ALLOCNOW" 632.It Dv BUS_DMA_ALLOCNOW 633Pre-allocate enough resources to handle at least one map load operation on 634this tag. 635If sufficient resources are not available, 636.Er ENOMEM 637is returned. 638This should not be used for tags that only describe buffers that will be 639allocated with 640.Fn bus_dmamem_alloc . 641Also, due to resource sharing with other tags, this flag does not guarantee 642that resources will be allocated or reserved exclusively for this tag. 643It should be treated only as a minor optimization. 644.It Dv BUS_DMA_COHERENT 645Indicate that the DMA engine and CPU are cache-coherent. 646Cached memory may be used to back allocations created by 647.Fn bus_dmamem_alloc . 648For 649.Fn bus_dma_tag_create , 650the 651.Dv BUS_DMA_COHERENT 652flag is currently implemented on arm64. 653.El 654.It Fa lockfunc 655Optional lock manipulation function (may be 656.Dv NULL ) 657to be called when busdma 658needs to manipulate a lock on behalf of the client. 659If 660.Dv NULL 661is specified, 662.Fn dflt_lock 663is used. 664.It Fa lockfuncarg 665Optional argument to be passed to the function specified by 666.Fa lockfunc . 667.It Fa dmat 668Pointer to a bus_dma_tag_t where the resulting DMA tag will 669be stored. 670.El 671.Pp 672Returns 673.Er ENOMEM 674if sufficient memory is not available for tag creation 675or allocating mapping resources. 676Returns 677.Er EINVAL 678if either 679.Fa filtfunc 680or 681.Fa filtarg 682arguments are not 683.Dv NULL . 684.It Fn bus_dma_tag_destroy "dmat" 685Deallocate the DMA tag 686.Fa dmat 687that was created by 688.Fn bus_dma_tag_create . 689.Pp 690Returns 691.Er EBUSY 692if any DMA maps remain associated with 693.Fa dmat 694or 695.Ql 0 696on success. 697.It Fn bus_dma_template_init "*template" "parent" 698Initializes a 699.Fa bus_dma_template_t 700structure. 701If the 702.Fa parent 703argument is non-NULL, this parent tag is associated with the template and 704will be compiled into the dma tag that is later created. 705The values of the parent are not copied into the template. 706During tag creation in 707.Fn bus_dma_tag_template , 708any parameters from the parent tag that are more restrictive than what is 709in the provided template will overwrite what goes into the new tag. 710.It Fn bus_dma_template_tag "*template" "*dmat" 711Unpacks a template into a tag, and returns the tag via the 712.Fa dmat . 713All return values are identical to 714.Fn bus_dma_tag_create . 715The template is not modified by this function, and can be reused and/or 716freed upon return. 717.It Fn bus_dma_template_clone "*template" "dmat" 718Copies the fields from an existing tag to a template. 719The template does not need to be initialized first. 720All of its fields will be overwritten by the values contained in the tag. 721When paired with 722.Fn bus_dma_template_tag , 723this function is useful for creating copies of tags. 724.It Fn bus_dma_template_fill "*template" "params[]" "count" 725Fills in the selected fields of the template with the keyed values from the 726.Fa params 727array. 728This is not meant to be called directly, use 729.Fn BUS_DMA_TEMPLATE_FILL 730instead. 731.It Fn BUS_DMA_TEMPLATE_FILL "*template" "param ..." 732Fills in the selected fields of the template with a variable number of 733key-value parameters. 734The macros listed below take an argument of the specified type and encapsulate 735it into a key-value structure that is directly usable as a parameter argument. 736Muliple parameters may be provided at once. 737.Bd -literal 738 BD_PARENT() void * 739 BD_ALIGNMENT() uintmax_t 740 BD_BOUNDARY() uintmax_t 741 BD_LOWADDR() vm_paddr_t 742 BD_HIGHADDR() vm_paddr_t 743 BD_MAXSIZE() uintmax_t 744 BD_NSEGMENTS() uintmax_t 745 BD_MAXSEGSIZE() uintmax_t 746 BD_FLAGS() uintmax_t 747 BD_LOCKFUNC() void * 748 BD_LOCKFUNCARG() void * 749.Ed 750.It Fn bus_dmamap_create "dmat" "flags" "*mapp" 751Allocates and initializes a DMA map. 752Arguments are as follows: 753.Bl -tag -width ".Fa nsegments" 754.It Fa dmat 755DMA tag. 756.It Fa flags 757Are as follows: 758.Bl -tag -width ".Dv BUS_DMA_COHERENT" 759.It Dv BUS_DMA_COHERENT 760Attempt to map the memory loaded with this map such that cache sync 761operations are as cheap as possible. 762This flag is typically set on maps when the memory loaded with these will 763be accessed by both a CPU and a DMA engine, frequently such as control data 764and as opposed to streamable data such as receive and transmit buffers. 765Use of this flag does not remove the requirement of using 766.Fn bus_dmamap_sync , 767but it may reduce the cost of performing these operations. 768.El 769.It Fa mapp 770Pointer to a 771.Vt bus_dmamap_t 772where the resulting DMA map will be stored. 773.El 774.Pp 775Returns 776.Er ENOMEM 777if sufficient memory is not available for creating the 778map or allocating mapping resources. 779.It Fn bus_dmamap_destroy "dmat" "map" 780Frees all resources associated with a given DMA map. 781Arguments are as follows: 782.Bl -tag -width ".Fa dmat" 783.It Fa dmat 784DMA tag used to allocate 785.Fa map . 786.It Fa map 787The DMA map to destroy. 788.El 789.Pp 790Returns 791.Er EBUSY 792if a mapping is still active for 793.Fa map . 794.It Fn bus_dmamap_load "dmat" "map" "buf" "buflen" "*callback" \ 795"callback_arg" "flags" 796Creates a mapping in device visible address space of 797.Fa buflen 798bytes of 799.Fa buf , 800associated with the DMA map 801.Fa map . 802This call will always return immediately and will not block for any reason. 803Arguments are as follows: 804.Bl -tag -width ".Fa buflen" 805.It Fa dmat 806DMA tag used to allocate 807.Fa map . 808.It Fa map 809A DMA map without a currently active mapping. 810.It Fa buf 811A kernel virtual address pointer to a contiguous (in KVA) buffer, to be 812mapped into device visible address space. 813.It Fa buflen 814The size of the buffer. 815.It Fa callback Fa callback_arg 816The callback function, and its argument. 817This function is called once sufficient mapping resources are available for 818the DMA operation. 819If resources are temporarily unavailable, this function will be deferred until 820later, but the load operation will still return immediately to the caller. 821Thus, callers should not assume that the callback will be called before the 822load returns, and code should be structured appropriately to handle this. 823See below for specific flags and error codes that control this behavior. 824.It Fa flags 825Are as follows: 826.Bl -tag -width ".Dv BUS_DMA_NOWAIT" 827.It Dv BUS_DMA_NOWAIT 828The load should not be deferred in case of insufficient mapping resources, 829and instead should return immediately with an appropriate error. 830.It Dv BUS_DMA_NOCACHE 831The generated transactions to and from the virtual page are non-cacheable. 832.El 833.El 834.Pp 835Return values to the caller are as follows: 836.Bl -tag -width ".Er EINPROGRESS" 837.It 0 838The callback has been called and completed. 839The status of the mapping has been delivered to the callback. 840.It Er EINPROGRESS 841The mapping has been deferred for lack of resources. 842The callback will be called as soon as resources are available. 843Callbacks are serviced in FIFO order. 844.Pp 845Note that subsequent load operations for the same tag that do not require 846extra resources will still succeed. 847This may result in out-of-order processing of requests. 848If the caller requires the order of requests to be preserved, 849then the caller is required to stall subsequent requests until a pending 850request's callback is invoked. 851.It Er ENOMEM 852The load request has failed due to insufficient resources, and the caller 853specifically used the 854.Dv BUS_DMA_NOWAIT 855flag. 856.It Er EINVAL 857The load request was invalid. 858The callback has been called and has been provided the same error. 859This error value may indicate that 860.Fa dmat , 861.Fa map , 862.Fa buf , 863or 864.Fa callback 865were invalid, or 866.Fa buflen 867was larger than the 868.Fa maxsize 869argument used to create the dma tag 870.Fa dmat . 871.El 872.Pp 873When the callback is called, it is presented with an error value 874indicating the disposition of the mapping. 875Error may be one of the following: 876.Bl -tag -width ".Er EINPROGRESS" 877.It 0 878The mapping was successful and the 879.Fa dm_segs 880callback argument contains an array of 881.Vt bus_dma_segment_t 882elements describing the mapping. 883This array is only valid during the scope of the callback function. 884.It Er EFBIG 885A mapping could not be achieved within the segment constraints provided 886in the tag even though the requested allocation size was less than maxsize. 887.El 888.It Fn bus_dmamap_load_bio "dmat" "map" "bio" "callback" "callback_arg" "flags" 889This is a variation of 890.Fn bus_dmamap_load 891which maps buffers pointed to by 892.Fa bio 893for DMA transfers. 894.Fa bio 895may point to either a mapped or unmapped buffer. 896.It Fn bus_dmamap_load_ccb "dmat" "map" "ccb" "callback" "callback_arg" "flags" 897This is a variation of 898.Fn bus_dmamap_load 899which maps data pointed to by 900.Fa ccb 901for DMA transfers. 902The data for 903.Fa ccb 904may be any of the following types: 905.Bl -tag -width ".Er CAM_DATA_SG_PADDR" 906.It CAM_DATA_VADDR 907The data is a single KVA buffer. 908.It CAM_DATA_PADDR 909The data is a single bus address range. 910.It CAM_DATA_SG 911The data is a scatter/gather list of KVA buffers. 912.It CAM_DATA_SG_PADDR 913The data is a scatter/gather list of bus address ranges. 914.It CAM_DATA_BIO 915The data is contained in a 916.Vt struct bio 917attached to the CCB. 918.El 919.Pp 920.Fn bus_dmamap_load_ccb 921supports the following CCB XPT function codes: 922.Pp 923.Bl -item -offset indent -compact 924.It 925XPT_ATA_IO 926.It 927XPT_CONT_TARGET_IO 928.It 929XPT_SCSI_IO 930.El 931.It Fn bus_dmamap_load_crp "dmat" "map" "crp" "callback" "callback_arg" "flags" 932This is a variation of 933.Fn bus_dmamap_load 934which maps the input buffer pointed to by 935.Fa crp 936for DMA transfers. 937The 938.Dv BUS_DMA_NOWAIT 939flag is implied, thus no callback deferral will happen. 940.It Fn bus_dmamap_load_crp_buffer "dmat" "map" "cb" "callback" "callback_arg" \ 941"flags" 942This is a variation of 943.Fn bus_dmamap_load 944which maps the crypto data buffer pointed to by 945.Fa cb 946for DMA transfers. 947The 948.Dv BUS_DMA_NOWAIT 949flag is implied, thus no callback deferral will happen. 950.It Fn bus_dmamap_load_mbuf "dmat" "map" "mbuf" "callback2" "callback_arg" \ 951"flags" 952This is a variation of 953.Fn bus_dmamap_load 954which maps mbuf chains 955for DMA transfers. 956A 957.Vt bus_size_t 958argument is also passed to the callback routine, which 959contains the mbuf chain's packet header length. 960The 961.Dv BUS_DMA_NOWAIT 962flag is implied, thus no callback deferral will happen. 963.Pp 964Mbuf chains are assumed to be in kernel virtual address space. 965.Pp 966Beside the error values listed for 967.Fn bus_dmamap_load , 968.Er EINVAL 969will be returned if the size of the mbuf chain exceeds the maximum limit of the 970DMA tag. 971.It Fn bus_dmamap_load_mbuf_sg "dmat" "map" "mbuf" "segs" "nsegs" "flags" 972This is just like 973.Fn bus_dmamap_load_mbuf 974except that it returns immediately without calling a callback function. 975It is provided for efficiency. 976The scatter/gather segment array 977.Va segs 978is provided by the caller and filled in directly by the function. 979The 980.Va nsegs 981argument is returned with the number of segments filled in. 982Returns the same errors as 983.Fn bus_dmamap_load_mbuf . 984.It Fn bus_dmamap_load_uio "dmat" "map" "uio" "callback2" "callback_arg" "flags" 985This is a variation of 986.Fn bus_dmamap_load 987which maps buffers pointed to by 988.Fa uio 989for DMA transfers. 990A 991.Vt bus_size_t 992argument is also passed to the callback routine, which contains the size of 993.Fa uio , 994i.e. 995.Fa uio->uio_resid . 996The 997.Dv BUS_DMA_NOWAIT 998flag is implied, thus no callback deferral will happen. 999Returns the same errors as 1000.Fn bus_dmamap_load . 1001.Pp 1002If 1003.Fa uio->uio_segflg 1004is 1005.Dv UIO_USERSPACE , 1006then it is assumed that the buffer, 1007.Fa uio 1008is in 1009.Fa "uio->uio_td->td_proc" Ns 's 1010address space. 1011User space memory must be in-core and wired prior to attempting a map 1012load operation. 1013Pages may be locked using 1014.Xr vslock 9 . 1015.It Fn bus_dmamap_unload "dmat" "map" 1016Unloads a DMA map. 1017Arguments are as follows: 1018.Bl -tag -width ".Fa dmam" 1019.It Fa dmat 1020DMA tag used to allocate 1021.Fa map . 1022.It Fa map 1023The DMA map that is to be unloaded. 1024.El 1025.Pp 1026.Fn bus_dmamap_unload 1027will not perform any implicit synchronization of DMA buffers. 1028This must be done explicitly by a call to 1029.Fn bus_dmamap_sync 1030prior to unloading the map. 1031.It Fn bus_dmamap_sync "dmat" "map" "op" 1032Performs synchronization of a device visible mapping with the CPU visible 1033memory referenced by that mapping. 1034Arguments are as follows: 1035.Bl -tag -width ".Fa dmat" 1036.It Fa dmat 1037DMA tag used to allocate 1038.Fa map . 1039.It Fa map 1040The DMA mapping to be synchronized. 1041.It Fa op 1042Type of synchronization operation to perform. 1043See the definition of 1044.Vt bus_dmasync_op_t 1045for a description of the acceptable values for 1046.Fa op . 1047.El 1048.Pp 1049The 1050.Fn bus_dmamap_sync 1051function 1052is the method used to ensure that CPU's and device's direct 1053memory access (DMA) to shared 1054memory is coherent. 1055For example, the CPU might be used to set up the contents of a buffer 1056that is to be made available to a device. 1057To ensure that the data are visible via the device's mapping of that 1058memory, the buffer must be loaded and a DMA sync operation of 1059.Dv BUS_DMASYNC_PREWRITE 1060must be performed after the CPU has updated the buffer and before the device 1061access is initiated. 1062If the CPU modifies this buffer again later, another 1063.Dv BUS_DMASYNC_PREWRITE 1064sync operation must be performed before an additional device 1065access. 1066Conversely, suppose a device updates memory that is to be read by a CPU. 1067In this case, the buffer must be loaded, and a DMA sync operation of 1068.Dv BUS_DMASYNC_PREREAD 1069must be performed before the device access is initiated. 1070The CPU will only be able to see the results of this memory update 1071once the DMA operation has completed and a 1072.Dv BUS_DMASYNC_POSTREAD 1073sync operation has been performed. 1074.Pp 1075If read and write operations are not preceded and followed by the 1076appropriate synchronization operations, behavior is undefined. 1077.It Fn bus_dmamem_alloc "dmat" "**vaddr" "flags" "*mapp" 1078Allocates memory that is mapped into KVA at the address returned 1079in 1080.Fa vaddr 1081and that is permanently loaded into the newly created 1082.Vt bus_dmamap_t 1083returned via 1084.Fa mapp . 1085Arguments are as follows: 1086.Bl -tag -width ".Fa alignment" 1087.It Fa dmat 1088DMA tag describing the constraints of the DMA mapping. 1089.It Fa vaddr 1090Pointer to a pointer that will hold the returned KVA mapping of 1091the allocated region. 1092.It Fa flags 1093Flags are defined as follows: 1094.Bl -tag -width ".Dv BUS_DMA_NOWAIT" 1095.It Dv BUS_DMA_WAITOK 1096The routine can safely wait (sleep) for resources. 1097.It Dv BUS_DMA_NOWAIT 1098The routine is not allowed to wait for resources. 1099If resources are not available, 1100.Dv ENOMEM 1101is returned. 1102.It Dv BUS_DMA_COHERENT 1103Attempt to map this memory in a coherent fashion. 1104See 1105.Fn bus_dmamap_create 1106above for a description of this flag. 1107For 1108.Fn bus_dmamem_alloc , 1109the 1110.Dv BUS_DMA_COHERENT 1111flag is currently implemented on arm and arm64. 1112.It Dv BUS_DMA_ZERO 1113Causes the allocated memory to be set to all zeros. 1114.It Dv BUS_DMA_NOCACHE 1115The allocated memory will not be cached in the processor caches. 1116All memory accesses appear on the bus and are executed 1117without reordering. 1118For 1119.Fn bus_dmamem_alloc , 1120the 1121.Dv BUS_DMA_NOCACHE 1122flag is currently implemented on amd64 and i386 where it results in the 1123Strong Uncacheable PAT to be set for the allocated virtual address range. 1124.El 1125.It Fa mapp 1126Pointer to a 1127.Vt bus_dmamap_t 1128where the resulting DMA map will be stored. 1129.El 1130.Pp 1131The size of memory to be allocated is 1132.Fa maxsize 1133as specified in the call to 1134.Fn bus_dma_tag_create 1135for 1136.Fa dmat . 1137.Pp 1138The current implementation of 1139.Fn bus_dmamem_alloc 1140will allocate all requests as a single segment. 1141.Pp 1142An initial load operation is required to obtain the bus address of the allocated 1143memory, and an unload operation is required before freeing the memory, as 1144described below in 1145.Fn bus_dmamem_free . 1146Maps are automatically handled by this function and should not be explicitly 1147allocated or destroyed. 1148.Pp 1149Although an explicit load is not required for each access to the memory 1150referenced by the returned map, the synchronization requirements 1151as described in the 1152.Fn bus_dmamap_sync 1153section still apply and should be used to achieve portability on architectures 1154without coherent buses. 1155.Pp 1156Returns 1157.Er ENOMEM 1158if sufficient memory is not available for completing 1159the operation. 1160.It Fn bus_dmamem_free "dmat" "*vaddr" "map" 1161Frees memory previously allocated by 1162.Fn bus_dmamem_alloc . 1163Any mappings 1164will be invalidated. 1165Arguments are as follows: 1166.Bl -tag -width ".Fa vaddr" 1167.It Fa dmat 1168DMA tag. 1169.It Fa vaddr 1170Kernel virtual address of the memory. 1171.It Fa map 1172DMA map to be invalidated. 1173.El 1174.El 1175.Sh RETURN VALUES 1176Behavior is undefined if invalid arguments are passed to 1177any of the above functions. 1178If sufficient resources cannot be allocated for a given 1179transaction, 1180.Er ENOMEM 1181is returned. 1182All 1183routines that are not of type 1184.Vt void 1185will return 0 on success or an error 1186code on failure as discussed above. 1187.Pp 1188All 1189.Vt void 1190routines will succeed if provided with valid arguments. 1191.Sh LOCKING 1192Two locking protocols are used by 1193.Nm . 1194The first is a private global lock that is used to synchronize access to the 1195bounce buffer pool on the architectures that make use of them. 1196This lock is strictly a leaf lock that is only used internally to 1197.Nm 1198and is not exposed to clients of the API. 1199.Pp 1200The second protocol involves protecting various resources stored in the tag. 1201Since almost all 1202.Nm 1203operations are done through requests from the driver that created the tag, 1204the most efficient way to protect the tag resources is through the lock that 1205the driver uses. 1206In cases where 1207.Nm 1208acts on its own without being called by the driver, the lock primitive 1209specified in the tag is acquired and released automatically. 1210An example of this is when the 1211.Fn bus_dmamap_load 1212callback function is called from a deferred context instead of the driver 1213context. 1214This means that certain 1215.Nm 1216functions must always be called with the same lock held that is specified in the 1217tag. 1218These functions include: 1219.Pp 1220.Bl -item -offset indent -compact 1221.It 1222.Fn bus_dmamap_load 1223.It 1224.Fn bus_dmamap_load_bio 1225.It 1226.Fn bus_dmamap_load_ccb 1227.It 1228.Fn bus_dmamap_load_mbuf 1229.It 1230.Fn bus_dmamap_load_mbuf_sg 1231.It 1232.Fn bus_dmamap_load_uio 1233.It 1234.Fn bus_dmamap_unload 1235.It 1236.Fn bus_dmamap_sync 1237.El 1238.Pp 1239There is one exception to this rule. 1240It is common practice to call some of these functions during driver start-up 1241without any locks held. 1242So long as there is a guarantee of no possible concurrent use of the tag by 1243different threads during this operation, it is safe to not hold a lock for 1244these functions. 1245.Pp 1246Certain 1247.Nm 1248operations should not be called with the driver lock held, either because 1249they are already protected by an internal lock, or because they might sleep 1250due to memory or resource allocation. 1251The following functions must not be 1252called with any non-sleepable locks held: 1253.Pp 1254.Bl -item -offset indent -compact 1255.It 1256.Fn bus_dma_tag_create 1257.It 1258.Fn bus_dmamap_create 1259.It 1260.Fn bus_dmamem_alloc 1261.El 1262.Pp 1263All other functions do not have a locking protocol and can thus be 1264called with or without any system or driver locks held. 1265.Sh SEE ALSO 1266.Xr devclass 9 , 1267.Xr device 9 , 1268.Xr driver 9 , 1269.Xr rman 9 , 1270.Xr vslock 9 1271.Pp 1272.Rs 1273.%A "Jason R. Thorpe" 1274.%T "A Machine-Independent DMA Framework for NetBSD" 1275.%J "Proceedings of the Summer 1998 USENIX Technical Conference" 1276.%Q "USENIX Association" 1277.%D "June 1998" 1278.Re 1279.Sh HISTORY 1280The 1281.Nm 1282interface first appeared in 1283.Nx 1.3 . 1284.Pp 1285The 1286.Nm 1287API was adopted from 1288.Nx 1289for use in the CAM SCSI subsystem. 1290The alterations to the original API were aimed to remove the need for 1291a 1292.Vt bus_dma_segment_t 1293array stored in each 1294.Vt bus_dmamap_t 1295while allowing callers to queue up on scarce resources. 1296.Sh AUTHORS 1297The 1298.Nm 1299interface was designed and implemented by 1300.An Jason R. Thorpe 1301of the Numerical Aerospace Simulation Facility, NASA Ames Research Center. 1302Additional input on the 1303.Nm 1304design was provided by 1305.An -nosplit 1306.An Chris Demetriou , 1307.An Charles Hannum , 1308.An Ross Harvey , 1309.An Matthew Jacob , 1310.An Jonathan Stone , 1311and 1312.An Matt Thomas . 1313.Pp 1314The 1315.Nm 1316interface in 1317.Fx 1318benefits from the contributions of 1319.An Justin T. Gibbs , 1320.An Peter Wemm , 1321.An Doug Rabson , 1322.An Matthew N. Dodd , 1323.An Sam Leffler , 1324.An Maxime Henrion , 1325.An Jake Burkholder , 1326.An Takahashi Yoshihiro , 1327.An Scott Long 1328and many others. 1329.Pp 1330This manual page was written by 1331.An Hiten M. Pandya 1332and 1333.An Justin T. Gibbs . 1334