1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_PCI_DMA_H 27 #define _SYS_PCI_DMA_H 28 29 #pragma ident "%Z%%M% %I% %E% SMI" 30 31 #ifdef __cplusplus 32 extern "C" { 33 #endif 34 35 typedef pfn_t iopfn_t; 36 #define MAKE_DMA_COOKIE(cp, address, size) \ 37 { \ 38 (cp)->dmac_notused = 0; \ 39 (cp)->dmac_type = 0; \ 40 (cp)->dmac_laddress = (address); \ 41 (cp)->dmac_size = (size); \ 42 } 43 44 #define HAS_REDZONE(mp) (((mp)->dmai_rflags & DDI_DMA_REDZONE) ? 1 : 0) 45 46 #define PCI_DMA_HAT_NUM_CB_COOKIES 5 47 48 typedef struct pci_dma_hdl { 49 ddi_dma_impl_t pdh_ddi_hdl; 50 ddi_dma_attr_t pdh_attr_dev; 51 uint64_t pdh_sync_buf_pa; 52 void *pdh_cbcookie[PCI_DMA_HAT_NUM_CB_COOKIES]; 53 } pci_dma_hdl_t; 54 55 struct pci_dma_impl { /* forthdebug only, keep in sync with ddi_dma_impl_t */ 56 ulong_t dmai_mapping; 57 uint_t dmai_size; 58 off_t dmai_offset; 59 uint_t dmai_minxfer; 60 uint_t dmai_burstsizes; 61 uint_t dmai_ndvmapages; 62 uint_t dmai_roffset; 63 uint_t dmai_rflags; 64 uint_t dmai_flags; 65 uint_t dmai_nwin; 66 uint_t dmai_winsize; 67 caddr_t dmai_tte_fdvma; 68 void *dmai_pfnlst; 69 uint_t *dmai_pfn0; 70 void *dmai_winlst; 71 dev_info_t *dmai_rdip; 72 ddi_dma_obj_t dmai_object; 73 ddi_dma_attr_t dmai_attr_aug; 74 ddi_dma_cookie_t *dmai_cookie; 75 76 int (*dmai_fault_check)(struct ddi_dma_impl *handle); 77 void (*dmai_fault_notify)(struct ddi_dma_impl *handle); 78 int dmai_fault; 79 80 ddi_dma_attr_t pdh_attr_dev; 81 uint64_t pdh_sync_buf_pa; 82 void *pdh_cbcookie[PCI_DMA_HAT_NUM_CB_COOKIES]; 83 }; 84 85 /* 86 * flags for overloading dmai_inuse field of the dma request 87 * structure: 88 */ 89 #define dmai_flags dmai_inuse 90 #define dmai_tte dmai_nexus_private 91 #define dmai_fdvma dmai_nexus_private 92 #define dmai_pfnlst dmai_iopte 93 #define dmai_winlst dmai_minfo 94 #define dmai_pfn0 dmai_sbi 95 #define dmai_roffset dmai_pool 96 97 #define MP_PFN0(mp) ((iopfn_t)(mp)->dmai_pfn0) 98 #define MP_HAT_CB_COOKIE(mp, i) ((i < PCI_DMA_HAT_NUM_CB_COOKIES)? \ 99 (((pci_dma_hdl_t *)(mp))->pdh_cbcookie[i]) : NULL) 100 #define MP_HAT_CB_COOKIE_PTR(mp, i) \ 101 ((i < PCI_DMA_HAT_NUM_CB_COOKIES)? \ 102 &(((pci_dma_hdl_t *)(mp))->pdh_cbcookie[i]) : NULL) 103 #define WINLST(mp) ((pci_dma_win_t *)(mp)->dmai_winlst) 104 #define DEV_ATTR(mp) (&((pci_dma_hdl_t *)(mp))->pdh_attr_dev) 105 #define SYNC_BUF_PA(mp) (((pci_dma_hdl_t *)(mp))->pdh_sync_buf_pa) 106 #define SET_DMAATTR(p, lo, hi, nocross, cntmax) \ 107 (p)->dma_attr_addr_lo = (lo); \ 108 (p)->dma_attr_addr_hi = (hi); \ 109 (p)->dma_attr_seg = (nocross); \ 110 (p)->dma_attr_count_max = (cntmax); 111 112 #define SET_DMAALIGN(p, align) \ 113 (p)->dma_attr_align = (align); 114 115 #define DMAI_FLAGS_INUSE 0x1 116 #define DMAI_FLAGS_BYPASSREQ 0x2 117 #define DMAI_FLAGS_PEER_ONLY 0x4 118 #define DMAI_FLAGS_NOCTX 0x8 119 #define DMAI_FLAGS_DVMA 0x10 120 #define DMAI_FLAGS_BYPASS 0x20 121 #define DMAI_FLAGS_PEER_TO_PEER 0x40 122 #define DMAI_FLAGS_DMA (DMAI_FLAGS_BYPASS | DMAI_FLAGS_PEER_TO_PEER) 123 #define DMAI_FLAGS_DMA_TYPE (DMAI_FLAGS_DMA | DMAI_FLAGS_DVMA) 124 #define DMAI_FLAGS_CONTEXT 0x100 125 #define DMAI_FLAGS_FASTTRACK 0x200 126 #define DMAI_FLAGS_VMEMCACHE 0x400 127 #define DMAI_FLAGS_PGPFN 0x800 128 #define DMAI_FLAGS_NOSYSLIMIT 0x1000 129 #define DMAI_FLAGS_NOFASTLIMIT 0x2000 130 #define DMAI_FLAGS_NOSYNC 0x4000 131 #define DMAI_FLAGS_RELOC 0x8000 132 #define DMAI_FLAGS_MAPPED 0x10000 133 #define DMAI_FLAGS_PRESERVE (DMAI_FLAGS_PEER_ONLY | DMAI_FLAGS_BYPASSREQ | \ 134 DMAI_FLAGS_NOSYSLIMIT | DMAI_FLAGS_NOFASTLIMIT | DMAI_FLAGS_NOCTX) 135 136 #define HAS_NOFASTLIMIT(mp) ((mp)->dmai_flags & DMAI_FLAGS_NOFASTLIMIT) 137 #define HAS_NOSYSLIMIT(mp) ((mp)->dmai_flags & DMAI_FLAGS_NOSYSLIMIT) 138 #define PCI_DMA_ISPEERONLY(mp) ((mp)->dmai_flags & DMAI_FLAGS_PEER_ONLY) 139 #define PCI_DMA_ISPGPFN(mp) ((mp)->dmai_flags & DMAI_FLAGS_PGPFN) 140 #define PCI_DMA_TYPE(mp) ((mp)->dmai_flags & DMAI_FLAGS_DMA_TYPE) 141 #define PCI_DMA_ISDVMA(mp) (PCI_DMA_TYPE(mp) == DMAI_FLAGS_DVMA) 142 #define PCI_DMA_ISBYPASS(mp) (PCI_DMA_TYPE(mp) == DMAI_FLAGS_BYPASS) 143 #define PCI_DMA_ISPTP(mp) (PCI_DMA_TYPE(mp) == DMAI_FLAGS_PEER_TO_PEER) 144 #define PCI_DMA_CANFAST(mp) (((mp)->dmai_ndvmapages + HAS_REDZONE(mp) \ 145 <= pci_dvma_page_cache_clustsz) && HAS_NOFASTLIMIT(mp)) 146 #define PCI_DMA_WINNPGS(mp) IOMMU_BTOP((mp)->dmai_winsize) 147 #define PCI_DMA_CANCACHE(mp) (!HAS_REDZONE(mp) && \ 148 (PCI_DMA_WINNPGS(mp) == 1) && HAS_NOSYSLIMIT(mp)) 149 #define PCI_DMA_CANRELOC(mp) ((mp)->dmai_flags & DMAI_FLAGS_RELOC) 150 #define PCI_DMA_ISMAPPED(mp) ((mp)->dmai_flags & DMAI_FLAGS_MAPPED) 151 152 #define PCI_SYNC_FLAG_SZSHIFT 6 153 #define PCI_SYNC_FLAG_SIZE (1 << PCI_SYNC_FLAG_SZSHIFT) 154 #define PCI_SYNC_FLAG_FAILED 1 155 #define PCI_SYNC_FLAG_LOCKED 2 156 157 #define PCI_DMA_SYNC_DDI_FLAGS ((1 << 16) - 1) /* Look for only DDI flags */ 158 #define PCI_DMA_SYNC_EXT (1 << 30) /* enable/disable extension */ 159 #define PCI_DMA_SYNC_UNBIND (1 << 28) /* internal: part of unbind */ 160 #define PCI_DMA_SYNC_BAR (1 << 26) /* wait for all posted sync */ 161 #define PCI_DMA_SYNC_POST (1 << 25) /* post request and return */ 162 #define PCI_DMA_SYNC_PRIVATE (1 << 24) /* alloc private sync buffer */ 163 #define PCI_DMA_SYNC_DURING (1 << 22) /* sync in-progress dma */ 164 #define PCI_DMA_SYNC_BEFORE (1 << 21) /* before read or write */ 165 #define PCI_DMA_SYNC_AFTER (1 << 20) /* after read or write */ 166 #define PCI_DMA_SYNC_WRITE (1 << 17) /* data from device to mem */ 167 #define PCI_DMA_SYNC_READ (1 << 16) /* data from memory to dev */ 168 169 #define PCI_FLOW_ID_TO_PA(flow_p, flow_id) \ 170 ((flow_p)->flow_buf_pa + ((flow_id) << PCI_SYNC_FLAG_SZSHIFT)) 171 172 #define DEV_NOFASTLIMIT(lo, hi, fastlo, fasthi, align_pg) \ 173 (((lo) <= (fastlo)) && ((hi) >= (fasthi)) && \ 174 ((align_pg) <= pci_dvma_page_cache_clustsz)) 175 176 #define DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align_pg) \ 177 (((lo) <= (syslo)) && ((hi) >= (syshi)) && (align_pg == 1)) 178 179 #define PCI_DMA_NOCTX(rdip) (!pci_use_contexts || (pci_ctx_no_active_flush && \ 180 ddi_prop_exists(DDI_DEV_T_ANY, rdip, \ 181 DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "active-dma-flush"))) 182 #define PCI_DMA_USECTX(mp) (!(mp->dmai_flags & DMAI_FLAGS_NOCTX)) 183 184 #define PCI_DMA_BYPASS_PREFIX(mp, pfn) \ 185 (PCI_DMA_ISBYPASS(mp) ? COMMON_IOMMU_BYPASS_BASE | \ 186 (pf_is_memory(pfn) ? 0 : COMMON_IOMMU_BYPASS_NONCACHE) : 0) 187 #define PCI_DMA_BADPTP(pfn, attrp) \ 188 ((IOMMU_PTOB(pfn) < attrp->dma_attr_addr_lo) || \ 189 (IOMMU_PTOB(pfn) > attrp->dma_attr_addr_hi)) 190 #define PCI_DMA_CURWIN(mp) \ 191 (((mp)->dmai_offset + (mp)->dmai_roffset) / (mp)->dmai_winsize) 192 193 #ifdef PCI_DMA_PROF 194 195 /* collect fast track failure statistics */ 196 #define PCI_DVMA_FASTTRAK_PROF(mp) { \ 197 if ((mp->dmai_ndvmapages + HAS_REDZONE(mp)) > pci_dvma_page_cache_clustsz) \ 198 pci_dvmaft_npages++; \ 199 else if (!HAS_NOFASTLIMIT(mp)) \ 200 pci_dvmaft_limit++; \ 201 } 202 203 #else /* !PCI_DMA_PROF */ 204 205 #define PCI_DVMA_FASTTRAK_PROF(mp) 206 207 #endif /* PCI_DMA_PROF */ 208 209 typedef struct pci_dma_win { 210 struct pci_dma_win *win_next; 211 uint32_t win_ncookies; 212 uint32_t win_curseg; 213 uint64_t win_size; 214 uint64_t win_offset; 215 /* cookie table: sizeof (ddi_dma_cookie_t) * win_ncookies */ 216 } pci_dma_win_t; 217 218 /* dvma debug records */ 219 struct dvma_rec { 220 char *dvma_addr; 221 uint_t len; 222 ddi_dma_impl_t *mp; 223 struct dvma_rec *next; 224 }; 225 226 typedef struct pbm pbm_t; 227 extern int pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, 228 ddi_dma_handle_t handle, off_t off, size_t len, uint32_t sync_flags); 229 230 extern int pci_dma_win(dev_info_t *dip, dev_info_t *rdip, 231 ddi_dma_handle_t handle, uint_t win, off_t *offp, 232 size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp); 233 234 extern ddi_dma_impl_t *pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, 235 int (*waitfp)(caddr_t), caddr_t arg); 236 extern void pci_dma_freemp(ddi_dma_impl_t *mp); 237 extern void pci_dma_freepfn(ddi_dma_impl_t *mp); 238 extern ddi_dma_impl_t *pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, 239 iommu_t *iommu_p, ddi_dma_req_t *dmareq); 240 extern int pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp); 241 extern uint32_t pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p); 242 extern int pci_dma_type(pci_t *pci_p, ddi_dma_req_t *req, ddi_dma_impl_t *mp); 243 extern int pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *req, ddi_dma_impl_t *mp); 244 extern int pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *r, ddi_dma_impl_t *mp); 245 extern void pci_dma_freewin(ddi_dma_impl_t *mp); 246 extern int pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp); 247 extern int pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, 248 iommu_t *iommu_p); 249 extern void pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp); 250 extern void pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, 251 ddi_dma_impl_t *mp); 252 extern int pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, 253 ddi_dma_impl_t *mp); 254 extern int pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, 255 ddi_dma_impl_t *mp, enum ddi_dma_ctlops cmd, off_t *offp, 256 size_t *lenp, caddr_t *objp, uint_t cache_flags); 257 extern int pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, 258 ddi_dma_impl_t *mp, enum ddi_dma_ctlops cmd, off_t *offp, 259 size_t *lenp, caddr_t *objp, uint_t cache_flags); 260 extern void pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages, 261 int vmemcache); 262 263 #define PCI_GET_MP_NCOOKIES(mp) ((mp)->dmai_ncookies) 264 #define PCI_SET_MP_NCOOKIES(mp, nc) ((mp)->dmai_ncookies = (nc)) 265 #define PCI_GET_MP_PFN1_ADDR(mp) (((iopfn_t *)(mp)->dmai_pfnlst) + 1) 266 267 #define PCI_GET_MP_TTE(tte) \ 268 (((uint64_t)(uintptr_t)(tte) >> 5) << (32 + 5) | \ 269 ((uint32_t)(uintptr_t)(tte)) & 0x12) 270 #define PCI_SAVE_MP_TTE(mp, tte) \ 271 (mp)->dmai_tte = (caddr_t)(HI32(tte) | ((tte) & 0x12)) 272 273 #define PCI_GET_MP_PFN1(mp, page_no) (((iopfn_t *)(mp)->dmai_pfnlst)[page_no]) 274 #define PCI_GET_MP_PFN(mp, page_no) ((mp)->dmai_ndvmapages == 1 ? \ 275 (iopfn_t)(mp)->dmai_pfnlst : PCI_GET_MP_PFN1(mp, page_no)) 276 277 #define PCI_SET_MP_PFN(mp, page_no, pfn) { \ 278 if ((mp)->dmai_ndvmapages == 1) { \ 279 ASSERT(!((page_no) || (mp)->dmai_pfnlst)); \ 280 (mp)->dmai_pfnlst = (void *)(pfn); \ 281 } else \ 282 ((iopfn_t *)(mp)->dmai_pfnlst)[page_no] = (iopfn_t)(pfn); \ 283 } 284 #define PCI_SET_MP_PFN1(mp, page_no, pfn) { \ 285 ((iopfn_t *)(mp)->dmai_pfnlst)[page_no] = (pfn); \ 286 } 287 288 #define GET_TTE_TEMPLATE(mp) MAKE_TTE_TEMPLATE(PCI_GET_MP_PFN((mp), 0), (mp)) 289 290 extern int pci_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 291 ddi_dma_handle_t handle); 292 293 int pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t handle); 294 295 #if defined(DEBUG) 296 extern void dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp); 297 #else 298 #define dump_dma_handle(flag, dip, hp) 299 #endif 300 301 #ifdef __cplusplus 302 } 303 #endif 304 305 #endif /* _SYS_PCI_DMA_H */ 306