1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #ifndef _SYS_ROOTNEX_H 26 #define _SYS_ROOTNEX_H 27 28 /* 29 * x86 root nexus implementation specific state 30 */ 31 32 #include <sys/types.h> 33 #include <sys/conf.h> 34 #include <sys/modctl.h> 35 #include <sys/sunddi.h> 36 #include <sys/iommulib.h> 37 #include <sys/sdt.h> 38 39 #ifdef __cplusplus 40 extern "C" { 41 #endif 42 43 44 /* size of buffer used for ctlop reportdev */ 45 #define REPORTDEV_BUFSIZE 1024 46 47 /* min and max interrupt vectors */ 48 #define VEC_MIN 1 49 #define VEC_MAX 255 50 51 /* atomic increment/decrement to keep track of outstanding binds, etc */ 52 #ifdef DEBUG 53 #define ROOTNEX_DPROF_INC(addr) atomic_inc_64(addr) 54 #define ROOTNEX_DPROF_DEC(addr) atomic_add_64(addr, -1) 55 #define ROOTNEX_DPROBE1(name, type1, arg1) \ 56 DTRACE_PROBE1(name, type1, arg1) 57 #define ROOTNEX_DPROBE2(name, type1, arg1, type2, arg2) \ 58 DTRACE_PROBE2(name, type1, arg1, type2, arg2) 59 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ 60 DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) 61 #else 62 #define ROOTNEX_DPROF_INC(addr) 63 #define ROOTNEX_DPROF_DEC(addr) 64 #define ROOTNEX_DPROBE1(name, type1, arg1) 65 #define ROOTNEX_DPROBE2(name, type1, arg1, type2, arg2) 66 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) 67 #endif 68 69 /* set in dmac_type to signify that this cookie uses the copy buffer */ 70 #define ROOTNEX_USES_COPYBUF 0x80000000 71 72 /* 73 * integer or boolean property name and value. A few static rootnex properties 74 * are created during rootnex attach from an array of rootnex_intprop_t.. 75 */ 76 typedef struct rootnex_intprop_s { 77 char *prop_name; 78 int prop_value; 79 } rootnex_intprop_t; 80 81 /* 82 * sgl related information which is visible to rootnex_get_sgl(). Trying to 83 * isolate get_sgl() as much as possible so it can be easily replaced. 84 */ 85 typedef struct rootnex_sglinfo_s { 86 /* 87 * These are passed into rootnex_get_sgl(). 88 * 89 * si_min_addr - the minimum physical address 90 * si_max_addr - the maximum physical address 91 * si_max_cookie_size - the maximum size of a physically contiguous 92 * piece of memory that we can handle in a sgl. 93 * si_segmask - segment mask to determine if we cross a segment boundary 94 * si_flags - dma_attr_flags 95 * si_max_pages - max number of pages this sgl could occupy (which 96 * is also the maximum number of cookies we might see. 97 */ 98 uint64_t si_min_addr; 99 uint64_t si_max_addr; 100 uint64_t si_max_cookie_size; 101 uint64_t si_segmask; 102 uint_t si_flags; 103 uint_t si_max_pages; 104 105 /* 106 * these are returned by rootnex_get_sgl() 107 * 108 * si_bounce_on_seg - if we need to use bounce buffer for pages above 109 * ddi_dma_seg 110 * si_copybuf_req - amount of copy buffer needed by the buffer. 111 * si_buf_offset - The initial offset into the first page of the buffer. 112 * It's set in get sgl and used in the bind slow path to help 113 * calculate the current page index & offset from the current offset 114 * which is relative to the start of the buffer. 115 * si_asp - address space of buffer passed in. 116 * si_sgl_size - The actual number of cookies in the sgl. This does 117 * not reflect and sharing that we might do on window boundaries. 118 */ 119 boolean_t si_bounce_on_seg; 120 size_t si_copybuf_req; 121 off_t si_buf_offset; 122 struct as *si_asp; 123 uint_t si_sgl_size; 124 } rootnex_sglinfo_t; 125 126 /* 127 * When we have to use the copy buffer, we allocate one of these structures per 128 * buffer page to track which pages need the copy buffer, what the kernel 129 * virtual address is (which the device can't reach), and what the copy buffer 130 * virtual address is (where the device dma's to/from). For 32-bit kernels, 131 * since we can't use seg kpm, we also need to keep the page_t around and state 132 * if we've currently mapped in the page into KVA space for buffers which don't 133 * have kva already and when we have multiple windows because we used up all our 134 * copy buffer space. 135 */ 136 typedef struct rootnex_pgmap_s { 137 boolean_t pm_uses_copybuf; 138 #if !defined(__amd64) 139 boolean_t pm_mapped; 140 page_t *pm_pp; 141 caddr_t pm_vaddr; 142 #endif 143 caddr_t pm_kaddr; 144 caddr_t pm_cbaddr; 145 } rootnex_pgmap_t; 146 147 /* 148 * We only need to trim a buffer when we have multiple windows. Each window has 149 * trim state. We might have trimmed the end of the previous window, leaving the 150 * first cookie of this window trimmed[tr_trim_first] (which basically means we 151 * won't start with a new cookie), or we might need to trim the end of the 152 * current window [tr_trim_last] (which basically means we won't end with a 153 * complete cookie). We keep the same state for the first & last cookie in a 154 * window (a window can have one or more cookies). However, when we trim the 155 * last cookie, we keep a pointer to the last cookie in the trim state since we 156 * only need this info when we trim. The pointer to the first cookie in the 157 * window is in the window state since we need to know what the first cookie in 158 * the window is in various places. 159 * 160 * If we do trim a cookie, we save away the physical address and size of the 161 * cookie so that we can over write the cookie when we switch windows (the 162 * space for a cookie which is in two windows is shared between the windows. 163 * We keep around the same information for the last page in a window. 164 * 165 * if we happened to trim on a page that uses the copy buffer, and that page 166 * is also in the middle of a window boundary because we have filled up the 167 * copy buffer, we need to remember the copy buffer address for both windows 168 * since the same page will have different copy buffer addresses in the two 169 * windows. We need to due the same for kaddr in the 32-bit kernel since we 170 * have a limited kva space which we map to. 171 */ 172 typedef struct rootnex_trim_s { 173 boolean_t tr_trim_first; 174 boolean_t tr_trim_last; 175 ddi_dma_cookie_t *tr_last_cookie; 176 uint64_t tr_first_paddr; 177 uint64_t tr_last_paddr; 178 size_t tr_first_size; 179 size_t tr_last_size; 180 181 boolean_t tr_first_copybuf_win; 182 boolean_t tr_last_copybuf_win; 183 uint_t tr_first_pidx; 184 uint_t tr_last_pidx; 185 caddr_t tr_first_cbaddr; 186 caddr_t tr_last_cbaddr; 187 #if !defined(__amd64) 188 caddr_t tr_first_kaddr; 189 caddr_t tr_last_kaddr; 190 #endif 191 } rootnex_trim_t; 192 193 /* 194 * per window state. A bound DMA handle can have multiple windows. Each window 195 * will have the following state. We track if this window needs to sync, 196 * the offset into the buffer where the window starts, the size of the window. 197 * a pointer to the first cookie in the window, the number of cookies in the 198 * window, and the trim state for the window. For the 32-bit kernel, we keep 199 * track of if we need to remap the copy buffer when we switch to a this window 200 */ 201 typedef struct rootnex_window_s { 202 boolean_t wd_dosync; 203 uint_t wd_cookie_cnt; 204 off_t wd_offset; 205 size_t wd_size; 206 ddi_dma_cookie_t *wd_first_cookie; 207 rootnex_trim_t wd_trim; 208 #if !defined(__amd64) 209 boolean_t wd_remap_copybuf; 210 #endif 211 } rootnex_window_t; 212 213 typedef struct dvcookie { 214 uint64_t dvck_dvma; 215 uint64_t dvck_npages; 216 } dvcookie_t; 217 218 typedef struct dcookie { 219 paddr_t dck_paddr; 220 uint64_t dck_npages; 221 } dcookie_t; 222 223 /* per dma handle private state */ 224 typedef struct rootnex_dma_s { 225 /* 226 * sgl related state used to build and describe the sgl. 227 * 228 * dp_partial_required - used in the bind slow path to identify if we 229 * need to do a partial mapping or not. 230 * dp_trim_required - used in the bind slow path to identify if we 231 * need to trim when switching to a new window. This should only be 232 * set when partial is set. 233 * dp_granularity_power_2 - set in alloc handle and used in bind slow 234 * path to determine if we & or % to calculate the trim. 235 * dp_dma - copy of dma "object" passed in during bind 236 * dp_maxxfer - trimmed dma_attr_maxxfer so that it is a whole 237 * multiple of granularity 238 * dp_sglinfo - See rootnex_sglinfo_t above. 239 */ 240 boolean_t dp_partial_required; 241 boolean_t dp_trim_required; 242 boolean_t dp_granularity_power_2; 243 uint64_t dp_maxxfer; 244 ddi_dma_obj_t dp_dma; 245 rootnex_sglinfo_t dp_sglinfo; 246 247 /* 248 * Copy buffer related state 249 * 250 * dp_copybuf_size - the actual size of the copy buffer that we are 251 * using. This can be smaller that dp_copybuf_req, i.e. bind size > 252 * max copy buffer size. 253 * dp_cbaddr - kernel address of copy buffer. Used to determine where 254 * where to copy to/from. 255 * dp_cbsize - the "real" size returned from the copy buffer alloc. 256 * Set in the copybuf alloc and used to free copybuf. 257 * dp_pgmap - page map used in sync to determine which pages in the 258 * buffer use the copy buffer and what addresses to use to copy to/ 259 * from. 260 * dp_cb_remaping - status if this bind causes us to have to remap 261 * the copybuf when switching to new windows. This is only used in 262 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 263 * this case. 264 * dp_kva - kernel heap arena vmem space for mapping to buffers which 265 * we don't have a kernel VA to bcopy to/from. This is only used in 266 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 267 * this case. 268 */ 269 size_t dp_copybuf_size; 270 caddr_t dp_cbaddr; 271 size_t dp_cbsize; 272 rootnex_pgmap_t *dp_pgmap; 273 #if !defined(__amd64) 274 boolean_t dp_cb_remaping; 275 caddr_t dp_kva; 276 #endif 277 278 /* 279 * window related state. The pointer to the window state array which may 280 * be a pointer into the pre allocated state, or we may have had to 281 * allocate the window array on the fly because it wouldn't fit. If 282 * we allocate it, we'll use dp_need_to_free_window and dp_window_size 283 * during cleanup. dp_current_win keeps track of the current window. 284 * dp_max_win is the maximum number of windows we could have. 285 */ 286 uint_t dp_current_win; 287 rootnex_window_t *dp_window; 288 boolean_t dp_need_to_free_window; 289 uint_t dp_window_size; 290 uint_t dp_max_win; 291 292 /* dip of driver which "owns" handle. set to rdip in alloc_handle() */ 293 dev_info_t *dp_dip; 294 295 /* 296 * dp_mutex and dp_inuse are only used to see if a driver is trying to 297 * bind to an already bound dma handle. dp_mutex only used for dp_inuse 298 */ 299 kmutex_t dp_mutex; 300 boolean_t dp_inuse; 301 302 /* 303 * cookie related state. The pointer to the cookies (dp_cookies) may 304 * be a pointer into the pre allocated state, or we may have had to 305 * allocate the cookie array on the fly because it wouldn't fit. If 306 * we allocate it, we'll use dp_need_to_free_cookie and dp_cookie_size 307 * during cleanup. dp_current_cookie is only used in the obsoleted 308 * interfaces to determine when we've used up all the cookies in a 309 * window during nextseg().. 310 */ 311 size_t dp_cookie_size; 312 ddi_dma_cookie_t *dp_cookies; 313 boolean_t dp_need_to_free_cookie; 314 uint_t dp_current_cookie; /* for obsoleted I/Fs */ 315 ddi_dma_cookie_t *dp_saved_cookies; 316 boolean_t dp_need_to_switch_cookies; 317 318 /* 319 * pre allocated space for the bind state, allocated during alloc 320 * handle. For a lot of devices, this will save us from having to do 321 * kmem_alloc's during the bind most of the time. kmem_alloc's can be 322 * expensive on x86 when the cpu count goes up since xcalls are 323 * expensive on x86. 324 */ 325 uchar_t *dp_prealloc_buffer; 326 327 /* 328 * Intel IOMMU (immu) related state 329 * dv_cookies saves the dvma allocated for this handler 330 * max index of dvcookies in dvmax 331 */ 332 dvcookie_t *dp_dvcookies; 333 uint64_t dp_dvmax; 334 dcookie_t *dp_dcookies; 335 uint64_t dp_dmax; 336 uint64_t dp_max_cookies; 337 uint64_t dp_max_dcookies; 338 339 /* 340 * sleep flags set on bind and unset on unbind 341 */ 342 int dp_sleep_flags; 343 } rootnex_dma_t; 344 345 /* 346 * profile/performance counters. Most things will be dtrace probes, but there 347 * are a couple of things we want to keep track all the time. We track the 348 * total number of active handles and binds (i.e. an alloc without a free or 349 * a bind without an unbind) since rootnex attach. We also track the total 350 * number of binds which have failed since rootnex attach. 351 */ 352 typedef enum { 353 ROOTNEX_CNT_ACTIVE_HDLS = 0, 354 ROOTNEX_CNT_ACTIVE_BINDS = 1, 355 ROOTNEX_CNT_ALLOC_FAIL = 2, 356 ROOTNEX_CNT_BIND_FAIL = 3, 357 ROOTNEX_CNT_SYNC_FAIL = 4, 358 ROOTNEX_CNT_GETWIN_FAIL = 5, 359 360 /* This one must be last */ 361 ROOTNEX_CNT_LAST 362 } rootnex_cnt_t; 363 364 /* 365 * global driver state. 366 * r_dmahdl_cache - dma_handle kmem_cache 367 * r_dvma_call_list_id - ddi_set_callback() id 368 * r_peekpoke_mutex - serialize peeks and pokes. 369 * r_dip - rootnex dip 370 * r_reserved_msg_printed - ctlops reserve message threshold 371 * r_counters - profile/performance counters 372 */ 373 typedef struct rootnex_state_s { 374 uint_t r_prealloc_cookies; 375 uint_t r_prealloc_size; 376 kmem_cache_t *r_dmahdl_cache; 377 uintptr_t r_dvma_call_list_id; 378 kmutex_t r_peekpoke_mutex; 379 dev_info_t *r_dip; 380 ddi_iblock_cookie_t r_err_ibc; 381 boolean_t r_reserved_msg_printed; 382 uint64_t r_counters[ROOTNEX_CNT_LAST]; 383 iommulib_nexhandle_t r_iommulib_handle; 384 } rootnex_state_t; 385 386 #ifdef __cplusplus 387 } 388 #endif 389 390 #endif /* _SYS_ROOTNEX_H */ 391