1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_ROOTNEX_H 27 #define _SYS_ROOTNEX_H 28 29 /* 30 * x86 root nexus implementation specific state 31 */ 32 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/modctl.h> 36 #include <sys/sunddi.h> 37 #include <sys/iommulib.h> 38 #include <sys/sdt.h> 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 45 /* size of buffer used for ctlop reportdev */ 46 #define REPORTDEV_BUFSIZE 1024 47 48 /* min and max interrupt vectors */ 49 #define VEC_MIN 1 50 #define VEC_MAX 255 51 52 /* atomic increment/decrement to keep track of outstanding binds, etc */ 53 #ifdef DEBUG 54 #define ROOTNEX_DPROF_INC(addr) atomic_inc_64(addr) 55 #define ROOTNEX_DPROF_DEC(addr) atomic_add_64(addr, -1) 56 #define ROOTNEX_DPROBE1(name, type1, arg1) \ 57 DTRACE_PROBE1(name, type1, arg1) 58 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ 59 DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) 60 #else 61 #define ROOTNEX_DPROF_INC(addr) 62 #define ROOTNEX_DPROF_DEC(addr) 63 #define ROOTNEX_DPROBE1(name, type1, arg1) 64 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) 65 #endif 66 #define ROOTNEX_PROF_INC(addr) atomic_inc_64(addr) 67 #define ROOTNEX_PROF_DEC(addr) atomic_add_64(addr, -1) 68 69 /* set in dmac_type to signify that this cookie uses the copy buffer */ 70 #define ROOTNEX_USES_COPYBUF 0x80000000 71 72 /* 73 * integer or boolean property name and value. A few static rootnex properties 74 * are created during rootnex attach from an array of rootnex_intprop_t.. 75 */ 76 typedef struct rootnex_intprop_s { 77 char *prop_name; 78 int prop_value; 79 } rootnex_intprop_t; 80 81 /* 82 * sgl related information which is visible to rootnex_get_sgl(). Trying to 83 * isolate get_sgl() as much as possible so it can be easily replaced. 84 */ 85 typedef struct rootnex_sglinfo_s { 86 /* 87 * These are passed into rootnex_get_sgl(). 88 * 89 * si_min_addr - the minimum physical address 90 * si_max_addr - the maximum physical address 91 * si_max_cookie_size - the maximum size of a physically contiguous 92 * piece of memory that we can handle in a sgl. 93 * si_segmask - segment mask to determine if we cross a segment boundary 94 * si_max_pages - max number of pages this sgl could occupy (which 95 * is also the maximum number of cookies we might see. 96 */ 97 uint64_t si_min_addr; 98 uint64_t si_max_addr; 99 uint64_t si_max_cookie_size; 100 uint64_t si_segmask; 101 uint_t si_max_pages; 102 103 /* 104 * these are returned by rootnex_get_sgl() 105 * 106 * si_copybuf_req - amount of copy buffer needed by the buffer. 107 * si_buf_offset - The initial offset into the first page of the buffer. 108 * It's set in get sgl and used in the bind slow path to help 109 * calculate the current page index & offset from the current offset 110 * which is relative to the start of the buffer. 111 * si_asp - address space of buffer passed in. 112 * si_sgl_size - The actual number of cookies in the sgl. This does 113 * not reflect and sharing that we might do on window boundaries. 114 */ 115 size_t si_copybuf_req; 116 off_t si_buf_offset; 117 struct as *si_asp; 118 uint_t si_sgl_size; 119 } rootnex_sglinfo_t; 120 121 /* 122 * When we have to use the copy buffer, we allocate one of these structures per 123 * buffer page to track which pages need the copy buffer, what the kernel 124 * virtual address is (which the device can't reach), and what the copy buffer 125 * virtual address is (where the device dma's to/from). For 32-bit kernels, 126 * since we can't use seg kpm, we also need to keep the page_t around and state 127 * if we've currently mapped in the page into KVA space for buffers which don't 128 * have kva already and when we have multiple windows because we used up all our 129 * copy buffer space. 130 */ 131 typedef struct rootnex_pgmap_s { 132 boolean_t pm_uses_copybuf; 133 #if !defined(__amd64) 134 boolean_t pm_mapped; 135 page_t *pm_pp; 136 caddr_t pm_vaddr; 137 #endif 138 caddr_t pm_kaddr; 139 caddr_t pm_cbaddr; 140 } rootnex_pgmap_t; 141 142 /* 143 * We only need to trim a buffer when we have multiple windows. Each window has 144 * trim state. We might have trimmed the end of the previous window, leaving the 145 * first cookie of this window trimmed[tr_trim_first] (which basically means we 146 * won't start with a new cookie), or we might need to trim the end of the 147 * current window [tr_trim_last] (which basically means we won't end with a 148 * complete cookie). We keep the same state for the first & last cookie in a 149 * window (a window can have one or more cookies). However, when we trim the 150 * last cookie, we keep a pointer to the last cookie in the trim state since we 151 * only need this info when we trim. The pointer to the first cookie in the 152 * window is in the window state since we need to know what the first cookie in 153 * the window is in various places. 154 * 155 * If we do trim a cookie, we save away the physical address and size of the 156 * cookie so that we can over write the cookie when we switch windows (the 157 * space for a cookie which is in two windows is shared between the windows. 158 * We keep around the same information for the last page in a window. 159 * 160 * if we happened to trim on a page that uses the copy buffer, and that page 161 * is also in the middle of a window boundary because we have filled up the 162 * copy buffer, we need to remember the copy buffer address for both windows 163 * since the same page will have different copy buffer addresses in the two 164 * windows. We need to due the same for kaddr in the 32-bit kernel since we 165 * have a limited kva space which we map to. 166 */ 167 typedef struct rootnex_trim_s { 168 boolean_t tr_trim_first; 169 boolean_t tr_trim_last; 170 ddi_dma_cookie_t *tr_last_cookie; 171 uint64_t tr_first_paddr; 172 uint64_t tr_last_paddr; 173 size_t tr_first_size; 174 size_t tr_last_size; 175 176 boolean_t tr_first_copybuf_win; 177 boolean_t tr_last_copybuf_win; 178 uint_t tr_first_pidx; 179 uint_t tr_last_pidx; 180 caddr_t tr_first_cbaddr; 181 caddr_t tr_last_cbaddr; 182 #if !defined(__amd64) 183 caddr_t tr_first_kaddr; 184 caddr_t tr_last_kaddr; 185 #endif 186 } rootnex_trim_t; 187 188 /* 189 * per window state. A bound DMA handle can have multiple windows. Each window 190 * will have the following state. We track if this window needs to sync, 191 * the offset into the buffer where the window starts, the size of the window. 192 * a pointer to the first cookie in the window, the number of cookies in the 193 * window, and the trim state for the window. For the 32-bit kernel, we keep 194 * track of if we need to remap the copy buffer when we switch to a this window 195 */ 196 typedef struct rootnex_window_s { 197 boolean_t wd_dosync; 198 uint_t wd_cookie_cnt; 199 off_t wd_offset; 200 size_t wd_size; 201 ddi_dma_cookie_t *wd_first_cookie; 202 rootnex_trim_t wd_trim; 203 #if !defined(__amd64) 204 boolean_t wd_remap_copybuf; 205 #endif 206 } rootnex_window_t; 207 208 typedef struct dvcookie { 209 uint64_t dvck_dvma; 210 uint64_t dvck_npages; 211 uint64_t dvck_sidx; 212 uint64_t dvck_eidx; 213 } dvcookie_t; 214 215 typedef struct dcookie { 216 paddr_t dck_paddr; 217 uint64_t dck_npages; 218 } dcookie_t; 219 220 /* per dma handle private state */ 221 typedef struct rootnex_dma_s { 222 /* 223 * sgl related state used to build and describe the sgl. 224 * 225 * dp_partial_required - used in the bind slow path to identify if we 226 * need to do a partial mapping or not. 227 * dp_trim_required - used in the bind slow path to identify if we 228 * need to trim when switching to a new window. This should only be 229 * set when partial is set. 230 * dp_granularity_power_2 - set in alloc handle and used in bind slow 231 * path to determine if we & or % to calculate the trim. 232 * dp_dma - copy of dma "object" passed in during bind 233 * dp_maxxfer - trimmed dma_attr_maxxfer so that it is a whole 234 * multiple of granularity 235 * dp_sglinfo - See rootnex_sglinfo_t above. 236 */ 237 boolean_t dp_partial_required; 238 boolean_t dp_trim_required; 239 boolean_t dp_granularity_power_2; 240 uint64_t dp_maxxfer; 241 ddi_dma_obj_t dp_dma; 242 rootnex_sglinfo_t dp_sglinfo; 243 244 /* 245 * Copy buffer related state 246 * 247 * dp_copybuf_size - the actual size of the copy buffer that we are 248 * using. This can be smaller that dp_copybuf_req, i.e. bind size > 249 * max copy buffer size. 250 * dp_cbaddr - kernel address of copy buffer. Used to determine where 251 * where to copy to/from. 252 * dp_cbsize - the "real" size returned from the copy buffer alloc. 253 * Set in the copybuf alloc and used to free copybuf. 254 * dp_pgmap - page map used in sync to determine which pages in the 255 * buffer use the copy buffer and what addresses to use to copy to/ 256 * from. 257 * dp_cb_remaping - status if this bind causes us to have to remap 258 * the copybuf when switching to new windows. This is only used in 259 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 260 * this case. 261 * dp_kva - kernel heap arena vmem space for mapping to buffers which 262 * we don't have a kernel VA to bcopy to/from. This is only used in 263 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 264 * this case. 265 */ 266 size_t dp_copybuf_size; 267 caddr_t dp_cbaddr; 268 size_t dp_cbsize; 269 rootnex_pgmap_t *dp_pgmap; 270 #if !defined(__amd64) 271 boolean_t dp_cb_remaping; 272 caddr_t dp_kva; 273 #endif 274 275 /* 276 * window related state. The pointer to the window state array which may 277 * be a pointer into the pre allocated state, or we may have had to 278 * allocate the window array on the fly because it wouldn't fit. If 279 * we allocate it, we'll use dp_need_to_free_window and dp_window_size 280 * during cleanup. dp_current_win keeps track of the current window. 281 * dp_max_win is the maximum number of windows we could have. 282 */ 283 uint_t dp_current_win; 284 rootnex_window_t *dp_window; 285 boolean_t dp_need_to_free_window; 286 uint_t dp_window_size; 287 uint_t dp_max_win; 288 289 /* dip of driver which "owns" handle. set to rdip in alloc_handle() */ 290 dev_info_t *dp_dip; 291 292 /* 293 * dp_mutex and dp_inuse are only used to see if a driver is trying to 294 * bind to an already bound dma handle. dp_mutex only used for dp_inuse 295 */ 296 kmutex_t dp_mutex; 297 boolean_t dp_inuse; 298 299 /* 300 * cookie related state. The pointer to the cookies (dp_cookies) may 301 * be a pointer into the pre allocated state, or we may have had to 302 * allocate the cookie array on the fly because it wouldn't fit. If 303 * we allocate it, we'll use dp_need_to_free_cookie and dp_cookie_size 304 * during cleanup. dp_current_cookie is only used in the obsoleted 305 * interfaces to determine when we've used up all the cookies in a 306 * window during nextseg().. 307 */ 308 size_t dp_cookie_size; 309 ddi_dma_cookie_t *dp_cookies; 310 boolean_t dp_need_to_free_cookie; 311 uint_t dp_current_cookie; /* for obsoleted I/Fs */ 312 ddi_dma_cookie_t *dp_saved_cookies; 313 boolean_t dp_need_to_switch_cookies; 314 315 /* 316 * pre allocated space for the bind state, allocated during alloc 317 * handle. For a lot of devices, this will save us from having to do 318 * kmem_alloc's during the bind most of the time. kmem_alloc's can be 319 * expensive on x86 when the cpu count goes up since xcalls are 320 * expensive on x86. 321 */ 322 uchar_t *dp_prealloc_buffer; 323 324 /* 325 * Intel IOMMU (immu) related state 326 * dv_cookies saves the dvma allocated for this handler 327 * max index of dvcookies in dvmax 328 */ 329 dvcookie_t *dp_dvcookies; 330 uint64_t dp_dvmax; 331 dcookie_t *dp_dcookies; 332 uint64_t dp_dmax; 333 uint64_t dp_max_cookies; 334 uint64_t dp_max_dcookies; 335 336 /* 337 * sleep flags set on bind and unset on unbind 338 */ 339 int dp_sleep_flags; 340 } rootnex_dma_t; 341 342 /* 343 * profile/performance counters. Most things will be dtrace probes, but there 344 * are a couple of things we want to keep track all the time. We track the 345 * total number of active handles and binds (i.e. an alloc without a free or 346 * a bind without an unbind) since rootnex attach. We also track the total 347 * number of binds which have failed since rootnex attach. 348 */ 349 typedef enum { 350 ROOTNEX_CNT_ACTIVE_HDLS = 0, 351 ROOTNEX_CNT_ACTIVE_BINDS = 1, 352 ROOTNEX_CNT_ALLOC_FAIL = 2, 353 ROOTNEX_CNT_BIND_FAIL = 3, 354 ROOTNEX_CNT_SYNC_FAIL = 4, 355 ROOTNEX_CNT_GETWIN_FAIL = 5, 356 357 /* This one must be last */ 358 ROOTNEX_CNT_LAST 359 } rootnex_cnt_t; 360 361 /* 362 * global driver state. 363 * r_dmahdl_cache - dma_handle kmem_cache 364 * r_dvma_call_list_id - ddi_set_callback() id 365 * r_peekpoke_mutex - serialize peeks and pokes. 366 * r_dip - rootnex dip 367 * r_reserved_msg_printed - ctlops reserve message threshold 368 * r_counters - profile/performance counters 369 */ 370 typedef struct rootnex_state_s { 371 uint_t r_prealloc_cookies; 372 uint_t r_prealloc_size; 373 kmem_cache_t *r_dmahdl_cache; 374 uintptr_t r_dvma_call_list_id; 375 kmutex_t r_peekpoke_mutex; 376 dev_info_t *r_dip; 377 ddi_iblock_cookie_t r_err_ibc; 378 boolean_t r_reserved_msg_printed; 379 uint64_t r_counters[ROOTNEX_CNT_LAST]; 380 iommulib_nexhandle_t r_iommulib_handle; 381 } rootnex_state_t; 382 383 #ifdef __cplusplus 384 } 385 #endif 386 387 #endif /* _SYS_ROOTNEX_H */ 388