1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_ROOTNEX_H 27 #define _SYS_ROOTNEX_H 28 29 /* 30 * x86 root nexus implementation specific state 31 */ 32 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/modctl.h> 36 #include <sys/sunddi.h> 37 #include <sys/iommulib.h> 38 #include <sys/sdt.h> 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 45 /* size of buffer used for ctlop reportdev */ 46 #define REPORTDEV_BUFSIZE 1024 47 48 /* min and max interrupt vectors */ 49 #define VEC_MIN 1 50 #define VEC_MAX 255 51 52 /* atomic increment/decrement to keep track of outstanding binds, etc */ 53 #ifdef DEBUG 54 #define ROOTNEX_DPROF_INC(addr) atomic_inc_64(addr) 55 #define ROOTNEX_DPROF_DEC(addr) atomic_add_64(addr, -1) 56 #define ROOTNEX_DPROBE1(name, type1, arg1) \ 57 DTRACE_PROBE1(name, type1, arg1) 58 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ 59 DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) 60 #else 61 #define ROOTNEX_DPROF_INC(addr) 62 #define ROOTNEX_DPROF_DEC(addr) 63 #define ROOTNEX_DPROBE1(name, type1, arg1) 64 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) 65 #endif 66 #define ROOTNEX_PROF_INC(addr) atomic_inc_64(addr) 67 #define ROOTNEX_PROF_DEC(addr) atomic_add_64(addr, -1) 68 69 /* set in dmac_type to signify that this cookie uses the copy buffer */ 70 #define ROOTNEX_USES_COPYBUF 0x80000000 71 72 /* 73 * integer or boolean property name and value. A few static rootnex properties 74 * are created during rootnex attach from an array of rootnex_intprop_t.. 75 */ 76 typedef struct rootnex_intprop_s { 77 char *prop_name; 78 int prop_value; 79 } rootnex_intprop_t; 80 81 /* 82 * sgl related information which is visible to rootnex_get_sgl(). Trying to 83 * isolate get_sgl() as much as possible so it can be easily replaced. 84 */ 85 typedef struct rootnex_sglinfo_s { 86 /* 87 * These are passed into rootnex_get_sgl(). 88 * 89 * si_min_addr - the minimum physical address 90 * si_max_addr - the maximum physical address 91 * si_max_cookie_size - the maximum size of a physically contiguous 92 * piece of memory that we can handle in a sgl. 93 * si_segmask - segment mask to determine if we cross a segment boundary 94 * si_max_pages - max number of pages this sgl could occupy (which 95 * is also the maximum number of cookies we might see. 96 */ 97 uint64_t si_min_addr; 98 uint64_t si_max_addr; 99 uint64_t si_max_cookie_size; 100 uint64_t si_segmask; 101 uint_t si_max_pages; 102 103 /* 104 * these are returned by rootnex_get_sgl() 105 * 106 * si_copybuf_req - amount of copy buffer needed by the buffer. 107 * si_buf_offset - The initial offset into the first page of the buffer. 108 * It's set in get sgl and used in the bind slow path to help 109 * calculate the current page index & offset from the current offset 110 * which is relative to the start of the buffer. 111 * si_asp - address space of buffer passed in. 112 * si_sgl_size - The actual number of cookies in the sgl. This does 113 * not reflect and sharing that we might do on window boundaries. 114 */ 115 size_t si_copybuf_req; 116 off_t si_buf_offset; 117 struct as *si_asp; 118 uint_t si_sgl_size; 119 } rootnex_sglinfo_t; 120 121 /* 122 * When we have to use the copy buffer, we allocate one of these structures per 123 * buffer page to track which pages need the copy buffer, what the kernel 124 * virtual address is (which the device can't reach), and what the copy buffer 125 * virtual address is (where the device dma's to/from). For 32-bit kernels, 126 * since we can't use seg kpm, we also need to keep the page_t around and state 127 * if we've currently mapped in the page into KVA space for buffers which don't 128 * have kva already and when we have multiple windows because we used up all our 129 * copy buffer space. 130 */ 131 typedef struct rootnex_pgmap_s { 132 boolean_t pm_uses_copybuf; 133 #if !defined(__amd64) 134 boolean_t pm_mapped; 135 page_t *pm_pp; 136 caddr_t pm_vaddr; 137 #endif 138 caddr_t pm_kaddr; 139 caddr_t pm_cbaddr; 140 } rootnex_pgmap_t; 141 142 /* 143 * We only need to trim a buffer when we have multiple windows. Each window has 144 * trim state. We might have trimmed the end of the previous window, leaving the 145 * first cookie of this window trimmed[tr_trim_first] (which basically means we 146 * won't start with a new cookie), or we might need to trim the end of the 147 * current window [tr_trim_last] (which basically means we won't end with a 148 * complete cookie). We keep the same state for the first & last cookie in a 149 * window (a window can have one or more cookies). However, when we trim the 150 * last cookie, we keep a pointer to the last cookie in the trim state since we 151 * only need this info when we trim. The pointer to the first cookie in the 152 * window is in the window state since we need to know what the first cookie in 153 * the window is in various places. 154 * 155 * If we do trim a cookie, we save away the physical address and size of the 156 * cookie so that we can over write the cookie when we switch windows (the 157 * space for a cookie which is in two windows is shared between the windows. 158 * We keep around the same information for the last page in a window. 159 * 160 * if we happened to trim on a page that uses the copy buffer, and that page 161 * is also in the middle of a window boundary because we have filled up the 162 * copy buffer, we need to remember the copy buffer address for both windows 163 * since the same page will have different copy buffer addresses in the two 164 * windows. We need to due the same for kaddr in the 32-bit kernel since we 165 * have a limited kva space which we map to. 166 */ 167 typedef struct rootnex_trim_s { 168 boolean_t tr_trim_first; 169 boolean_t tr_trim_last; 170 ddi_dma_cookie_t *tr_last_cookie; 171 uint64_t tr_first_paddr; 172 uint64_t tr_last_paddr; 173 size_t tr_first_size; 174 size_t tr_last_size; 175 176 boolean_t tr_first_copybuf_win; 177 boolean_t tr_last_copybuf_win; 178 uint_t tr_first_pidx; 179 uint_t tr_last_pidx; 180 caddr_t tr_first_cbaddr; 181 caddr_t tr_last_cbaddr; 182 #if !defined(__amd64) 183 caddr_t tr_first_kaddr; 184 caddr_t tr_last_kaddr; 185 #endif 186 } rootnex_trim_t; 187 188 /* 189 * per window state. A bound DMA handle can have multiple windows. Each window 190 * will have the following state. We track if this window needs to sync, 191 * the offset into the buffer where the window starts, the size of the window. 192 * a pointer to the first cookie in the window, the number of cookies in the 193 * window, and the trim state for the window. For the 32-bit kernel, we keep 194 * track of if we need to remap the copy buffer when we switch to a this window 195 */ 196 typedef struct rootnex_window_s { 197 boolean_t wd_dosync; 198 uint_t wd_cookie_cnt; 199 off_t wd_offset; 200 size_t wd_size; 201 ddi_dma_cookie_t *wd_first_cookie; 202 rootnex_trim_t wd_trim; 203 #if !defined(__amd64) 204 boolean_t wd_remap_copybuf; 205 #endif 206 } rootnex_window_t; 207 208 typedef struct dvcookie { 209 uint64_t dvck_dvma; 210 uint64_t dvck_npages; 211 } dvcookie_t; 212 213 typedef struct dcookie { 214 paddr_t dck_paddr; 215 uint64_t dck_npages; 216 } dcookie_t; 217 218 /* per dma handle private state */ 219 typedef struct rootnex_dma_s { 220 /* 221 * sgl related state used to build and describe the sgl. 222 * 223 * dp_partial_required - used in the bind slow path to identify if we 224 * need to do a partial mapping or not. 225 * dp_trim_required - used in the bind slow path to identify if we 226 * need to trim when switching to a new window. This should only be 227 * set when partial is set. 228 * dp_granularity_power_2 - set in alloc handle and used in bind slow 229 * path to determine if we & or % to calculate the trim. 230 * dp_dma - copy of dma "object" passed in during bind 231 * dp_maxxfer - trimmed dma_attr_maxxfer so that it is a whole 232 * multiple of granularity 233 * dp_sglinfo - See rootnex_sglinfo_t above. 234 */ 235 boolean_t dp_partial_required; 236 boolean_t dp_trim_required; 237 boolean_t dp_granularity_power_2; 238 uint64_t dp_maxxfer; 239 ddi_dma_obj_t dp_dma; 240 rootnex_sglinfo_t dp_sglinfo; 241 242 /* 243 * Copy buffer related state 244 * 245 * dp_copybuf_size - the actual size of the copy buffer that we are 246 * using. This can be smaller that dp_copybuf_req, i.e. bind size > 247 * max copy buffer size. 248 * dp_cbaddr - kernel address of copy buffer. Used to determine where 249 * where to copy to/from. 250 * dp_cbsize - the "real" size returned from the copy buffer alloc. 251 * Set in the copybuf alloc and used to free copybuf. 252 * dp_pgmap - page map used in sync to determine which pages in the 253 * buffer use the copy buffer and what addresses to use to copy to/ 254 * from. 255 * dp_cb_remaping - status if this bind causes us to have to remap 256 * the copybuf when switching to new windows. This is only used in 257 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 258 * this case. 259 * dp_kva - kernel heap arena vmem space for mapping to buffers which 260 * we don't have a kernel VA to bcopy to/from. This is only used in 261 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 262 * this case. 263 */ 264 size_t dp_copybuf_size; 265 caddr_t dp_cbaddr; 266 size_t dp_cbsize; 267 rootnex_pgmap_t *dp_pgmap; 268 #if !defined(__amd64) 269 boolean_t dp_cb_remaping; 270 caddr_t dp_kva; 271 #endif 272 273 /* 274 * window related state. The pointer to the window state array which may 275 * be a pointer into the pre allocated state, or we may have had to 276 * allocate the window array on the fly because it wouldn't fit. If 277 * we allocate it, we'll use dp_need_to_free_window and dp_window_size 278 * during cleanup. dp_current_win keeps track of the current window. 279 * dp_max_win is the maximum number of windows we could have. 280 */ 281 uint_t dp_current_win; 282 rootnex_window_t *dp_window; 283 boolean_t dp_need_to_free_window; 284 uint_t dp_window_size; 285 uint_t dp_max_win; 286 287 /* dip of driver which "owns" handle. set to rdip in alloc_handle() */ 288 dev_info_t *dp_dip; 289 290 /* 291 * dp_mutex and dp_inuse are only used to see if a driver is trying to 292 * bind to an already bound dma handle. dp_mutex only used for dp_inuse 293 */ 294 kmutex_t dp_mutex; 295 boolean_t dp_inuse; 296 297 /* 298 * cookie related state. The pointer to the cookies (dp_cookies) may 299 * be a pointer into the pre allocated state, or we may have had to 300 * allocate the cookie array on the fly because it wouldn't fit. If 301 * we allocate it, we'll use dp_need_to_free_cookie and dp_cookie_size 302 * during cleanup. dp_current_cookie is only used in the obsoleted 303 * interfaces to determine when we've used up all the cookies in a 304 * window during nextseg().. 305 */ 306 size_t dp_cookie_size; 307 ddi_dma_cookie_t *dp_cookies; 308 boolean_t dp_need_to_free_cookie; 309 uint_t dp_current_cookie; /* for obsoleted I/Fs */ 310 ddi_dma_cookie_t *dp_saved_cookies; 311 boolean_t dp_need_to_switch_cookies; 312 313 /* 314 * pre allocated space for the bind state, allocated during alloc 315 * handle. For a lot of devices, this will save us from having to do 316 * kmem_alloc's during the bind most of the time. kmem_alloc's can be 317 * expensive on x86 when the cpu count goes up since xcalls are 318 * expensive on x86. 319 */ 320 uchar_t *dp_prealloc_buffer; 321 322 /* 323 * Intel IOMMU (immu) related state 324 * dv_cookies saves the dvma allocated for this handler 325 * max index of dvcookies in dvmax 326 */ 327 dvcookie_t *dp_dvcookies; 328 uint64_t dp_dvmax; 329 dcookie_t *dp_dcookies; 330 uint64_t dp_dmax; 331 uint64_t dp_max_cookies; 332 uint64_t dp_max_dcookies; 333 334 /* 335 * sleep flags set on bind and unset on unbind 336 */ 337 int dp_sleep_flags; 338 } rootnex_dma_t; 339 340 /* 341 * profile/performance counters. Most things will be dtrace probes, but there 342 * are a couple of things we want to keep track all the time. We track the 343 * total number of active handles and binds (i.e. an alloc without a free or 344 * a bind without an unbind) since rootnex attach. We also track the total 345 * number of binds which have failed since rootnex attach. 346 */ 347 typedef enum { 348 ROOTNEX_CNT_ACTIVE_HDLS = 0, 349 ROOTNEX_CNT_ACTIVE_BINDS = 1, 350 ROOTNEX_CNT_ALLOC_FAIL = 2, 351 ROOTNEX_CNT_BIND_FAIL = 3, 352 ROOTNEX_CNT_SYNC_FAIL = 4, 353 ROOTNEX_CNT_GETWIN_FAIL = 5, 354 355 /* This one must be last */ 356 ROOTNEX_CNT_LAST 357 } rootnex_cnt_t; 358 359 /* 360 * global driver state. 361 * r_dmahdl_cache - dma_handle kmem_cache 362 * r_dvma_call_list_id - ddi_set_callback() id 363 * r_peekpoke_mutex - serialize peeks and pokes. 364 * r_dip - rootnex dip 365 * r_reserved_msg_printed - ctlops reserve message threshold 366 * r_counters - profile/performance counters 367 */ 368 typedef struct rootnex_state_s { 369 uint_t r_prealloc_cookies; 370 uint_t r_prealloc_size; 371 kmem_cache_t *r_dmahdl_cache; 372 uintptr_t r_dvma_call_list_id; 373 kmutex_t r_peekpoke_mutex; 374 dev_info_t *r_dip; 375 ddi_iblock_cookie_t r_err_ibc; 376 boolean_t r_reserved_msg_printed; 377 uint64_t r_counters[ROOTNEX_CNT_LAST]; 378 iommulib_nexhandle_t r_iommulib_handle; 379 } rootnex_state_t; 380 381 #ifdef __cplusplus 382 } 383 #endif 384 385 #endif /* _SYS_ROOTNEX_H */ 386