1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_ROOTNEX_H 27 #define _SYS_ROOTNEX_H 28 29 /* 30 * x86 root nexus implementation specific state 31 */ 32 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/modctl.h> 36 #include <sys/sunddi.h> 37 #include <sys/iommulib.h> 38 #include <sys/sdt.h> 39 40 #ifdef __cplusplus 41 extern "C" { 42 #endif 43 44 45 /* size of buffer used for ctlop reportdev */ 46 #define REPORTDEV_BUFSIZE 1024 47 48 /* min and max interrupt vectors */ 49 #define VEC_MIN 1 50 #define VEC_MAX 255 51 52 /* atomic increment/decrement to keep track of outstanding binds, etc */ 53 #ifdef DEBUG 54 #define ROOTNEX_DPROF_INC(addr) atomic_inc_64(addr) 55 #define ROOTNEX_DPROF_DEC(addr) atomic_add_64(addr, -1) 56 #define ROOTNEX_DPROBE1(name, type1, arg1) \ 57 DTRACE_PROBE1(name, type1, arg1) 58 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \ 59 DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3) 60 #else 61 #define ROOTNEX_DPROF_INC(addr) 62 #define ROOTNEX_DPROF_DEC(addr) 63 #define ROOTNEX_DPROBE1(name, type1, arg1) 64 #define ROOTNEX_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) 65 #endif 66 #define ROOTNEX_PROF_INC(addr) atomic_inc_64(addr) 67 #define ROOTNEX_PROF_DEC(addr) atomic_add_64(addr, -1) 68 69 /* set in dmac_type to signify that this cookie uses the copy buffer */ 70 #define ROOTNEX_USES_COPYBUF 0x80000000 71 72 /* 73 * integer or boolean property name and value. A few static rootnex properties 74 * are created during rootnex attach from an array of rootnex_intprop_t.. 75 */ 76 typedef struct rootnex_intprop_s { 77 char *prop_name; 78 int prop_value; 79 } rootnex_intprop_t; 80 81 /* 82 * sgl related information which is visible to rootnex_get_sgl(). Trying to 83 * isolate get_sgl() as much as possible so it can be easily replaced. 84 */ 85 typedef struct rootnex_sglinfo_s { 86 /* 87 * These are passed into rootnex_get_sgl(). 88 * 89 * si_min_addr - the minimum physical address 90 * si_max_addr - the maximum physical address 91 * si_max_cookie_size - the maximum size of a physically contiguous 92 * piece of memory that we can handle in a sgl. 93 * si_segmask - segment mask to determine if we cross a segment boundary 94 * si_max_pages - max number of pages this sgl could occupy (which 95 * is also the maximum number of cookies we might see. 96 */ 97 uint64_t si_min_addr; 98 uint64_t si_max_addr; 99 uint64_t si_max_cookie_size; 100 uint64_t si_segmask; 101 uint_t si_max_pages; 102 103 /* 104 * these are returned by rootnex_get_sgl() 105 * 106 * si_copybuf_req - amount of copy buffer needed by the buffer. 107 * si_buf_offset - The initial offset into the first page of the buffer. 108 * It's set in get sgl and used in the bind slow path to help 109 * calculate the current page index & offset from the current offset 110 * which is relative to the start of the buffer. 111 * si_asp - address space of buffer passed in. 112 * si_sgl_size - The actual number of cookies in the sgl. This does 113 * not reflect and sharing that we might do on window boundaries. 114 */ 115 size_t si_copybuf_req; 116 off_t si_buf_offset; 117 struct as *si_asp; 118 uint_t si_sgl_size; 119 } rootnex_sglinfo_t; 120 121 /* 122 * When we have to use the copy buffer, we allocate one of these structures per 123 * buffer page to track which pages need the copy buffer, what the kernel 124 * virtual address is (which the device can't reach), and what the copy buffer 125 * virtual address is (where the device dma's to/from). For 32-bit kernels, 126 * since we can't use seg kpm, we also need to keep the page_t around and state 127 * if we've currently mapped in the page into KVA space for buffers which don't 128 * have kva already and when we have multiple windows because we used up all our 129 * copy buffer space. 130 */ 131 typedef struct rootnex_pgmap_s { 132 boolean_t pm_uses_copybuf; 133 #if !defined(__amd64) 134 boolean_t pm_mapped; 135 page_t *pm_pp; 136 caddr_t pm_vaddr; 137 #endif 138 caddr_t pm_kaddr; 139 caddr_t pm_cbaddr; 140 } rootnex_pgmap_t; 141 142 /* 143 * We only need to trim a buffer when we have multiple windows. Each window has 144 * trim state. We might have trimmed the end of the previous window, leaving the 145 * first cookie of this window trimmed[tr_trim_first] (which basically means we 146 * won't start with a new cookie), or we might need to trim the end of the 147 * current window [tr_trim_last] (which basically means we won't end with a 148 * complete cookie). We keep the same state for the first & last cookie in a 149 * window (a window can have one or more cookies). However, when we trim the 150 * last cookie, we keep a pointer to the last cookie in the trim state since we 151 * only need this info when we trim. The pointer to the first cookie in the 152 * window is in the window state since we need to know what the first cookie in 153 * the window is in various places. 154 * 155 * If we do trim a cookie, we save away the physical address and size of the 156 * cookie so that we can over write the cookie when we switch windows (the 157 * space for a cookie which is in two windows is shared between the windows. 158 * We keep around the same information for the last page in a window. 159 * 160 * if we happened to trim on a page that uses the copy buffer, and that page 161 * is also in the middle of a window boundary because we have filled up the 162 * copy buffer, we need to remember the copy buffer address for both windows 163 * since the same page will have different copy buffer addresses in the two 164 * windows. We need to due the same for kaddr in the 32-bit kernel since we 165 * have a limited kva space which we map to. 166 */ 167 typedef struct rootnex_trim_s { 168 boolean_t tr_trim_first; 169 boolean_t tr_trim_last; 170 ddi_dma_cookie_t *tr_last_cookie; 171 uint64_t tr_first_paddr; 172 uint64_t tr_last_paddr; 173 size_t tr_first_size; 174 size_t tr_last_size; 175 176 boolean_t tr_first_copybuf_win; 177 boolean_t tr_last_copybuf_win; 178 uint_t tr_first_pidx; 179 uint_t tr_last_pidx; 180 caddr_t tr_first_cbaddr; 181 caddr_t tr_last_cbaddr; 182 #if !defined(__amd64) 183 caddr_t tr_first_kaddr; 184 caddr_t tr_last_kaddr; 185 #endif 186 } rootnex_trim_t; 187 188 /* 189 * per window state. A bound DMA handle can have multiple windows. Each window 190 * will have the following state. We track if this window needs to sync, 191 * the offset into the buffer where the window starts, the size of the window. 192 * a pointer to the first cookie in the window, the number of cookies in the 193 * window, and the trim state for the window. For the 32-bit kernel, we keep 194 * track of if we need to remap the copy buffer when we switch to a this window 195 */ 196 typedef struct rootnex_window_s { 197 boolean_t wd_dosync; 198 uint_t wd_cookie_cnt; 199 off_t wd_offset; 200 size_t wd_size; 201 ddi_dma_cookie_t *wd_first_cookie; 202 rootnex_trim_t wd_trim; 203 #if !defined(__amd64) 204 boolean_t wd_remap_copybuf; 205 #endif 206 } rootnex_window_t; 207 208 /* per dma handle private state */ 209 typedef struct rootnex_dma_s { 210 /* 211 * sgl related state used to build and describe the sgl. 212 * 213 * dp_partial_required - used in the bind slow path to identify if we 214 * need to do a partial mapping or not. 215 * dp_trim_required - used in the bind slow path to identify if we 216 * need to trim when switching to a new window. This should only be 217 * set when partial is set. 218 * dp_granularity_power_2 - set in alloc handle and used in bind slow 219 * path to determine if we & or % to calculate the trim. 220 * dp_dma - copy of dma "object" passed in during bind 221 * dp_maxxfer - trimmed dma_attr_maxxfer so that it is a whole 222 * multiple of granularity 223 * dp_sglinfo - See rootnex_sglinfo_t above. 224 */ 225 boolean_t dp_partial_required; 226 boolean_t dp_trim_required; 227 boolean_t dp_granularity_power_2; 228 uint64_t dp_maxxfer; 229 ddi_dma_obj_t dp_dma; 230 rootnex_sglinfo_t dp_sglinfo; 231 232 /* 233 * Copy buffer related state 234 * 235 * dp_copybuf_size - the actual size of the copy buffer that we are 236 * using. This can be smaller that dp_copybuf_req, i.e. bind size > 237 * max copy buffer size. 238 * dp_cbaddr - kernel address of copy buffer. Used to determine where 239 * where to copy to/from. 240 * dp_cbsize - the "real" size returned from the copy buffer alloc. 241 * Set in the copybuf alloc and used to free copybuf. 242 * dp_pgmap - page map used in sync to determine which pages in the 243 * buffer use the copy buffer and what addresses to use to copy to/ 244 * from. 245 * dp_cb_remaping - status if this bind causes us to have to remap 246 * the copybuf when switching to new windows. This is only used in 247 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 248 * this case. 249 * dp_kva - kernel heap arena vmem space for mapping to buffers which 250 * we don't have a kernel VA to bcopy to/from. This is only used in 251 * the 32-bit kernel since we use seg kpm in the 64-bit kernel for 252 * this case. 253 */ 254 size_t dp_copybuf_size; 255 caddr_t dp_cbaddr; 256 size_t dp_cbsize; 257 rootnex_pgmap_t *dp_pgmap; 258 #if !defined(__amd64) 259 boolean_t dp_cb_remaping; 260 caddr_t dp_kva; 261 #endif 262 263 /* 264 * window related state. The pointer to the window state array which may 265 * be a pointer into the pre allocated state, or we may have had to 266 * allocate the window array on the fly because it wouldn't fit. If 267 * we allocate it, we'll use dp_need_to_free_window and dp_window_size 268 * during cleanup. dp_current_win keeps track of the current window. 269 * dp_max_win is the maximum number of windows we could have. 270 */ 271 uint_t dp_current_win; 272 rootnex_window_t *dp_window; 273 boolean_t dp_need_to_free_window; 274 uint_t dp_window_size; 275 uint_t dp_max_win; 276 277 /* dip of driver which "owns" handle. set to rdip in alloc_handle() */ 278 dev_info_t *dp_dip; 279 280 /* 281 * dp_mutex and dp_inuse are only used to see if a driver is trying to 282 * bind to an already bound dma handle. dp_mutex only used for dp_inuse 283 */ 284 kmutex_t dp_mutex; 285 boolean_t dp_inuse; 286 287 /* 288 * cookie related state. The pointer to the cookies (dp_cookies) may 289 * be a pointer into the pre allocated state, or we may have had to 290 * allocate the cookie array on the fly because it wouldn't fit. If 291 * we allocate it, we'll use dp_need_to_free_cookie and dp_cookie_size 292 * during cleanup. dp_current_cookie is only used in the obsoleted 293 * interfaces to determine when we've used up all the cookies in a 294 * window during nextseg().. 295 */ 296 size_t dp_cookie_size; 297 ddi_dma_cookie_t *dp_cookies; 298 boolean_t dp_need_to_free_cookie; 299 uint_t dp_current_cookie; /* for obsoleted I/Fs */ 300 ddi_dma_cookie_t *dp_saved_cookies; 301 boolean_t dp_need_to_switch_cookies; 302 303 /* 304 * pre allocated space for the bind state, allocated during alloc 305 * handle. For a lot of devices, this will save us from having to do 306 * kmem_alloc's during the bind most of the time. kmem_alloc's can be 307 * expensive on x86 when the cpu count goes up since xcalls are 308 * expensive on x86. 309 */ 310 uchar_t *dp_prealloc_buffer; 311 312 /* 313 * intel iommu related state 314 * dvma_cookies saves the dvma allocated for this handler, it has the 315 * size of si_max_pages, set when bind handler and freed when unbind 316 */ 317 void *dp_dvma_cookies; 318 319 /* 320 * sleep flags set on bind and unset on unbind 321 */ 322 int dp_sleep_flags; 323 } rootnex_dma_t; 324 325 /* 326 * profile/performance counters. Most things will be dtrace probes, but there 327 * are a couple of things we want to keep track all the time. We track the 328 * total number of active handles and binds (i.e. an alloc without a free or 329 * a bind without an unbind) since rootnex attach. We also track the total 330 * number of binds which have failed since rootnex attach. 331 */ 332 typedef enum { 333 ROOTNEX_CNT_ACTIVE_HDLS = 0, 334 ROOTNEX_CNT_ACTIVE_BINDS = 1, 335 ROOTNEX_CNT_ALLOC_FAIL = 2, 336 ROOTNEX_CNT_BIND_FAIL = 3, 337 ROOTNEX_CNT_SYNC_FAIL = 4, 338 ROOTNEX_CNT_GETWIN_FAIL = 5, 339 340 /* This one must be last */ 341 ROOTNEX_CNT_LAST 342 } rootnex_cnt_t; 343 344 /* 345 * global driver state. 346 * r_dmahdl_cache - dma_handle kmem_cache 347 * r_dvma_call_list_id - ddi_set_callback() id 348 * r_peekpoke_mutex - serialize peeks and pokes. 349 * r_dip - rootnex dip 350 * r_reserved_msg_printed - ctlops reserve message threshold 351 * r_counters - profile/performance counters 352 * r_intel_iommu_enabled - intel iommu enabled 353 */ 354 typedef struct rootnex_state_s { 355 uint_t r_prealloc_cookies; 356 uint_t r_prealloc_size; 357 kmem_cache_t *r_dmahdl_cache; 358 uintptr_t r_dvma_call_list_id; 359 kmutex_t r_peekpoke_mutex; 360 dev_info_t *r_dip; 361 ddi_iblock_cookie_t r_err_ibc; 362 boolean_t r_reserved_msg_printed; 363 uint64_t r_counters[ROOTNEX_CNT_LAST]; 364 boolean_t r_intel_iommu_enabled; 365 iommulib_nexhandle_t r_iommulib_handle; 366 } rootnex_state_t; 367 368 369 #ifdef __cplusplus 370 } 371 #endif 372 373 #endif /* _SYS_ROOTNEX_H */ 374