1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Portions Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Copyright (c) 2009, Intel Corporation. 29 * All rights reserved. 30 */ 31 32 #include <sys/ddi.h> 33 #include <sys/archsystm.h> 34 #include <vm/hat_i86.h> 35 #include <sys/types.h> 36 #include <sys/sysmacros.h> 37 #include <sys/immu.h> 38 39 /* invalidation queue table entry size */ 40 #define QINV_ENTRY_SIZE 0x10 41 42 /* max value of Queue Size field of Invalidation Queue Address Register */ 43 #define QINV_MAX_QUEUE_SIZE 0x7 44 45 /* status data size of invalidation wait descriptor */ 46 #define QINV_SYNC_DATA_SIZE 0x4 47 48 /* status data value of invalidation wait descriptor */ 49 #define QINV_SYNC_DATA_FENCE 1 50 #define QINV_SYNC_DATA_UNFENCE 2 51 52 /* invalidation queue head and tail */ 53 #define QINV_IQA_HEAD(QH) BITX((QH), 18, 4) 54 #define QINV_IQA_TAIL_SHIFT 4 55 56 /* invalidation queue entry structure */ 57 typedef struct qinv_inv_dsc { 58 uint64_t lo; 59 uint64_t hi; 60 } qinv_dsc_t; 61 62 /* 63 * struct iotlb_cache_node 64 * the pending data for iotlb flush 65 */ 66 typedef struct iotlb_pend_node { 67 dvcookie_t *icn_dvcookies; /* ptr to dvma cookie array */ 68 uint_t icn_count; /* valid cookie count */ 69 uint_t icn_array_size; /* array size */ 70 list_node_t node; 71 } qinv_iotlb_pend_node_t; 72 73 /* 74 * struct iotlb_cache_head 75 * the pending head for the iotlb flush 76 */ 77 typedef struct iotlb_pend_head { 78 /* the pending node cache list */ 79 kmutex_t ich_mem_lock; 80 list_t ich_mem_list; 81 } qinv_iotlb_pend_head_t; 82 83 /* 84 * qinv_iotlb_t 85 * pending data for qiueued invalidation iotlb flush 86 */ 87 typedef struct qinv_iotlb { 88 dvcookie_t *qinv_iotlb_dvcookies; 89 uint_t qinv_iotlb_count; 90 uint_t qinv_iotlb_size; 91 list_node_t qinv_iotlb_node; 92 } qinv_iotlb_t; 93 94 /* physical contigous pages for invalidation queue */ 95 typedef struct qinv_mem { 96 kmutex_t qinv_mem_lock; 97 ddi_dma_handle_t qinv_mem_dma_hdl; 98 ddi_acc_handle_t qinv_mem_acc_hdl; 99 caddr_t qinv_mem_vaddr; 100 paddr_t qinv_mem_paddr; 101 uint_t qinv_mem_size; 102 uint16_t qinv_mem_head; 103 uint16_t qinv_mem_tail; 104 } qinv_mem_t; 105 106 107 /* 108 * invalidation queue state 109 * This structure describes the state information of the 110 * invalidation queue table and related status memeory for 111 * invalidation wait descriptor 112 * 113 * qinv_table - invalidation queue table 114 * qinv_sync - sync status memory for invalidation wait descriptor 115 * qinv_iotlb_pend_node - pending iotlb node 116 */ 117 typedef struct qinv { 118 qinv_mem_t qinv_table; 119 qinv_mem_t qinv_sync; 120 qinv_iotlb_pend_head_t qinv_pend_head; 121 qinv_iotlb_pend_node_t **qinv_iotlb_pend_node; 122 } qinv_t; 123 124 125 /* helper macro for making queue invalidation descriptor */ 126 #define INV_DSC_TYPE(dsc) ((dsc)->lo & 0xF) 127 #define CC_INV_DSC_HIGH (0) 128 #define CC_INV_DSC_LOW(fm, sid, did, g) (((uint64_t)(fm) << 48) | \ 129 ((uint64_t)(sid) << 32) | \ 130 ((uint64_t)(did) << 16) | \ 131 ((uint64_t)(g) << 4) | \ 132 1) 133 134 #define IOTLB_INV_DSC_HIGH(addr, ih, am) (((uint64_t)(addr)) | \ 135 ((uint64_t)(ih) << 6) | \ 136 ((uint64_t)(am))) 137 138 #define IOTLB_INV_DSC_LOW(did, dr, dw, g) (((uint64_t)(did) << 16) | \ 139 ((uint64_t)(dr) << 7) | \ 140 ((uint64_t)(dw) << 6) | \ 141 ((uint64_t)(g) << 4) | \ 142 2) 143 144 #define DEV_IOTLB_INV_DSC_HIGH(addr, s) (((uint64_t)(addr)) | (s)) 145 146 #define DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd) ( \ 147 ((uint64_t)(sid) << 32) | \ 148 ((uint64_t)(max_invs_pd) << 16) | \ 149 3) 150 151 #define IEC_INV_DSC_HIGH (0) 152 #define IEC_INV_DSC_LOW(idx, im, g) (((uint64_t)(idx) << 32) | \ 153 ((uint64_t)(im) << 27) | \ 154 ((uint64_t)(g) << 4) | \ 155 4) 156 157 #define INV_WAIT_DSC_HIGH(saddr) ((uint64_t)(saddr)) 158 159 #define INV_WAIT_DSC_LOW(sdata, fn, sw, iflag) (((uint64_t)(sdata) << 32) | \ 160 ((uint64_t)(fn) << 6) | \ 161 ((uint64_t)(sw) << 5) | \ 162 ((uint64_t)(iflag) << 4) | \ 163 5) 164 165 /* 166 * QS field of Invalidation Queue Address Register 167 * the size of invalidation queue is 1 << (qinv_iqa_qs + 8) 168 */ 169 static uint_t qinv_iqa_qs = 6; 170 171 /* 172 * the invalidate desctiptor type of queued invalidation interface 173 */ 174 static char *qinv_dsc_type[] = { 175 "Reserved", 176 "Context Cache Invalidate Descriptor", 177 "IOTLB Invalidate Descriptor", 178 "Device-IOTLB Invalidate Descriptor", 179 "Interrupt Entry Cache Invalidate Descriptor", 180 "Invalidation Wait Descriptor", 181 "Incorrect queue invalidation type" 182 }; 183 184 #define QINV_MAX_DSC_TYPE (sizeof (qinv_dsc_type) / sizeof (char *)) 185 186 /* 187 * the queued invalidation interface functions 188 */ 189 static void qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc); 190 static void qinv_context_common(immu_t *immu, uint8_t function_mask, 191 uint16_t source_id, uint_t domain_id, ctt_inv_g_t type); 192 static void qinv_iotlb_common(immu_t *immu, uint_t domain_id, 193 uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type); 194 static void qinv_iec_common(immu_t *immu, uint_t iidx, 195 uint_t im, uint_t g); 196 static uint_t qinv_alloc_sync_mem_entry(immu_t *immu); 197 static void qinv_wait_async_unfence(immu_t *immu, 198 qinv_iotlb_pend_node_t *node); 199 static void qinv_wait_sync(immu_t *immu); 200 static int qinv_wait_async_finish(immu_t *immu, int *count); 201 /*LINTED*/ 202 static void qinv_wait_async_fence(immu_t *immu); 203 /*LINTED*/ 204 static void qinv_dev_iotlb_common(immu_t *immu, uint16_t sid, 205 uint64_t addr, uint_t size, uint_t max_invs_pd); 206 207 208 /* submit invalidation request descriptor to invalidation queue */ 209 static void 210 qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc) 211 { 212 qinv_t *qinv; 213 qinv_mem_t *qinv_table; 214 uint_t tail; 215 216 qinv = (qinv_t *)immu->immu_qinv; 217 qinv_table = &(qinv->qinv_table); 218 219 mutex_enter(&qinv_table->qinv_mem_lock); 220 tail = qinv_table->qinv_mem_tail; 221 qinv_table->qinv_mem_tail++; 222 223 if (qinv_table->qinv_mem_tail == qinv_table->qinv_mem_size) 224 qinv_table->qinv_mem_tail = 0; 225 226 while (qinv_table->qinv_mem_head == qinv_table->qinv_mem_tail) { 227 /* 228 * inv queue table exhausted, wait hardware to fetch 229 * next descriptor 230 */ 231 qinv_table->qinv_mem_head = QINV_IQA_HEAD( 232 immu_regs_get64(immu, IMMU_REG_INVAL_QH)); 233 } 234 235 bcopy(dsc, qinv_table->qinv_mem_vaddr + tail * QINV_ENTRY_SIZE, 236 QINV_ENTRY_SIZE); 237 238 immu_regs_put64(immu, IMMU_REG_INVAL_QT, 239 qinv_table->qinv_mem_tail << QINV_IQA_TAIL_SHIFT); 240 241 mutex_exit(&qinv_table->qinv_mem_lock); 242 } 243 244 /* queued invalidation interface -- invalidate context cache */ 245 static void 246 qinv_context_common(immu_t *immu, uint8_t function_mask, 247 uint16_t source_id, uint_t domain_id, ctt_inv_g_t type) 248 { 249 qinv_dsc_t dsc; 250 251 dsc.lo = CC_INV_DSC_LOW(function_mask, source_id, domain_id, type); 252 dsc.hi = CC_INV_DSC_HIGH; 253 254 qinv_submit_inv_dsc(immu, &dsc); 255 } 256 257 /* queued invalidation interface -- invalidate iotlb */ 258 static void 259 qinv_iotlb_common(immu_t *immu, uint_t domain_id, 260 uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type) 261 { 262 qinv_dsc_t dsc; 263 uint8_t dr = 0; 264 uint8_t dw = 0; 265 266 if (IMMU_CAP_GET_DRD(immu->immu_regs_cap)) 267 dr = 1; 268 if (IMMU_CAP_GET_DWD(immu->immu_regs_cap)) 269 dw = 1; 270 271 switch (type) { 272 case TLB_INV_G_PAGE: 273 if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap) || 274 am > IMMU_CAP_GET_MAMV(immu->immu_regs_cap) || 275 addr & IMMU_PAGEOFFSET) { 276 type = TLB_INV_G_DOMAIN; 277 goto qinv_ignore_psi; 278 } 279 dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type); 280 dsc.hi = IOTLB_INV_DSC_HIGH(addr, hint, am); 281 break; 282 283 qinv_ignore_psi: 284 case TLB_INV_G_DOMAIN: 285 dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type); 286 dsc.hi = 0; 287 break; 288 289 case TLB_INV_G_GLOBAL: 290 dsc.lo = IOTLB_INV_DSC_LOW(0, dr, dw, type); 291 dsc.hi = 0; 292 break; 293 default: 294 ddi_err(DER_WARN, NULL, "incorrect iotlb flush type"); 295 return; 296 } 297 298 qinv_submit_inv_dsc(immu, &dsc); 299 } 300 301 /* queued invalidation interface -- invalidate dev_iotlb */ 302 static void 303 qinv_dev_iotlb_common(immu_t *immu, uint16_t sid, 304 uint64_t addr, uint_t size, uint_t max_invs_pd) 305 { 306 qinv_dsc_t dsc; 307 308 dsc.lo = DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd); 309 dsc.hi = DEV_IOTLB_INV_DSC_HIGH(addr, size); 310 311 qinv_submit_inv_dsc(immu, &dsc); 312 } 313 314 /* queued invalidation interface -- invalidate interrupt entry cache */ 315 static void 316 qinv_iec_common(immu_t *immu, uint_t iidx, uint_t im, uint_t g) 317 { 318 qinv_dsc_t dsc; 319 320 dsc.lo = IEC_INV_DSC_LOW(iidx, im, g); 321 dsc.hi = IEC_INV_DSC_HIGH; 322 323 qinv_submit_inv_dsc(immu, &dsc); 324 } 325 326 /* 327 * alloc free entry from sync status table 328 */ 329 static uint_t 330 qinv_alloc_sync_mem_entry(immu_t *immu) 331 { 332 qinv_mem_t *sync_mem; 333 uint_t tail; 334 qinv_t *qinv; 335 336 qinv = (qinv_t *)immu->immu_qinv; 337 sync_mem = &qinv->qinv_sync; 338 339 sync_mem_exhausted: 340 mutex_enter(&sync_mem->qinv_mem_lock); 341 tail = sync_mem->qinv_mem_tail; 342 sync_mem->qinv_mem_tail++; 343 if (sync_mem->qinv_mem_tail == sync_mem->qinv_mem_size) 344 sync_mem->qinv_mem_tail = 0; 345 346 if (sync_mem->qinv_mem_head == sync_mem->qinv_mem_tail) { 347 /* should never happen */ 348 ddi_err(DER_WARN, NULL, "sync mem exhausted"); 349 sync_mem->qinv_mem_tail = tail; 350 mutex_exit(&sync_mem->qinv_mem_lock); 351 delay(IMMU_ALLOC_RESOURCE_DELAY); 352 goto sync_mem_exhausted; 353 } 354 mutex_exit(&sync_mem->qinv_mem_lock); 355 356 return (tail); 357 } 358 359 /* 360 * queued invalidation interface -- invalidation wait descriptor 361 * fence flag not set, need status data to indicate the invalidation 362 * wait descriptor completion 363 */ 364 static void 365 qinv_wait_async_unfence(immu_t *immu, qinv_iotlb_pend_node_t *node) 366 { 367 qinv_dsc_t dsc; 368 qinv_mem_t *sync_mem; 369 uint64_t saddr; 370 uint_t tail; 371 qinv_t *qinv; 372 373 qinv = (qinv_t *)immu->immu_qinv; 374 sync_mem = &qinv->qinv_sync; 375 tail = qinv_alloc_sync_mem_entry(immu); 376 377 /* plant an iotlb pending node */ 378 qinv->qinv_iotlb_pend_node[tail] = node; 379 380 saddr = sync_mem->qinv_mem_paddr + tail * QINV_SYNC_DATA_SIZE; 381 382 /* 383 * sdata = QINV_SYNC_DATA_UNFENCE, fence = 0, sw = 1, if = 0 384 * indicate the invalidation wait descriptor completion by 385 * performing a coherent DWORD write to the status address, 386 * not by generating an invalidation completion event 387 */ 388 dsc.lo = INV_WAIT_DSC_LOW(QINV_SYNC_DATA_UNFENCE, 0, 1, 0); 389 dsc.hi = INV_WAIT_DSC_HIGH(saddr); 390 391 qinv_submit_inv_dsc(immu, &dsc); 392 } 393 394 /* 395 * queued invalidation interface -- invalidation wait descriptor 396 * fence flag set, indicate descriptors following the invalidation 397 * wait descriptor must be processed by hardware only after the 398 * invalidation wait descriptor completes. 399 */ 400 static void 401 qinv_wait_async_fence(immu_t *immu) 402 { 403 qinv_dsc_t dsc; 404 405 /* sw = 0, fence = 1, iflag = 0 */ 406 dsc.lo = INV_WAIT_DSC_LOW(0, 1, 0, 0); 407 dsc.hi = 0; 408 qinv_submit_inv_dsc(immu, &dsc); 409 } 410 411 /* 412 * queued invalidation interface -- invalidation wait descriptor 413 * wait until the invalidation request finished 414 */ 415 static void 416 qinv_wait_sync(immu_t *immu) 417 { 418 qinv_dsc_t dsc; 419 qinv_mem_t *sync_mem; 420 uint64_t saddr; 421 uint_t tail; 422 qinv_t *qinv; 423 volatile uint32_t *status; 424 425 qinv = (qinv_t *)immu->immu_qinv; 426 sync_mem = &qinv->qinv_sync; 427 tail = qinv_alloc_sync_mem_entry(immu); 428 saddr = sync_mem->qinv_mem_paddr + tail * QINV_SYNC_DATA_SIZE; 429 status = (uint32_t *)(sync_mem->qinv_mem_vaddr + tail * 430 QINV_SYNC_DATA_SIZE); 431 432 /* 433 * sdata = QINV_SYNC_DATA_FENCE, fence = 1, sw = 1, if = 0 434 * indicate the invalidation wait descriptor completion by 435 * performing a coherent DWORD write to the status address, 436 * not by generating an invalidation completion event 437 */ 438 dsc.lo = INV_WAIT_DSC_LOW(QINV_SYNC_DATA_FENCE, 1, 1, 0); 439 dsc.hi = INV_WAIT_DSC_HIGH(saddr); 440 441 qinv_submit_inv_dsc(immu, &dsc); 442 443 while ((*status) != QINV_SYNC_DATA_FENCE) 444 iommu_cpu_nop(); 445 *status = QINV_SYNC_DATA_UNFENCE; 446 } 447 448 /* get already completed invalidation wait requests */ 449 static int 450 qinv_wait_async_finish(immu_t *immu, int *cnt) 451 { 452 qinv_mem_t *sync_mem; 453 int index; 454 qinv_t *qinv; 455 volatile uint32_t *value; 456 457 ASSERT((*cnt) == 0); 458 459 qinv = (qinv_t *)immu->immu_qinv; 460 sync_mem = &qinv->qinv_sync; 461 462 mutex_enter(&sync_mem->qinv_mem_lock); 463 index = sync_mem->qinv_mem_head; 464 value = (uint32_t *)(sync_mem->qinv_mem_vaddr + index 465 * QINV_SYNC_DATA_SIZE); 466 while (*value == QINV_SYNC_DATA_UNFENCE) { 467 *value = 0; 468 (*cnt)++; 469 sync_mem->qinv_mem_head++; 470 if (sync_mem->qinv_mem_head == sync_mem->qinv_mem_size) { 471 sync_mem->qinv_mem_head = 0; 472 value = (uint32_t *)(sync_mem->qinv_mem_vaddr); 473 } else 474 value = (uint32_t *)((char *)value + 475 QINV_SYNC_DATA_SIZE); 476 } 477 478 mutex_exit(&sync_mem->qinv_mem_lock); 479 if ((*cnt) > 0) 480 return (index); 481 else 482 return (-1); 483 } 484 485 /* 486 * call ddi_dma_mem_alloc to allocate physical contigous 487 * pages for invalidation queue table 488 */ 489 static int 490 qinv_setup(immu_t *immu) 491 { 492 qinv_t *qinv; 493 size_t size; 494 495 ddi_dma_attr_t qinv_dma_attr = { 496 DMA_ATTR_V0, 497 0U, 498 0xffffffffU, 499 0xffffffffU, 500 MMU_PAGESIZE, /* page aligned */ 501 0x1, 502 0x1, 503 0xffffffffU, 504 0xffffffffU, 505 1, 506 4, 507 0 508 }; 509 510 ddi_device_acc_attr_t qinv_acc_attr = { 511 DDI_DEVICE_ATTR_V0, 512 DDI_NEVERSWAP_ACC, 513 DDI_STRICTORDER_ACC 514 }; 515 516 mutex_init(&(immu->immu_qinv_lock), NULL, MUTEX_DRIVER, NULL); 517 518 519 mutex_enter(&(immu->immu_qinv_lock)); 520 521 immu->immu_qinv = NULL; 522 if (!IMMU_ECAP_GET_QI(immu->immu_regs_excap) || 523 immu_qinv_enable == B_FALSE) { 524 mutex_exit(&(immu->immu_qinv_lock)); 525 return (DDI_SUCCESS); 526 } 527 528 if (qinv_iqa_qs > QINV_MAX_QUEUE_SIZE) 529 qinv_iqa_qs = QINV_MAX_QUEUE_SIZE; 530 531 qinv = kmem_zalloc(sizeof (qinv_t), KM_SLEEP); 532 533 if (ddi_dma_alloc_handle(root_devinfo, 534 &qinv_dma_attr, DDI_DMA_SLEEP, NULL, 535 &(qinv->qinv_table.qinv_mem_dma_hdl)) != DDI_SUCCESS) { 536 ddi_err(DER_WARN, root_devinfo, 537 "alloc invalidation queue table handler failed"); 538 goto queue_table_handle_failed; 539 } 540 541 if (ddi_dma_alloc_handle(root_devinfo, 542 &qinv_dma_attr, DDI_DMA_SLEEP, NULL, 543 &(qinv->qinv_sync.qinv_mem_dma_hdl)) != DDI_SUCCESS) { 544 ddi_err(DER_WARN, root_devinfo, 545 "alloc invalidation queue sync mem handler failed"); 546 goto sync_table_handle_failed; 547 } 548 549 qinv->qinv_table.qinv_mem_size = (1 << (qinv_iqa_qs + 8)); 550 size = qinv->qinv_table.qinv_mem_size * QINV_ENTRY_SIZE; 551 552 /* alloc physical contiguous pages for invalidation queue */ 553 if (ddi_dma_mem_alloc(qinv->qinv_table.qinv_mem_dma_hdl, 554 size, 555 &qinv_acc_attr, 556 DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED, 557 DDI_DMA_SLEEP, 558 NULL, 559 &(qinv->qinv_table.qinv_mem_vaddr), 560 &size, 561 &(qinv->qinv_table.qinv_mem_acc_hdl)) != DDI_SUCCESS) { 562 ddi_err(DER_WARN, root_devinfo, 563 "alloc invalidation queue table failed"); 564 goto queue_table_mem_failed; 565 } 566 567 ASSERT(!((uintptr_t)qinv->qinv_table.qinv_mem_vaddr & MMU_PAGEOFFSET)); 568 bzero(qinv->qinv_table.qinv_mem_vaddr, size); 569 570 /* get the base physical address of invalidation request queue */ 571 qinv->qinv_table.qinv_mem_paddr = pfn_to_pa( 572 hat_getpfnum(kas.a_hat, qinv->qinv_table.qinv_mem_vaddr)); 573 574 qinv->qinv_table.qinv_mem_head = qinv->qinv_table.qinv_mem_tail = 0; 575 576 qinv->qinv_sync.qinv_mem_size = qinv->qinv_table.qinv_mem_size; 577 size = qinv->qinv_sync.qinv_mem_size * QINV_SYNC_DATA_SIZE; 578 579 /* alloc status memory for invalidation wait descriptor */ 580 if (ddi_dma_mem_alloc(qinv->qinv_sync.qinv_mem_dma_hdl, 581 size, 582 &qinv_acc_attr, 583 DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED, 584 DDI_DMA_SLEEP, 585 NULL, 586 &(qinv->qinv_sync.qinv_mem_vaddr), 587 &size, 588 &(qinv->qinv_sync.qinv_mem_acc_hdl)) != DDI_SUCCESS) { 589 ddi_err(DER_WARN, root_devinfo, 590 "alloc invalidation queue sync mem failed"); 591 goto sync_table_mem_failed; 592 } 593 594 ASSERT(!((uintptr_t)qinv->qinv_sync.qinv_mem_vaddr & MMU_PAGEOFFSET)); 595 bzero(qinv->qinv_sync.qinv_mem_vaddr, size); 596 qinv->qinv_sync.qinv_mem_paddr = pfn_to_pa( 597 hat_getpfnum(kas.a_hat, qinv->qinv_sync.qinv_mem_vaddr)); 598 599 qinv->qinv_sync.qinv_mem_head = qinv->qinv_sync.qinv_mem_tail = 0; 600 601 mutex_init(&(qinv->qinv_table.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL); 602 mutex_init(&(qinv->qinv_sync.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL); 603 604 /* 605 * init iotlb pend node for submitting invalidation iotlb 606 * queue request 607 */ 608 qinv->qinv_iotlb_pend_node = (qinv_iotlb_pend_node_t **) 609 kmem_zalloc(qinv->qinv_sync.qinv_mem_size 610 * sizeof (qinv_iotlb_pend_node_t *), KM_SLEEP); 611 612 /* set invalidation queue structure */ 613 immu->immu_qinv = qinv; 614 615 mutex_exit(&(immu->immu_qinv_lock)); 616 617 return (DDI_SUCCESS); 618 619 sync_table_mem_failed: 620 ddi_dma_mem_free(&(qinv->qinv_table.qinv_mem_acc_hdl)); 621 622 queue_table_mem_failed: 623 ddi_dma_free_handle(&(qinv->qinv_sync.qinv_mem_dma_hdl)); 624 625 sync_table_handle_failed: 626 ddi_dma_free_handle(&(qinv->qinv_table.qinv_mem_dma_hdl)); 627 628 queue_table_handle_failed: 629 kmem_free(qinv, sizeof (qinv_t)); 630 631 mutex_exit(&(immu->immu_qinv_lock)); 632 633 return (DDI_FAILURE); 634 } 635 636 /* 637 * ########################################################################### 638 * 639 * Functions exported by immu_qinv.c 640 * 641 * ########################################################################### 642 */ 643 644 /* 645 * initialize invalidation request queue structure. 646 */ 647 void 648 immu_qinv_setup(list_t *listp) 649 { 650 immu_t *immu; 651 652 if (immu_qinv_enable == B_FALSE) { 653 return; 654 } 655 656 immu = list_head(listp); 657 for (; immu; immu = list_next(listp, immu)) { 658 if (qinv_setup(immu) == DDI_SUCCESS) { 659 immu->immu_qinv_setup = B_TRUE; 660 } 661 } 662 } 663 664 void 665 immu_qinv_startup(immu_t *immu) 666 { 667 qinv_t *qinv; 668 uint64_t qinv_reg_value; 669 670 if (immu->immu_qinv_setup == B_FALSE) { 671 return; 672 } 673 674 qinv = (qinv_t *)immu->immu_qinv; 675 qinv_reg_value = qinv->qinv_table.qinv_mem_paddr | qinv_iqa_qs; 676 immu_regs_qinv_enable(immu, qinv_reg_value); 677 immu->immu_qinv_running = B_TRUE; 678 } 679 680 /* 681 * queued invalidation interface 682 * function based context cache invalidation 683 */ 684 void 685 immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask, 686 uint16_t source_id, uint_t domain_id) 687 { 688 qinv_context_common(immu, function_mask, source_id, 689 domain_id, CTT_INV_G_DEVICE); 690 qinv_wait_sync(immu); 691 } 692 693 /* 694 * queued invalidation interface 695 * domain based context cache invalidation 696 */ 697 void 698 immu_qinv_context_dsi(immu_t *immu, uint_t domain_id) 699 { 700 qinv_context_common(immu, 0, 0, domain_id, CTT_INV_G_DOMAIN); 701 qinv_wait_sync(immu); 702 } 703 704 /* 705 * queued invalidation interface 706 * invalidation global context cache 707 */ 708 void 709 immu_qinv_context_gbl(immu_t *immu) 710 { 711 qinv_context_common(immu, 0, 0, 0, CTT_INV_G_GLOBAL); 712 qinv_wait_sync(immu); 713 } 714 715 /* 716 * queued invalidation interface 717 * paged based iotlb invalidation 718 */ 719 void 720 immu_inv_iotlb_psi(immu_t *immu, uint_t domain_id, 721 uint64_t dvma, uint_t count, uint_t hint) 722 { 723 uint_t am = 0; 724 uint_t max_am; 725 726 max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap); 727 728 /* choose page specified invalidation */ 729 if (IMMU_CAP_GET_PSI(immu->immu_regs_cap)) { 730 while (am <= max_am) { 731 if ((ADDR_AM_OFFSET(IMMU_BTOP(dvma), am) + count) 732 <= ADDR_AM_MAX(am)) { 733 qinv_iotlb_common(immu, domain_id, 734 dvma, am, hint, TLB_INV_G_PAGE); 735 break; 736 } 737 am++; 738 } 739 if (am > max_am) { 740 qinv_iotlb_common(immu, domain_id, 741 dvma, 0, hint, TLB_INV_G_DOMAIN); 742 } 743 744 /* choose domain invalidation */ 745 } else { 746 qinv_iotlb_common(immu, domain_id, dvma, 747 0, hint, TLB_INV_G_DOMAIN); 748 } 749 } 750 751 /* 752 * queued invalidation interface 753 * domain based iotlb invalidation 754 */ 755 void 756 immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id) 757 { 758 qinv_iotlb_common(immu, domain_id, 0, 0, 0, TLB_INV_G_DOMAIN); 759 qinv_wait_sync(immu); 760 } 761 762 /* 763 * queued invalidation interface 764 * global iotlb invalidation 765 */ 766 void 767 immu_qinv_iotlb_gbl(immu_t *immu) 768 { 769 qinv_iotlb_common(immu, 0, 0, 0, 0, TLB_INV_G_GLOBAL); 770 qinv_wait_sync(immu); 771 } 772 773 774 775 /* 776 * the plant wait operation for queued invalidation interface 777 */ 778 void 779 immu_qinv_plant(immu_t *immu, dvcookie_t *dvcookies, 780 uint_t count, uint_t array_size) 781 { 782 qinv_t *qinv; 783 qinv_iotlb_pend_node_t *node = NULL; 784 qinv_iotlb_pend_head_t *head; 785 786 qinv = (qinv_t *)immu->immu_qinv; 787 788 head = &(qinv->qinv_pend_head); 789 mutex_enter(&(head->ich_mem_lock)); 790 node = list_head(&(head->ich_mem_list)); 791 if (node) { 792 list_remove(&(head->ich_mem_list), node); 793 } 794 mutex_exit(&(head->ich_mem_lock)); 795 796 /* no cache, alloc one */ 797 if (node == NULL) { 798 node = kmem_zalloc(sizeof (qinv_iotlb_pend_node_t), KM_SLEEP); 799 } 800 node->icn_dvcookies = dvcookies; 801 node->icn_count = count; 802 node->icn_array_size = array_size; 803 804 /* plant an invalidation wait descriptor, not wait its completion */ 805 qinv_wait_async_unfence(immu, node); 806 } 807 808 /* 809 * the reap wait operation for queued invalidation interface 810 */ 811 void 812 immu_qinv_reap(immu_t *immu) 813 { 814 int index, cnt = 0; 815 qinv_iotlb_pend_node_t *node; 816 qinv_iotlb_pend_head_t *head; 817 qinv_t *qinv; 818 819 qinv = (qinv_t *)immu->immu_qinv; 820 head = &(qinv->qinv_pend_head); 821 822 index = qinv_wait_async_finish(immu, &cnt); 823 824 while (cnt--) { 825 node = qinv->qinv_iotlb_pend_node[index]; 826 if (node == NULL) 827 continue; 828 mutex_enter(&(head->ich_mem_lock)); 829 list_insert_head(&(head->ich_mem_list), node); 830 mutex_exit(&(head->ich_mem_lock)); 831 qinv->qinv_iotlb_pend_node[index] = NULL; 832 index++; 833 if (index == qinv->qinv_sync.qinv_mem_size) 834 index = 0; 835 } 836 } 837 838 839 /* queued invalidation interface -- global invalidate interrupt entry cache */ 840 void 841 immu_qinv_intr_global(immu_t *immu) 842 { 843 qinv_iec_common(immu, 0, 0, IEC_INV_GLOBAL); 844 qinv_wait_sync(immu); 845 } 846 847 /* queued invalidation interface -- invalidate single interrupt entry cache */ 848 void 849 immu_qinv_intr_one_cache(immu_t *immu, uint_t iidx) 850 { 851 qinv_iec_common(immu, iidx, 0, IEC_INV_INDEX); 852 qinv_wait_sync(immu); 853 } 854 855 /* queued invalidation interface -- invalidate interrupt entry caches */ 856 void 857 immu_qinv_intr_caches(immu_t *immu, uint_t iidx, uint_t cnt) 858 { 859 uint_t i, mask = 0; 860 861 ASSERT(cnt != 0); 862 863 /* requested interrupt count is not a power of 2 */ 864 if (!ISP2(cnt)) { 865 for (i = 0; i < cnt; i++) { 866 qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX); 867 } 868 qinv_wait_sync(immu); 869 return; 870 } 871 872 while ((2 << mask) < cnt) { 873 mask++; 874 } 875 876 if (mask > IMMU_ECAP_GET_MHMV(immu->immu_regs_excap)) { 877 for (i = 0; i < cnt; i++) { 878 qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX); 879 } 880 qinv_wait_sync(immu); 881 return; 882 } 883 884 qinv_iec_common(immu, iidx, mask, IEC_INV_INDEX); 885 886 qinv_wait_sync(immu); 887 } 888 889 void 890 immu_qinv_report_fault(immu_t *immu) 891 { 892 uint16_t head; 893 qinv_dsc_t *dsc; 894 qinv_t *qinv; 895 896 /* access qinv data */ 897 mutex_enter(&(immu->immu_qinv_lock)); 898 899 qinv = (qinv_t *)(immu->immu_qinv); 900 901 head = QINV_IQA_HEAD( 902 immu_regs_get64(immu, IMMU_REG_INVAL_QH)); 903 904 dsc = (qinv_dsc_t *)(qinv->qinv_table.qinv_mem_vaddr 905 + (head * QINV_ENTRY_SIZE)); 906 907 /* report the error */ 908 ddi_err(DER_WARN, immu->immu_dip, 909 "generated a fault when fetching a descriptor from the" 910 "\tinvalidation queue, or detects that the fetched" 911 "\tdescriptor is invalid. The head register is " 912 "0x%" PRIx64 913 "\tthe type is %s", 914 head, 915 qinv_dsc_type[MIN(INV_DSC_TYPE(dsc), QINV_MAX_DSC_TYPE)]); 916 917 mutex_exit(&(immu->immu_qinv_lock)); 918 } 919