1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019 Joyent, Inc. 14 * Copyright 2017 Tegile Systems, Inc. All rights reserved. 15 */ 16 17 /* 18 * ------------------------- 19 * Interrupt Handling Theory 20 * ------------------------- 21 * 22 * There are a couple different sets of interrupts that we need to worry about: 23 * 24 * - Interrupts from receive queues 25 * - Interrupts from transmit queues 26 * - 'Other Interrupts', such as the administrative queue 27 * 28 * 'Other Interrupts' are asynchronous events such as a link status change event 29 * being posted to the administrative queue, unrecoverable ECC errors, and more. 30 * If we have something being posted to the administrative queue, then we go 31 * through and process it, because it's generally enabled as a separate logical 32 * interrupt. Note, we may need to do more here eventually. To re-enable the 33 * interrupts from the 'Other Interrupts' section, we need to clear the PBA and 34 * write ENA to PFINT_ICR0. 35 * 36 * Interrupts from the transmit and receive queues indicates that our requests 37 * have been processed. In the rx case, it means that we have data that we 38 * should take a look at and send up the stack. In the tx case, it means that 39 * data which we got from MAC has now been sent out on the wire and we can free 40 * the associated data. Most of the logic for acting upon the presence of this 41 * data can be found in i40e_transciever.c which handles all of the DMA, rx, and 42 * tx operations. This file is dedicated to handling and dealing with interrupt 43 * processing. 44 * 45 * All devices supported by this driver support three kinds of interrupts: 46 * 47 * o Extended Message Signaled Interrupts (MSI-X) 48 * o Message Signaled Interrupts (MSI) 49 * o Legacy PCI interrupts (INTx) 50 * 51 * Generally speaking the hardware logically handles MSI and INTx the same and 52 * restricts us to only using a single interrupt, which isn't the interesting 53 * case. With MSI-X available, each physical function of the device provides the 54 * opportunity for multiple interrupts which is what we'll focus on. 55 * 56 * -------------------- 57 * Interrupt Management 58 * -------------------- 59 * 60 * By default, the admin queue, which consists of the asynchronous other 61 * interrupts is always bound to MSI-X vector zero. Next, we spread out all of 62 * the other interrupts that we have available to us over the remaining 63 * interrupt vectors. 64 * 65 * This means that there may be multiple queues, both tx and rx, which are 66 * mapped to the same interrupt. When the interrupt fires, we'll have to check 67 * all of them for servicing, before we go through and indicate that the 68 * interrupt is claimed. 69 * 70 * The hardware provides the means of mapping various queues to MSI-X interrupts 71 * by programming the I40E_QINT_RQCTL() and I4OE_QINT_TQCTL() registers. These 72 * registers can also be used to enable and disable whether or not the queue is 73 * a source of interrupts. As part of this, the hardware requires that we 74 * maintain a linked list of queues for each interrupt vector. While it may seem 75 * like this is only there for the purproses of ITRs, that's not the case. The 76 * first queue must be programmed in I40E_QINT_LNKLSTN(%vector) register. Each 77 * queue defines the next one in either the I40E_QINT_RQCTL or I40E_QINT_TQCTL 78 * register. 79 * 80 * Finally, the individual interrupt vector itself has the ability to be enabled 81 * and disabled. The overall interrupt is controlled through the 82 * I40E_PFINT_DYN_CTLN() register. This is used to turn on and off the interrupt 83 * as a whole. 84 * 85 * Note that this means that both the individual queue and the interrupt as a 86 * whole can be toggled and re-enabled. 87 * 88 * ------------------- 89 * Non-MSIX Management 90 * ------------------- 91 * 92 * We may have a case where the Operating System is unable to actually allocate 93 * any MSI-X to the system. In such a world, there is only one transmit/receive 94 * queue pair and it is bound to the same interrupt with index zero. The 95 * hardware doesn't allow us access to additional interrupt vectors in these 96 * modes. Note that technically we could support more transmit/receive queues if 97 * we wanted. 98 * 99 * In this world, because the interrupts for the admin queue and traffic are 100 * mixed together, we have to consult ICR0 to determine what has occurred. The 101 * QINT_TQCTL and QINT_RQCTL registers have a field, 'MSI-X 0 index' which 102 * allows us to set a specific bit in ICR0. There are up to seven such bits; 103 * however, we only use the bit 0 and 1 for the rx and tx queue respectively. 104 * These are contained by the I40E_INTR_NOTX_{R|T}X_QUEUE and 105 * I40E_INTR_NOTX_{R|T}X_MASK registers respectively. 106 * 107 * Unfortunately, these corresponding queue bits have no corresponding entry in 108 * the ICR0_ENA register. So instead, when enabling interrupts on the queues, we 109 * end up enabling it on the queue registers rather than on the MSI-X registers. 110 * In the MSI-X world, because they can be enabled and disabled, this is 111 * different and the queues can always be enabled and disabled, but the 112 * interrupts themselves are toggled (ignoring the question of interrupt 113 * blanking for polling on rings). 114 * 115 * Finally, we still have to set up the interrupt linked list, but the list is 116 * instead rooted at the register I40E_PFINT_LNKLST0, rather than being tied to 117 * one of the other MSI-X registers. 118 * 119 * -------------------- 120 * Interrupt Moderation 121 * -------------------- 122 * 123 * The XL710 hardware has three different interrupt moderation registers per 124 * interrupt. Unsurprisingly, we use these for: 125 * 126 * o RX interrupts 127 * o TX interrupts 128 * o 'Other interrupts' (link status change, admin queue, etc.) 129 * 130 * By default, we throttle 'other interrupts' the most, then TX interrupts, and 131 * then RX interrupts. The default values for these were based on trying to 132 * reason about both the importance and frequency of events. Generally speaking 133 * 'other interrupts' are not very frequent and they're not important for the 134 * I/O data path in and of itself (though they may indicate issues with the I/O 135 * data path). 136 * 137 * On the flip side, when we're not polling, RX interrupts are very important. 138 * The longer we wait for them, the more latency that we inject into the system. 139 * However, if we allow interrupts to occur too frequently, we risk a few 140 * problems: 141 * 142 * 1) Abusing system resources. Without proper interrupt blanking and polling, 143 * we can see upwards of 200k-300k interrupts per second on the system. 144 * 145 * 2) Not enough data coalescing to enable polling. In other words, the more 146 * data that we allow to build up, the more likely we'll be able to enable 147 * polling mode and allowing us to better handle bulk data. 148 * 149 * In-between the 'other interrupts' and the TX interrupts we have the 150 * reclamation of TX buffers. This operation is not quite as important as we 151 * generally size the ring large enough that we should be able to reclaim a 152 * substantial amount of the descriptors that we have used per interrupt. So 153 * while it's important that this interrupt occur, we don't necessarily need it 154 * firing as frequently as RX; it doesn't, on its own, induce additional latency 155 * into the system. 156 * 157 * Based on all this we currently assign static ITR values for the system. While 158 * we could move to a dynamic system (the hardware supports that), we'd want to 159 * make sure that we're seeing problems from this that we believe would be 160 * generally helped by the added complexity. 161 * 162 * Based on this, the default values that we have allow for the following 163 * interrupt thresholds: 164 * 165 * o 20k interrupts/s for RX 166 * o 5k interrupts/s for TX 167 * o 2k interupts/s for 'Other Interrupts' 168 */ 169 170 #include "i40e_sw.h" 171 172 #define I40E_INTR_NOTX_QUEUE 0 173 #define I40E_INTR_NOTX_INTR 0 174 #define I40E_INTR_NOTX_RX_QUEUE 0 175 #define I40E_INTR_NOTX_RX_MASK (1 << I40E_PFINT_ICR0_QUEUE_0_SHIFT) 176 #define I40E_INTR_NOTX_TX_QUEUE 1 177 #define I40E_INTR_NOTX_TX_MASK (1 << I40E_PFINT_ICR0_QUEUE_1_SHIFT) 178 179 void 180 i40e_intr_set_itr(i40e_t *i40e, i40e_itr_index_t itr, uint_t val) 181 { 182 int i; 183 i40e_hw_t *hw = &i40e->i40e_hw_space; 184 185 VERIFY3U(val, <=, I40E_MAX_ITR); 186 VERIFY3U(itr, <, I40E_ITR_INDEX_NONE); 187 188 /* 189 * No matter the interrupt mode, the ITR for other interrupts is always 190 * on interrupt zero and the same is true if we're not using MSI-X. 191 */ 192 if (itr == I40E_ITR_INDEX_OTHER || 193 i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 194 I40E_WRITE_REG(hw, I40E_PFINT_ITR0(itr), val); 195 return; 196 } 197 198 for (i = 0; i < i40e->i40e_num_trqpairs; i++) { 199 I40E_WRITE_REG(hw, I40E_PFINT_ITRN(itr, i), val); 200 } 201 } 202 203 /* 204 * Re-enable the adminq. Note that the adminq doesn't have a traditional queue 205 * associated with it from an interrupt perspective and just lives on ICR0. 206 * However when MSI-X interrupts are not being used, then this also enables and 207 * disables those interrupts. 208 */ 209 static void 210 i40e_intr_adminq_enable(i40e_t *i40e) 211 { 212 i40e_hw_t *hw = &i40e->i40e_hw_space; 213 uint32_t reg; 214 215 reg = I40E_PFINT_DYN_CTL0_INTENA_MASK | 216 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 217 (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 218 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg); 219 i40e_flush(hw); 220 } 221 222 static void 223 i40e_intr_adminq_disable(i40e_t *i40e) 224 { 225 i40e_hw_t *hw = &i40e->i40e_hw_space; 226 uint32_t reg; 227 228 reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT; 229 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTL0, reg); 230 } 231 232 /* 233 * The next two functions enable/disable the reception of interrupts 234 * on the given vector. Only vectors 1..N are programmed by these 235 * functions; vector 0 is special and handled by a different register. 236 * We must subtract one from the vector because i40e implicitly adds 237 * one to the vector value. See section 10.2.2.10.13 for more details. 238 */ 239 static void 240 i40e_intr_io_enable(i40e_t *i40e, int vector) 241 { 242 uint32_t reg; 243 i40e_hw_t *hw = &i40e->i40e_hw_space; 244 245 ASSERT3S(vector, >, 0); 246 reg = I40E_PFINT_DYN_CTLN_INTENA_MASK | 247 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 248 (I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 249 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg); 250 } 251 252 static void 253 i40e_intr_io_disable(i40e_t *i40e, int vector) 254 { 255 uint32_t reg; 256 i40e_hw_t *hw = &i40e->i40e_hw_space; 257 258 ASSERT3S(vector, >, 0); 259 reg = I40E_ITR_INDEX_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT; 260 I40E_WRITE_REG(hw, I40E_PFINT_DYN_CTLN(vector - 1), reg); 261 } 262 263 /* 264 * When MSI-X interrupts are being used, then we can enable the actual 265 * interrupts themselves. However, when they are not, we instead have to turn 266 * towards the queue's CAUSE_ENA bit and enable that. 267 */ 268 void 269 i40e_intr_io_enable_all(i40e_t *i40e) 270 { 271 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 272 int i; 273 274 for (i = 1; i < i40e->i40e_intr_count; i++) { 275 i40e_intr_io_enable(i40e, i); 276 } 277 } else { 278 uint32_t reg; 279 i40e_hw_t *hw = &i40e->i40e_hw_space; 280 281 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE)); 282 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 283 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 284 285 reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE)); 286 reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; 287 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 288 } 289 } 290 291 /* 292 * When MSI-X interrupts are being used, then we can disable the actual 293 * interrupts themselves. However, when they are not, we instead have to turn 294 * towards the queue's CAUSE_ENA bit and disable that. 295 */ 296 void 297 i40e_intr_io_disable_all(i40e_t *i40e) 298 { 299 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 300 int i; 301 302 for (i = 1; i < i40e->i40e_intr_count; i++) { 303 i40e_intr_io_disable(i40e, i); 304 } 305 } else { 306 uint32_t reg; 307 i40e_hw_t *hw = &i40e->i40e_hw_space; 308 309 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE)); 310 reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 311 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 312 313 reg = I40E_READ_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE)); 314 reg &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 315 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 316 } 317 } 318 319 /* 320 * As part of disabling the tx and rx queue's we're technically supposed to 321 * remove the linked list entries. The simplest way is to clear the LNKLSTN 322 * register by setting it to I40E_QUEUE_TYPE_EOL (0x7FF). 323 * 324 * Note all of the FM register access checks are performed by the caller. 325 */ 326 void 327 i40e_intr_io_clear_cause(i40e_t *i40e) 328 { 329 uint32_t i; 330 i40e_hw_t *hw = &i40e->i40e_hw_space; 331 332 if (i40e->i40e_intr_type != DDI_INTR_TYPE_MSIX) { 333 uint32_t reg; 334 reg = I40E_QUEUE_TYPE_EOL; 335 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg); 336 return; 337 } 338 339 for (i = 0; i < i40e->i40e_intr_count - 1; i++) { 340 uint32_t reg; 341 342 reg = I40E_QUEUE_TYPE_EOL; 343 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(i), reg); 344 } 345 346 i40e_flush(hw); 347 } 348 349 /* 350 * Finalize interrupt handling. Mostly this disables the admin queue. 351 */ 352 void 353 i40e_intr_chip_fini(i40e_t *i40e) 354 { 355 #ifdef DEBUG 356 int i; 357 uint32_t reg; 358 359 i40e_hw_t *hw = &i40e->i40e_hw_space; 360 361 /* 362 * Take a look and verify that all other interrupts have been disabled 363 * and the interrupt linked lists have been zeroed. 364 */ 365 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 366 for (i = 0; i < i40e->i40e_intr_count - 1; i++) { 367 reg = I40E_READ_REG(hw, I40E_PFINT_DYN_CTLN(i)); 368 VERIFY0(reg & I40E_PFINT_DYN_CTLN_INTENA_MASK); 369 370 reg = I40E_READ_REG(hw, I40E_PFINT_LNKLSTN(i)); 371 VERIFY3U(reg, ==, I40E_QUEUE_TYPE_EOL); 372 } 373 } 374 #endif 375 376 i40e_intr_adminq_disable(i40e); 377 } 378 379 /* 380 * Set the head of the interrupt linked list. The PFINT_LNKLSTN[N] 381 * register actually refers to the 'N + 1' interrupt vector. E.g., 382 * PFINT_LNKLSTN[0] refers to interrupt vector 1. 383 */ 384 static void 385 i40e_set_lnklstn(i40e_t *i40e, uint_t vector, uint_t queue) 386 { 387 uint32_t reg; 388 i40e_hw_t *hw = &i40e->i40e_hw_space; 389 390 reg = (queue << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) | 391 (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT); 392 393 I40E_WRITE_REG(hw, I40E_PFINT_LNKLSTN(vector), reg); 394 DEBUGOUT2("PFINT_LNKLSTN[%u] = 0x%x", vector, reg); 395 } 396 397 /* 398 * Set the QINT_RQCTL[queue] register. The next queue is always the Tx 399 * queue associated with this Rx queue. Unlike PFINT_LNKLSTN, the 400 * vector should be the actual vector this queue is on -- i.e., it 401 * should be equal to itrq_rx_intrvec. 402 */ 403 static void 404 i40e_set_rqctl(i40e_t *i40e, uint_t vector, uint_t queue) 405 { 406 uint32_t reg; 407 i40e_hw_t *hw = &i40e->i40e_hw_space; 408 409 ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_rx_intrvec); 410 411 reg = (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 412 (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 413 (queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 414 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) | 415 I40E_QINT_RQCTL_CAUSE_ENA_MASK; 416 417 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 418 DEBUGOUT2("QINT_RQCTL[%u] = 0x%x", queue, reg); 419 } 420 421 /* 422 * Like i40e_set_rqctl(), but for QINT_TQCTL[queue]. The next queue is 423 * either the Rx queue of another TRQP, or EOL. 424 */ 425 static void 426 i40e_set_tqctl(i40e_t *i40e, uint_t vector, uint_t queue, uint_t next_queue) 427 { 428 uint32_t reg; 429 i40e_hw_t *hw = &i40e->i40e_hw_space; 430 431 ASSERT3U(vector, ==, i40e->i40e_trqpairs[queue].itrq_tx_intrvec); 432 433 reg = (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 434 (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 435 (next_queue << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 436 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) | 437 I40E_QINT_TQCTL_CAUSE_ENA_MASK; 438 439 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(queue), reg); 440 DEBUGOUT2("QINT_TQCTL[%u] = 0x%x", queue, reg); 441 } 442 443 /* 444 * Program the interrupt linked list. Each vector has a linked list of 445 * queues which act as event sources for that vector. When one of 446 * those sources has an event the associated interrupt vector is 447 * fired. This mapping must match the mapping found in 448 * i40e_map_intrs_to_vectors(). 449 * 450 * See section 7.5.3 for more information about the configuration of 451 * the interrupt linked list. 452 */ 453 static void 454 i40e_intr_init_queue_msix(i40e_t *i40e) 455 { 456 uint_t intr_count; 457 458 /* 459 * The 0th vector is for 'Other Interrupts' only (subject to 460 * change in the future). 461 */ 462 intr_count = i40e->i40e_intr_count - 1; 463 464 for (uint_t vec = 0; vec < intr_count; vec++) { 465 boolean_t head = B_TRUE; 466 467 for (uint_t qidx = vec; qidx < i40e->i40e_num_trqpairs; 468 qidx += intr_count) { 469 uint_t next_qidx = qidx + intr_count; 470 471 next_qidx = (next_qidx > i40e->i40e_num_trqpairs) ? 472 I40E_QUEUE_TYPE_EOL : next_qidx; 473 474 if (head) { 475 i40e_set_lnklstn(i40e, vec, qidx); 476 head = B_FALSE; 477 } 478 479 i40e_set_rqctl(i40e, vec + 1, qidx); 480 i40e_set_tqctl(i40e, vec + 1, qidx, next_qidx); 481 } 482 } 483 } 484 485 /* 486 * Set up a single queue to share the admin queue interrupt in the non-MSI-X 487 * world. Note we do not enable the queue as an interrupt cause at this time. We 488 * don't have any other vector of control here, unlike with the MSI-X interrupt 489 * case. 490 */ 491 static void 492 i40e_intr_init_queue_shared(i40e_t *i40e) 493 { 494 i40e_hw_t *hw = &i40e->i40e_hw_space; 495 uint32_t reg; 496 497 VERIFY(i40e->i40e_intr_type == DDI_INTR_TYPE_FIXED || 498 i40e->i40e_intr_type == DDI_INTR_TYPE_MSI); 499 500 reg = (I40E_INTR_NOTX_QUEUE << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) | 501 (I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT); 502 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, reg); 503 504 reg = (I40E_INTR_NOTX_INTR << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 505 (I40E_ITR_INDEX_RX << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 506 (I40E_INTR_NOTX_RX_QUEUE << I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) | 507 (I40E_INTR_NOTX_QUEUE << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 508 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 509 510 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(I40E_INTR_NOTX_QUEUE), reg); 511 512 reg = (I40E_INTR_NOTX_INTR << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 513 (I40E_ITR_INDEX_TX << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 514 (I40E_INTR_NOTX_TX_QUEUE << I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) | 515 (I40E_QUEUE_TYPE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 516 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 517 518 I40E_WRITE_REG(hw, I40E_QINT_TQCTL(I40E_INTR_NOTX_QUEUE), reg); 519 } 520 521 /* 522 * Enable the specified queue as a valid source of interrupts. Note, this should 523 * only be used as part of the GLDv3's interrupt blanking routines. The debug 524 * build assertions are specific to that. 525 */ 526 void 527 i40e_intr_rx_queue_enable(i40e_trqpair_t *itrq) 528 { 529 uint32_t reg; 530 uint_t queue = itrq->itrq_index; 531 i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space; 532 533 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); 534 ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs); 535 536 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue)); 537 ASSERT0(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK); 538 reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; 539 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 540 } 541 542 /* 543 * Disable the specified queue as a valid source of interrupts. Note, this 544 * should only be used as part of the GLDv3's interrupt blanking routines. The 545 * debug build assertions are specific to that. 546 */ 547 void 548 i40e_intr_rx_queue_disable(i40e_trqpair_t *itrq) 549 { 550 uint32_t reg; 551 uint_t queue = itrq->itrq_index; 552 i40e_hw_t *hw = &itrq->itrq_i40e->i40e_hw_space; 553 554 ASSERT(MUTEX_HELD(&itrq->itrq_rx_lock)); 555 ASSERT(queue < itrq->itrq_i40e->i40e_num_trqpairs); 556 557 reg = I40E_READ_REG(hw, I40E_QINT_RQCTL(queue)); 558 ASSERT3U(reg & I40E_QINT_RQCTL_CAUSE_ENA_MASK, ==, 559 I40E_QINT_RQCTL_CAUSE_ENA_MASK); 560 reg &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 561 I40E_WRITE_REG(hw, I40E_QINT_RQCTL(queue), reg); 562 } 563 564 /* 565 * Start up the various chip's interrupt handling. We not only configure the 566 * adminq here, but we also go through and configure all of the actual queues, 567 * the interrupt linked lists, and others. 568 */ 569 void 570 i40e_intr_chip_init(i40e_t *i40e) 571 { 572 i40e_hw_t *hw = &i40e->i40e_hw_space; 573 uint32_t reg; 574 575 /* 576 * Ensure that all non adminq interrupts are disabled at the chip level. 577 */ 578 i40e_intr_io_disable_all(i40e); 579 580 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, 0); 581 (void) I40E_READ_REG(hw, I40E_PFINT_ICR0); 582 583 /* 584 * Always enable all of the other-class interrupts to be on their own 585 * ITR. This only needs to be set on interrupt zero, which has its own 586 * special setting. 587 */ 588 reg = I40E_ITR_INDEX_OTHER << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT; 589 I40E_WRITE_REG(hw, I40E_PFINT_STAT_CTL0, reg); 590 591 /* 592 * Enable interrupt types we expect to receive. At the moment, this 593 * is limited to the adminq; however, we'll want to review 11.2.2.9.22 594 * for more types here as we add support for detecting them, handling 595 * them, and resetting the device as appropriate. 596 */ 597 reg = I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 598 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg); 599 600 /* 601 * Always set the interrupt linked list to empty. We'll come back and 602 * change this if MSI-X are actually on the scene. 603 */ 604 I40E_WRITE_REG(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_TYPE_EOL); 605 606 i40e_intr_adminq_enable(i40e); 607 608 /* 609 * Set up all of the queues and map them to interrupts based on the bit 610 * assignments. 611 */ 612 if (i40e->i40e_intr_type == DDI_INTR_TYPE_MSIX) { 613 i40e_intr_init_queue_msix(i40e); 614 } else { 615 i40e_intr_init_queue_shared(i40e); 616 } 617 618 /* 619 * Finally set all of the default ITRs for the interrupts. Note that the 620 * queues will have been set up above. 621 */ 622 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_RX, i40e->i40e_rx_itr); 623 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_TX, i40e->i40e_tx_itr); 624 i40e_intr_set_itr(i40e, I40E_ITR_INDEX_OTHER, i40e->i40e_other_itr); 625 } 626 627 static void 628 i40e_intr_adminq_work(i40e_t *i40e) 629 { 630 struct i40e_hw *hw = &i40e->i40e_hw_space; 631 struct i40e_arq_event_info evt; 632 uint16_t remain = 1; 633 634 bzero(&evt, sizeof (struct i40e_arq_event_info)); 635 evt.buf_len = I40E_ADMINQ_BUFSZ; 636 evt.msg_buf = i40e->i40e_aqbuf; 637 638 while (remain != 0) { 639 enum i40e_status_code ret; 640 uint16_t opcode; 641 642 /* 643 * At the moment, the only error code that seems to be returned 644 * is one saying that there's no work. In such a case we leave 645 * this be. 646 */ 647 ret = i40e_clean_arq_element(hw, &evt, &remain); 648 if (ret != I40E_SUCCESS) 649 break; 650 651 opcode = LE_16(evt.desc.opcode); 652 switch (opcode) { 653 case i40e_aqc_opc_get_link_status: 654 mutex_enter(&i40e->i40e_general_lock); 655 i40e_link_check(i40e); 656 mutex_exit(&i40e->i40e_general_lock); 657 break; 658 default: 659 /* 660 * Longer term we'll want to enable other causes here 661 * and get these cleaned up and doing something. 662 */ 663 break; 664 } 665 } 666 } 667 668 static void 669 i40e_intr_rx_work(i40e_t *i40e, i40e_trqpair_t *itrq) 670 { 671 mblk_t *mp = NULL; 672 673 mutex_enter(&itrq->itrq_rx_lock); 674 if (!itrq->itrq_intr_poll) 675 mp = i40e_ring_rx(itrq, I40E_POLL_NULL); 676 mutex_exit(&itrq->itrq_rx_lock); 677 678 if (mp == NULL) 679 return; 680 681 mac_rx_ring(i40e->i40e_mac_hdl, itrq->itrq_macrxring, mp, 682 itrq->itrq_rxgen); 683 } 684 685 /* ARGSUSED */ 686 static void 687 i40e_intr_tx_work(i40e_t *i40e, i40e_trqpair_t *itrq) 688 { 689 i40e_tx_recycle_ring(itrq); 690 } 691 692 /* 693 * At the moment, the only 'other' interrupt on ICR0 that we handle is the 694 * adminq. We should go through and support the other notifications at some 695 * point. 696 */ 697 static void 698 i40e_intr_other_work(i40e_t *i40e) 699 { 700 struct i40e_hw *hw = &i40e->i40e_hw_space; 701 uint32_t reg; 702 703 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0); 704 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 705 DDI_FM_OK) { 706 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED); 707 atomic_or_32(&i40e->i40e_state, I40E_ERROR); 708 return; 709 } 710 711 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 712 i40e_intr_adminq_work(i40e); 713 714 /* 715 * Make sure that the adminq interrupt is not masked and then explicitly 716 * enable the adminq and thus the other interrupt. 717 */ 718 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0_ENA); 719 reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 720 I40E_WRITE_REG(hw, I40E_PFINT_ICR0_ENA, reg); 721 722 i40e_intr_adminq_enable(i40e); 723 } 724 725 /* 726 * Handle an MSI-X interrupt. See section 7.5.1.3 for an overview of 727 * the MSI-X interrupt sequence. 728 */ 729 uint_t 730 i40e_intr_msix(void *arg1, void *arg2) 731 { 732 i40e_t *i40e = (i40e_t *)arg1; 733 uint_t vector_idx = (uint_t)(uintptr_t)arg2; 734 735 ASSERT3U(vector_idx, <, i40e->i40e_intr_count); 736 737 /* 738 * When using MSI-X interrupts, vector 0 is always reserved for the 739 * adminq at this time. Though longer term, we'll want to also bridge 740 * some I/O to them. 741 */ 742 if (vector_idx == 0) { 743 i40e_intr_other_work(i40e); 744 return (DDI_INTR_CLAIMED); 745 } 746 747 ASSERT3U(vector_idx, >, 0); 748 749 /* 750 * We determine the queue indexes via simple arithmetic (as 751 * opposed to keeping explicit state like a bitmap). While 752 * conveinent, it does mean that i40e_map_intrs_to_vectors(), 753 * i40e_intr_init_queue_msix(), and this function must be 754 * modified as a unit. 755 * 756 * We subtract 1 from the vector to offset the addition we 757 * performed during i40e_map_intrs_to_vectors(). 758 */ 759 for (uint_t i = vector_idx - 1; i < i40e->i40e_num_trqpairs; 760 i += (i40e->i40e_intr_count - 1)) { 761 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[i]; 762 763 ASSERT3U(i, <, i40e->i40e_num_trqpairs); 764 ASSERT3P(itrq, !=, NULL); 765 i40e_intr_rx_work(i40e, itrq); 766 i40e_intr_tx_work(i40e, itrq); 767 } 768 769 i40e_intr_io_enable(i40e, vector_idx); 770 return (DDI_INTR_CLAIMED); 771 } 772 773 static uint_t 774 i40e_intr_notx(i40e_t *i40e, boolean_t shared) 775 { 776 i40e_hw_t *hw = &i40e->i40e_hw_space; 777 uint32_t reg; 778 i40e_trqpair_t *itrq = &i40e->i40e_trqpairs[0]; 779 int ret = DDI_INTR_CLAIMED; 780 781 if (shared == B_TRUE) { 782 mutex_enter(&i40e->i40e_general_lock); 783 if (i40e->i40e_state & I40E_SUSPENDED) { 784 mutex_exit(&i40e->i40e_general_lock); 785 return (DDI_INTR_UNCLAIMED); 786 } 787 mutex_exit(&i40e->i40e_general_lock); 788 } 789 790 reg = I40E_READ_REG(hw, I40E_PFINT_ICR0); 791 if (i40e_check_acc_handle(i40e->i40e_osdep_space.ios_reg_handle) != 792 DDI_FM_OK) { 793 ddi_fm_service_impact(i40e->i40e_dip, DDI_SERVICE_DEGRADED); 794 atomic_or_32(&i40e->i40e_state, I40E_ERROR); 795 return (DDI_INTR_CLAIMED); 796 } 797 798 if (reg == 0) { 799 if (shared == B_TRUE) 800 ret = DDI_INTR_UNCLAIMED; 801 goto done; 802 } 803 804 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) 805 i40e_intr_adminq_work(i40e); 806 807 if (reg & I40E_INTR_NOTX_RX_MASK) 808 i40e_intr_rx_work(i40e, itrq); 809 810 if (reg & I40E_INTR_NOTX_TX_MASK) 811 i40e_intr_tx_work(i40e, itrq); 812 813 done: 814 i40e_intr_adminq_enable(i40e); 815 return (ret); 816 817 } 818 819 /* ARGSUSED */ 820 uint_t 821 i40e_intr_msi(void *arg1, void *arg2) 822 { 823 i40e_t *i40e = (i40e_t *)arg1; 824 825 return (i40e_intr_notx(i40e, B_FALSE)); 826 } 827 828 /* ARGSUSED */ 829 uint_t 830 i40e_intr_legacy(void *arg1, void *arg2) 831 { 832 i40e_t *i40e = (i40e_t *)arg1; 833 834 return (i40e_intr_notx(i40e, B_TRUE)); 835 } 836