1 /*
2 * BSD LICENSE
3 *
4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Cavium, Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include "lio_bsd.h"
35 #include "lio_common.h"
36 #include "lio_droq.h"
37 #include "lio_iq.h"
38 #include "lio_response_manager.h"
39 #include "lio_device.h"
40 #include "lio_ctrl.h"
41 #include "lio_main.h"
42 #include "lio_rxtx.h"
43 #include "lio_network.h"
44
45 int
lio_set_feature(if_t ifp,int cmd,uint16_t param1)46 lio_set_feature(if_t ifp, int cmd, uint16_t param1)
47 {
48 struct lio_ctrl_pkt nctrl;
49 struct lio *lio = if_getsoftc(ifp);
50 struct octeon_device *oct = lio->oct_dev;
51 int ret = 0;
52
53 bzero(&nctrl, sizeof(struct lio_ctrl_pkt));
54
55 nctrl.ncmd.cmd64 = 0;
56 nctrl.ncmd.s.cmd = cmd;
57 nctrl.ncmd.s.param1 = param1;
58 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
59 nctrl.wait_time = 100;
60 nctrl.lio = lio;
61 nctrl.cb_fn = lio_ctrl_cmd_completion;
62
63 ret = lio_send_ctrl_pkt(lio->oct_dev, &nctrl);
64 if (ret < 0) {
65 lio_dev_err(oct, "Feature change failed in core (ret: 0x%x)\n",
66 ret);
67 }
68
69 return (ret);
70 }
71
72 void
lio_ctrl_cmd_completion(void * nctrl_ptr)73 lio_ctrl_cmd_completion(void *nctrl_ptr)
74 {
75 struct lio_ctrl_pkt *nctrl = (struct lio_ctrl_pkt *)nctrl_ptr;
76 struct lio *lio;
77 struct octeon_device *oct;
78 uint8_t *mac;
79
80 lio = nctrl->lio;
81
82 if (lio->oct_dev == NULL)
83 return;
84
85 oct = lio->oct_dev;
86
87 switch (nctrl->ncmd.s.cmd) {
88 case LIO_CMD_CHANGE_DEVFLAGS:
89 case LIO_CMD_SET_MULTI_LIST:
90 break;
91
92 case LIO_CMD_CHANGE_MACADDR:
93 mac = ((uint8_t *)&nctrl->udd[0]) + 2;
94 if (nctrl->ncmd.s.param1) {
95 /* vfidx is 0 based, but vf_num (param1) is 1 based */
96 int vfidx = nctrl->ncmd.s.param1 - 1;
97 bool mac_is_admin_assigned = nctrl->ncmd.s.param2;
98
99 if (mac_is_admin_assigned)
100 lio_dev_info(oct, "MAC Address %pM is configured for VF %d\n",
101 mac, vfidx);
102 } else {
103 lio_dev_info(oct, "MAC Address changed to %02x:%02x:%02x:%02x:%02x:%02x\n",
104 mac[0], mac[1], mac[2], mac[3], mac[4],
105 mac[5]);
106 }
107 break;
108
109 case LIO_CMD_GPIO_ACCESS:
110 lio_dev_info(oct, "LED Flashing visual identification\n");
111 break;
112
113 case LIO_CMD_ID_ACTIVE:
114 lio_dev_info(oct, "LED Flashing visual identification\n");
115 break;
116
117 case LIO_CMD_LRO_ENABLE:
118 lio_dev_info(oct, "HW LRO Enabled\n");
119 break;
120
121 case LIO_CMD_LRO_DISABLE:
122 lio_dev_info(oct, "HW LRO Disabled\n");
123 break;
124
125 case LIO_CMD_VERBOSE_ENABLE:
126 lio_dev_info(oct, "Firmware debug enabled\n");
127 break;
128
129 case LIO_CMD_VERBOSE_DISABLE:
130 lio_dev_info(oct, "Firmware debug disabled\n");
131 break;
132
133 case LIO_CMD_VLAN_FILTER_CTL:
134 if (nctrl->ncmd.s.param1)
135 lio_dev_info(oct, "VLAN filter enabled\n");
136 else
137 lio_dev_info(oct, "VLAN filter disabled\n");
138 break;
139
140 case LIO_CMD_ADD_VLAN_FILTER:
141 lio_dev_info(oct, "VLAN filter %d added\n",
142 nctrl->ncmd.s.param1);
143 break;
144
145 case LIO_CMD_DEL_VLAN_FILTER:
146 lio_dev_info(oct, "VLAN filter %d removed\n",
147 nctrl->ncmd.s.param1);
148 break;
149
150 case LIO_CMD_SET_SETTINGS:
151 lio_dev_info(oct, "Settings changed\n");
152 break;
153
154 /*
155 * Case to handle "LIO_CMD_TNL_RX_CSUM_CTL"
156 * Command passed by NIC driver
157 */
158 case LIO_CMD_TNL_RX_CSUM_CTL:
159 if (nctrl->ncmd.s.param1 == LIO_CMD_RXCSUM_ENABLE) {
160 lio_dev_info(oct, "RX Checksum Offload Enabled\n");
161 } else if (nctrl->ncmd.s.param1 == LIO_CMD_RXCSUM_DISABLE) {
162 lio_dev_info(oct, "RX Checksum Offload Disabled\n");
163 }
164 break;
165
166 /*
167 * Case to handle "LIO_CMD_TNL_TX_CSUM_CTL"
168 * Command passed by NIC driver
169 */
170 case LIO_CMD_TNL_TX_CSUM_CTL:
171 if (nctrl->ncmd.s.param1 == LIO_CMD_TXCSUM_ENABLE) {
172 lio_dev_info(oct, "TX Checksum Offload Enabled\n");
173 } else if (nctrl->ncmd.s.param1 == LIO_CMD_TXCSUM_DISABLE) {
174 lio_dev_info(oct, "TX Checksum Offload Disabled\n");
175 }
176 break;
177
178 /*
179 * Case to handle "LIO_CMD_VXLAN_PORT_CONFIG"
180 * Command passed by NIC driver
181 */
182 case LIO_CMD_VXLAN_PORT_CONFIG:
183 if (nctrl->ncmd.s.more == LIO_CMD_VXLAN_PORT_ADD) {
184 lio_dev_info(oct, "VxLAN Destination UDP PORT:%d ADDED\n",
185 nctrl->ncmd.s.param1);
186 } else if (nctrl->ncmd.s.more == LIO_CMD_VXLAN_PORT_DEL) {
187 lio_dev_info(oct, "VxLAN Destination UDP PORT:%d DELETED\n",
188 nctrl->ncmd.s.param1);
189 }
190 break;
191
192 case LIO_CMD_SET_FLOW_CTL:
193 lio_dev_info(oct, "Set RX/TX flow control parameters\n");
194 break;
195
196 case LIO_CMD_SET_FNV:
197 if (nctrl->ncmd.s.param1 == LIO_CMD_FNV_ENABLE)
198 lio_dev_info(oct, "FNV Enabled\n");
199 else if (nctrl->ncmd.s.param1 == LIO_CMD_FNV_DISABLE)
200 lio_dev_info(oct, "FNV Disabled\n");
201 break;
202
203 case LIO_CMD_PKT_STEERING_CTL:
204 if (nctrl->ncmd.s.param1 == LIO_CMD_PKT_STEERING_ENABLE) {
205 lio_dev_info(oct, "Packet Steering Enabled\n");
206 } else if (nctrl->ncmd.s.param1 ==
207 LIO_CMD_PKT_STEERING_DISABLE) {
208 lio_dev_info(oct, "Packet Steering Disabled\n");
209 }
210
211 break;
212
213 case LIO_CMD_QUEUE_COUNT_CTL:
214 lio_dev_info(oct, "Queue count updated to %d\n",
215 nctrl->ncmd.s.param1);
216 break;
217
218 default:
219 lio_dev_err(oct, "%s Unknown cmd %d\n", __func__,
220 nctrl->ncmd.s.cmd);
221 }
222 }
223
224
225 /*
226 * \brief Setup output queue
227 * @param oct octeon device
228 * @param q_no which queue
229 * @param num_descs how many descriptors
230 * @param desc_size size of each descriptor
231 * @param app_ctx application context
232 */
233 static int
lio_setup_droq(struct octeon_device * oct,int q_no,int num_descs,int desc_size,void * app_ctx)234 lio_setup_droq(struct octeon_device *oct, int q_no, int num_descs,
235 int desc_size, void *app_ctx)
236 {
237 int ret_val = 0;
238
239 lio_dev_dbg(oct, "Creating Droq: %d\n", q_no);
240 /* droq creation and local register settings. */
241 ret_val = lio_create_droq(oct, q_no, num_descs, desc_size, app_ctx);
242 if (ret_val < 0)
243 return (ret_val);
244
245 if (ret_val == 1) {
246 lio_dev_dbg(oct, "Using default droq %d\n", q_no);
247 return (0);
248 }
249
250 /*
251 * Send Credit for Octeon Output queues. Credits are always
252 * sent after the output queue is enabled.
253 */
254 lio_write_csr32(oct, oct->droq[q_no]->pkts_credit_reg,
255 oct->droq[q_no]->max_count);
256
257 return (ret_val);
258 }
259
260 static void
lio_push_packet(void * m_buff,uint32_t len,union octeon_rh * rh,void * rxq,void * arg)261 lio_push_packet(void *m_buff, uint32_t len, union octeon_rh *rh, void *rxq,
262 void *arg)
263 {
264 struct mbuf *mbuf = m_buff;
265 if_t ifp = arg;
266 struct lio_droq *droq = rxq;
267
268 if (ifp != NULL) {
269 struct lio *lio = if_getsoftc(ifp);
270
271 /* Do not proceed if the interface is not in RUNNING state. */
272 if (!lio_ifstate_check(lio, LIO_IFSTATE_RUNNING)) {
273 lio_recv_buffer_free(mbuf);
274 droq->stats.rx_dropped++;
275 return;
276 }
277
278 if (rh->r_dh.has_hash) {
279 uint32_t hashtype, hashval;
280
281 if (rh->r_dh.has_hwtstamp) {
282 hashval = htobe32(*(uint32_t *)
283 (((uint8_t *)mbuf->m_data) +
284 ((rh->r_dh.len - 2) *
285 BYTES_PER_DHLEN_UNIT)));
286 hashtype =
287 htobe32(*(((uint32_t *)
288 (((uint8_t *)mbuf->m_data) +
289 ((rh->r_dh.len - 2) *
290 BYTES_PER_DHLEN_UNIT))) + 1));
291 } else {
292 hashval = htobe32(*(uint32_t *)
293 (((uint8_t *)mbuf->m_data) +
294 ((rh->r_dh.len - 1) *
295 BYTES_PER_DHLEN_UNIT)));
296 hashtype =
297 htobe32(*(((uint32_t *)
298 (((uint8_t *)mbuf->m_data) +
299 ((rh->r_dh.len - 1) *
300 BYTES_PER_DHLEN_UNIT))) + 1));
301 }
302
303 mbuf->m_pkthdr.flowid = hashval;
304
305 switch (hashtype) {
306 case LIO_RSS_HASH_IPV4:
307 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
308 break;
309 case LIO_RSS_HASH_TCP_IPV4:
310 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
311 break;
312 case LIO_RSS_HASH_IPV6:
313 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
314 break;
315 case LIO_RSS_HASH_TCP_IPV6:
316 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
317 break;
318 case LIO_RSS_HASH_IPV6_EX:
319 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6_EX);
320 break;
321 case LIO_RSS_HASH_TCP_IPV6_EX:
322 M_HASHTYPE_SET(mbuf,
323 M_HASHTYPE_RSS_TCP_IPV6_EX);
324 break;
325 default:
326 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
327 }
328
329 } else {
330 /*
331 * This case won't hit as FW will always set has_hash
332 * in rh.
333 */
334 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE);
335 mbuf->m_pkthdr.flowid = droq->q_no;
336 }
337
338 m_adj(mbuf, rh->r_dh.len * 8);
339 len -= rh->r_dh.len * 8;
340 mbuf->m_flags |= M_PKTHDR;
341
342 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) &&
343 (rh->r_dh.priority || rh->r_dh.vlan)) {
344 uint16_t priority = rh->r_dh.priority;
345 uint16_t vid = rh->r_dh.vlan;
346 uint16_t vtag;
347
348 vtag = priority << 13 | vid;
349 mbuf->m_pkthdr.ether_vtag = vtag;
350 mbuf->m_flags |= M_VLANTAG;
351 }
352
353 if (rh->r_dh.csum_verified & LIO_IPSUM_VERIFIED)
354 mbuf->m_pkthdr.csum_flags |= (CSUM_L3_CALC |
355 CSUM_L3_VALID);
356
357 if (rh->r_dh.csum_verified & LIO_L4SUM_VERIFIED) {
358 mbuf->m_pkthdr.csum_flags |= (CSUM_L4_CALC |
359 CSUM_L4_VALID);
360 mbuf->m_pkthdr.csum_flags |= (CSUM_DATA_VALID |
361 CSUM_PSEUDO_HDR);
362 mbuf->m_pkthdr.csum_data = htons(0xffff);
363 }
364
365 mbuf->m_pkthdr.rcvif = ifp;
366 mbuf->m_pkthdr.len = len;
367
368 if ((lio_hwlro == 0) &&
369 (if_getcapenable(ifp) & IFCAP_LRO) &&
370 (mbuf->m_pkthdr.csum_flags &
371 (CSUM_L3_VALID | CSUM_L4_VALID | CSUM_DATA_VALID |
372 CSUM_PSEUDO_HDR)) == (CSUM_L3_VALID | CSUM_L4_VALID |
373 CSUM_DATA_VALID |
374 CSUM_PSEUDO_HDR)) {
375 if (droq->lro.lro_cnt) {
376 if (tcp_lro_rx(&droq->lro, mbuf, 0) == 0) {
377 droq->stats.rx_bytes_received += len;
378 droq->stats.rx_pkts_received++;
379 return;
380 }
381 }
382 }
383
384 if_input(ifp, mbuf);
385
386 droq->stats.rx_bytes_received += len;
387 droq->stats.rx_pkts_received++;
388
389 } else {
390 lio_recv_buffer_free(mbuf);
391 droq->stats.rx_dropped++;
392 }
393 }
394
395 /*
396 * \brief Setup input and output queues
397 * @param octeon_dev octeon device
398 * @param ifidx Interface Index
399 *
400 * Note: Queues are with respect to the octeon device. Thus
401 * an input queue is for egress packets, and output queues
402 * are for ingress packets.
403 */
404 int
lio_setup_io_queues(struct octeon_device * octeon_dev,int ifidx,uint32_t num_iqs,uint32_t num_oqs)405 lio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
406 uint32_t num_iqs, uint32_t num_oqs)
407 {
408 struct lio_droq_ops droq_ops;
409 if_t ifp;
410 struct lio_droq *droq;
411 struct lio *lio;
412 static int cpu_id, cpu_id_modulus;
413 int num_tx_descs, q, q_no, retval = 0;
414
415 ifp = octeon_dev->props.ifp;
416
417 lio = if_getsoftc(ifp);
418
419 bzero(&droq_ops, sizeof(struct lio_droq_ops));
420
421 droq_ops.fptr = lio_push_packet;
422 droq_ops.farg = (void *)ifp;
423
424 cpu_id = 0;
425 cpu_id_modulus = mp_ncpus;
426 /* set up DROQs. */
427 for (q = 0; q < num_oqs; q++) {
428 q_no = lio->linfo.rxpciq[q].s.q_no;
429 lio_dev_dbg(octeon_dev, "lio_setup_io_queues index:%d linfo.rxpciq.s.q_no:%d\n",
430 q, q_no);
431 retval = lio_setup_droq(octeon_dev, q_no,
432 LIO_GET_NUM_RX_DESCS_NIC_IF_CFG(
433 lio_get_conf(octeon_dev),
434 lio->ifidx),
435 LIO_GET_NUM_RX_BUF_SIZE_NIC_IF_CFG(
436 lio_get_conf(octeon_dev),
437 lio->ifidx), NULL);
438 if (retval) {
439 lio_dev_err(octeon_dev, "%s : Runtime DROQ(RxQ) creation failed.\n",
440 __func__);
441 return (1);
442 }
443
444 droq = octeon_dev->droq[q_no];
445
446 /* designate a CPU for this droq */
447 droq->cpu_id = cpu_id;
448 cpu_id++;
449 if (cpu_id >= cpu_id_modulus)
450 cpu_id = 0;
451
452 lio_register_droq_ops(octeon_dev, q_no, &droq_ops);
453 }
454
455 /* set up IQs. */
456 for (q = 0; q < num_iqs; q++) {
457 num_tx_descs = LIO_GET_NUM_TX_DESCS_NIC_IF_CFG(
458 lio_get_conf(octeon_dev),
459 lio->ifidx);
460 retval = lio_setup_iq(octeon_dev, ifidx, q,
461 lio->linfo.txpciq[q], num_tx_descs);
462 if (retval) {
463 lio_dev_err(octeon_dev, " %s : Runtime IQ(TxQ) creation failed.\n",
464 __func__);
465 return (1);
466 }
467 }
468
469 return (0);
470 }
471
472 /*
473 * \brief Droq packet processor sceduler
474 * @param oct octeon device
475 */
476 static void
lio_schedule_droq_pkt_handlers(struct octeon_device * oct)477 lio_schedule_droq_pkt_handlers(struct octeon_device *oct)
478 {
479 struct lio_droq *droq;
480 uint64_t oq_no;
481
482 if (oct->int_status & LIO_DEV_INTR_PKT_DATA) {
483 for (oq_no = 0; oq_no < LIO_MAX_OUTPUT_QUEUES(oct); oq_no++) {
484 if (!(oct->io_qmask.oq & BIT_ULL(oq_no)))
485 continue;
486
487 droq = oct->droq[oq_no];
488
489 taskqueue_enqueue(droq->droq_taskqueue,
490 &droq->droq_task);
491 }
492 }
493 }
494
495 static void
lio_msix_intr_handler(void * vector)496 lio_msix_intr_handler(void *vector)
497 {
498 struct lio_ioq_vector *ioq_vector = (struct lio_ioq_vector *)vector;
499 struct octeon_device *oct = ioq_vector->oct_dev;
500 struct lio_droq *droq = oct->droq[ioq_vector->droq_index];
501 uint64_t ret;
502
503 ret = oct->fn_list.msix_interrupt_handler(ioq_vector);
504
505 if ((ret & LIO_MSIX_PO_INT) || (ret & LIO_MSIX_PI_INT)) {
506 struct lio_instr_queue *iq = oct->instr_queue[droq->q_no];
507 int reschedule, tx_done = 1;
508
509 reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget);
510
511 if (atomic_load_acq_int(&iq->instr_pending))
512 tx_done = lio_flush_iq(oct, iq, oct->tx_budget);
513
514 if ((oct->props.ifp != NULL) && (iq->br != NULL)) {
515 if (mtx_trylock(&iq->enq_lock)) {
516 if (!drbr_empty(oct->props.ifp, iq->br))
517 lio_mq_start_locked(oct->props.ifp,
518 iq);
519 mtx_unlock(&iq->enq_lock);
520 }
521 }
522
523 if (reschedule || !tx_done)
524 taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task);
525 else
526 lio_enable_irq(droq, iq);
527 }
528 }
529
530 static void
lio_intr_handler(void * dev)531 lio_intr_handler(void *dev)
532 {
533 struct octeon_device *oct = (struct octeon_device *)dev;
534
535 /* Disable our interrupts for the duration of ISR */
536 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
537
538 oct->fn_list.process_interrupt_regs(oct);
539
540 lio_schedule_droq_pkt_handlers(oct);
541
542 /* Re-enable our interrupts */
543 if (!(atomic_load_acq_int(&oct->status) == LIO_DEV_IN_RESET))
544 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR);
545 }
546
547 int
lio_setup_interrupt(struct octeon_device * oct,uint32_t num_ioqs)548 lio_setup_interrupt(struct octeon_device *oct, uint32_t num_ioqs)
549 {
550 device_t device;
551 struct lio_ioq_vector *ioq_vector;
552 int cpu_id, err, i;
553 int num_alloc_ioq_vectors;
554 int num_ioq_vectors;
555 int res_id;
556
557 if (!oct->msix_on)
558 return (1);
559
560 ioq_vector = oct->ioq_vector;
561
562 #ifdef RSS
563 if (oct->sriov_info.num_pf_rings != rss_getnumbuckets()) {
564 lio_dev_info(oct, "IOQ vectors (%d) are not equal number of RSS buckets (%d)\n",
565 oct->sriov_info.num_pf_rings, rss_getnumbuckets());
566 }
567 #endif
568
569 device = oct->device;
570
571 oct->num_msix_irqs = num_ioqs;
572 /* one non ioq interrupt for handling sli_mac_pf_int_sum */
573 oct->num_msix_irqs += 1;
574 num_alloc_ioq_vectors = oct->num_msix_irqs;
575
576 if (pci_alloc_msix(device, &num_alloc_ioq_vectors) ||
577 (num_alloc_ioq_vectors != oct->num_msix_irqs))
578 goto err;
579
580 num_ioq_vectors = oct->num_msix_irqs;
581
582 /* For PF, there is one non-ioq interrupt handler */
583 for (i = 0; i < num_ioq_vectors - 1; i++, ioq_vector++) {
584 res_id = i + 1;
585
586 ioq_vector->msix_res =
587 bus_alloc_resource_any(device, SYS_RES_IRQ, &res_id,
588 RF_SHAREABLE | RF_ACTIVE);
589 if (ioq_vector->msix_res == NULL) {
590 lio_dev_err(oct,
591 "Unable to allocate bus res msix[%d]\n", i);
592 goto err_1;
593 }
594
595 err = bus_setup_intr(device, ioq_vector->msix_res,
596 INTR_TYPE_NET | INTR_MPSAFE, NULL,
597 lio_msix_intr_handler, ioq_vector,
598 &ioq_vector->tag);
599 if (err) {
600 bus_release_resource(device, SYS_RES_IRQ, res_id,
601 ioq_vector->msix_res);
602 ioq_vector->msix_res = NULL;
603 lio_dev_err(oct, "Failed to register intr handler");
604 goto err_1;
605 }
606
607 bus_describe_intr(device, ioq_vector->msix_res, ioq_vector->tag,
608 "rxtx%u", i);
609 ioq_vector->vector = res_id;
610
611 #ifdef RSS
612 cpu_id = rss_getcpu(i % rss_getnumbuckets());
613 #else
614 cpu_id = i % mp_ncpus;
615 #endif
616 CPU_SETOF(cpu_id, &ioq_vector->affinity_mask);
617
618 /* Setting the IRQ affinity. */
619 err = bus_bind_intr(device, ioq_vector->msix_res, cpu_id);
620 if (err)
621 lio_dev_err(oct, "bus bind interrupt fail");
622 #ifdef RSS
623 lio_dev_dbg(oct, "Bound RSS bucket %d to CPU %d\n", i, cpu_id);
624 #else
625 lio_dev_dbg(oct, "Bound Queue %d to CPU %d\n", i, cpu_id);
626 #endif
627 }
628
629 lio_dev_dbg(oct, "MSI-X enabled\n");
630
631 res_id = num_ioq_vectors;
632 oct->msix_res = bus_alloc_resource_any(device, SYS_RES_IRQ, &res_id,
633 RF_SHAREABLE | RF_ACTIVE);
634 if (oct->msix_res == NULL) {
635 lio_dev_err(oct, "Unable to allocate bus res msix for non-ioq interrupt\n");
636 goto err_1;
637 }
638
639 err = bus_setup_intr(device, oct->msix_res, INTR_TYPE_NET | INTR_MPSAFE,
640 NULL, lio_intr_handler, oct, &oct->tag);
641 if (err) {
642 bus_release_resource(device, SYS_RES_IRQ, res_id,
643 oct->msix_res);
644 oct->msix_res = NULL;
645 lio_dev_err(oct, "Failed to register intr handler");
646 goto err_1;
647 }
648
649 bus_describe_intr(device, oct->msix_res, oct->tag, "aux");
650 oct->aux_vector = res_id;
651
652 return (0);
653 err_1:
654 if (oct->tag != NULL) {
655 bus_teardown_intr(device, oct->msix_res, oct->tag);
656 oct->tag = NULL;
657 }
658
659 while (i) {
660 i--;
661 ioq_vector--;
662
663 if (ioq_vector->tag != NULL) {
664 bus_teardown_intr(device, ioq_vector->msix_res,
665 ioq_vector->tag);
666 ioq_vector->tag = NULL;
667 }
668
669 if (ioq_vector->msix_res != NULL) {
670 bus_release_resource(device, SYS_RES_IRQ,
671 ioq_vector->vector,
672 ioq_vector->msix_res);
673 ioq_vector->msix_res = NULL;
674 }
675 }
676
677 if (oct->msix_res != NULL) {
678 bus_release_resource(device, SYS_RES_IRQ, oct->aux_vector,
679 oct->msix_res);
680 oct->msix_res = NULL;
681 }
682 err:
683 pci_release_msi(device);
684 lio_dev_err(oct, "MSI-X disabled\n");
685 return (1);
686 }
687