Lines Matching +full:cm +full:- +full:poll +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
6 * irdma_arp_table -manage arp table
25 spin_lock_irqsave(&rf->arp_lock, flags); in irdma_arp_table()
26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { in irdma_arp_table()
27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) in irdma_arp_table()
33 if (arp_index != rf->arp_table_size) { in irdma_arp_table()
34 arp_index = -1; in irdma_arp_table()
39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, in irdma_arp_table()
40 (u32 *)&arp_index, &rf->next_arp_index)) { in irdma_arp_table()
41 arp_index = -1; in irdma_arp_table()
45 memcpy(rf->arp_table[arp_index].ip_addr, ip, in irdma_arp_table()
46 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); in irdma_arp_table()
50 if (arp_index == rf->arp_table_size) in irdma_arp_table()
51 arp_index = -1; in irdma_arp_table()
54 if (arp_index == rf->arp_table_size) { in irdma_arp_table()
55 arp_index = -1; in irdma_arp_table()
59 memset(rf->arp_table[arp_index].ip_addr, 0, in irdma_arp_table()
60 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
61 eth_zero_addr(rf->arp_table[arp_index].mac_addr); in irdma_arp_table()
62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); in irdma_arp_table()
65 arp_index = -1; in irdma_arp_table()
69 spin_unlock_irqrestore(&rf->arp_lock, flags); in irdma_arp_table()
74 * irdma_add_arp - add a new arp entry if needed
86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) in irdma_add_arp()
89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, in irdma_add_arp()
99 * wr32 - write 32 bits to hw register
106 writel(val, hw->hw_addr + reg); in wr32()
110 * rd32 - read a 32 bit hw register
118 return readl(hw->hw_addr + reg); in rd32()
122 * rd64 - read a 64 bit hw register
130 return readq(hw->hw_addr + reg); in rd64()
144 * irdma_inetaddr_event - system notifier for ipv4 addr events
153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev; in irdma_inetaddr_event()
167 local_ipaddr = ntohl(ifa->ifa_address); in irdma_inetaddr_event()
168 ibdev_dbg(&iwdev->ibdev, in irdma_inetaddr_event()
170 event, &local_ipaddr, real_dev->dev_addr); in irdma_inetaddr_event()
173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, in irdma_inetaddr_event()
176 irdma_gid_change_event(&iwdev->ibdev); in irdma_inetaddr_event()
180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr); in irdma_inetaddr_event()
182 irdma_gid_change_event(&iwdev->ibdev); in irdma_inetaddr_event()
194 * irdma_inet6addr_event - system notifier for ipv6 addr events
203 struct net_device *real_dev, *netdev = ifa->idev->dev; in irdma_inet6addr_event()
217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32); in irdma_inet6addr_event()
218 ibdev_dbg(&iwdev->ibdev, in irdma_inet6addr_event()
220 event, local_ipaddr6, real_dev->dev_addr); in irdma_inet6addr_event()
223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr, in irdma_inet6addr_event()
226 irdma_gid_change_event(&iwdev->ibdev); in irdma_inet6addr_event()
230 irdma_add_arp(iwdev->rf, local_ipaddr6, false, in irdma_inet6addr_event()
231 real_dev->dev_addr); in irdma_inet6addr_event()
233 irdma_gid_change_event(&iwdev->ibdev); in irdma_inet6addr_event()
245 * irdma_net_event - system notifier for net events
254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev; in irdma_net_event()
271 p = (__be32 *)neigh->primary_key; in irdma_net_event()
272 if (neigh->tbl->family == AF_INET6) { in irdma_net_event()
279 ibdev_dbg(&iwdev->ibdev, in irdma_net_event()
281 iwdev->netdev, neigh->nud_state, local_ipaddr, in irdma_net_event()
282 neigh->ha); in irdma_net_event()
284 if (neigh->nud_state & NUD_VALID) in irdma_net_event()
285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha); in irdma_net_event()
288 irdma_manage_arp_cache(iwdev->rf, neigh->ha, in irdma_net_event()
301 * irdma_netdevice_event - system notifier for netdev events
318 iwdev->iw_status = 1; in irdma_netdevice_event()
321 iwdev->iw_status = 0; in irdma_netdevice_event()
332 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
345 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) || in irdma_add_ipv6_addr()
346 ip_dev == iwdev->netdev) && in irdma_add_ipv6_addr()
347 (READ_ONCE(ip_dev->flags) & IFF_UP)) { in irdma_add_ipv6_addr()
350 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n"); in irdma_add_ipv6_addr()
353 list_for_each_entry_safe (ifp, tmp, &idev->addr_list, in irdma_add_ipv6_addr()
355 ibdev_dbg(&iwdev->ibdev, in irdma_add_ipv6_addr()
357 &ifp->addr, in irdma_add_ipv6_addr()
359 ip_dev->dev_addr); in irdma_add_ipv6_addr()
362 ifp->addr.in6_u.u6_addr32); in irdma_add_ipv6_addr()
363 irdma_manage_arp_cache(iwdev->rf, in irdma_add_ipv6_addr()
364 ip_dev->dev_addr, in irdma_add_ipv6_addr()
374 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
386 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) || in irdma_add_ipv4_addr()
387 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) { in irdma_add_ipv4_addr()
395 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n", in irdma_add_ipv4_addr()
396 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev), in irdma_add_ipv4_addr()
397 dev->dev_addr); in irdma_add_ipv4_addr()
399 ip_addr = ntohl(ifa->ifa_address); in irdma_add_ipv4_addr()
400 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr, in irdma_add_ipv4_addr()
410 * irdma_add_ip - add ip addresses
422 * irdma_alloc_and_get_cqp_request - get cqp struct
424 * @wait: cqp to be used in wait mode
432 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
433 if (!list_empty(&cqp->cqp_avail_reqs)) { in irdma_alloc_and_get_cqp_request()
434 cqp_request = list_first_entry(&cqp->cqp_avail_reqs, in irdma_alloc_and_get_cqp_request()
436 list_del_init(&cqp_request->list); in irdma_alloc_and_get_cqp_request()
438 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
442 cqp_request->dynamic = true; in irdma_alloc_and_get_cqp_request()
444 init_waitqueue_head(&cqp_request->waitq); in irdma_alloc_and_get_cqp_request()
448 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory"); in irdma_alloc_and_get_cqp_request()
452 cqp_request->waiting = wait; in irdma_alloc_and_get_cqp_request()
453 refcount_set(&cqp_request->refcnt, 1); in irdma_alloc_and_get_cqp_request()
454 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); in irdma_alloc_and_get_cqp_request()
460 * irdma_get_cqp_request - increase refcount for cqp_request
465 refcount_inc(&cqp_request->refcnt); in irdma_get_cqp_request()
469 * irdma_free_cqp_request - free cqp request
478 if (cqp_request->dynamic) { in irdma_free_cqp_request()
481 WRITE_ONCE(cqp_request->request_done, false); in irdma_free_cqp_request()
482 cqp_request->callback_fcn = NULL; in irdma_free_cqp_request()
483 cqp_request->waiting = false; in irdma_free_cqp_request()
485 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_free_cqp_request()
486 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); in irdma_free_cqp_request()
487 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_free_cqp_request()
489 wake_up(&cqp->remove_wq); in irdma_free_cqp_request()
493 * irdma_put_cqp_request - dec ref count and free if 0
500 if (refcount_dec_and_test(&cqp_request->refcnt)) in irdma_put_cqp_request()
505 * irdma_free_pending_cqp_request -free pending cqp request objs
513 if (cqp_request->waiting) { in irdma_free_pending_cqp_request()
514 cqp_request->compl_info.error = true; in irdma_free_pending_cqp_request()
515 WRITE_ONCE(cqp_request->request_done, true); in irdma_free_pending_cqp_request()
516 wake_up(&cqp_request->waitq); in irdma_free_pending_cqp_request()
518 wait_event_timeout(cqp->remove_wq, in irdma_free_pending_cqp_request()
519 refcount_read(&cqp_request->refcnt) == 1, 1000); in irdma_free_pending_cqp_request()
524 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
530 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_cleanup_pending_cqp_op()
531 struct irdma_cqp *cqp = &rf->cqp; in irdma_cleanup_pending_cqp_op()
536 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
537 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
540 cqp->scratch_array[wqe_idx]; in irdma_cleanup_pending_cqp_op()
543 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
546 while (!list_empty(&dev->cqp_cmd_head)) { in irdma_cleanup_pending_cqp_op()
556 * irdma_wait_event - wait for completion
567 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); in irdma_wait_event()
569 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); in irdma_wait_event()
570 if (wait_event_timeout(cqp_request->waitq, in irdma_wait_event()
571 READ_ONCE(cqp_request->request_done), in irdma_wait_event()
575 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); in irdma_wait_event()
580 if (!rf->reset) { in irdma_wait_event()
581 rf->reset = true; in irdma_wait_event()
582 rf->gen_ops.request_reset(rf); in irdma_wait_event()
584 return -ETIMEDOUT; in irdma_wait_event()
587 cqp_error = cqp_request->compl_info.error; in irdma_wait_event()
589 err_code = -EIO; in irdma_wait_event()
590 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { in irdma_wait_event()
591 if (cqp_request->compl_info.min_err_code == 0x8002) in irdma_wait_event()
592 err_code = -EBUSY; in irdma_wait_event()
593 else if (cqp_request->compl_info.min_err_code == 0x8029) { in irdma_wait_event()
594 if (!rf->reset) { in irdma_wait_event()
595 rf->reset = true; in irdma_wait_event()
596 rf->gen_ops.request_reset(rf); in irdma_wait_event()
620 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
645 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
663 * irdma_cqp_crit_err - check if CQP error is critical
689 * irdma_handle_cqp_op - process cqp command
696 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_handle_cqp_op()
697 struct cqp_cmds_info *info = &cqp_request->info; in irdma_handle_cqp_op()
701 if (rf->reset) in irdma_handle_cqp_op()
702 return -EBUSY; in irdma_handle_cqp_op()
709 if (cqp_request->waiting) { in irdma_handle_cqp_op()
719 if (irdma_cqp_crit_err(dev, info->cqp_cmd, in irdma_handle_cqp_op()
720 cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
721 cqp_request->compl_info.min_err_code)) in irdma_handle_cqp_op()
722 ibdev_err(&rf->iwdev->ibdev, in irdma_handle_cqp_op()
724 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, in irdma_handle_cqp_op()
725 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
726 cqp_request->compl_info.min_err_code); in irdma_handle_cqp_op()
729 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_handle_cqp_op()
738 refcount_inc(&iwqp->refcnt); in irdma_qp_add_ref()
744 struct irdma_device *iwdev = iwqp->iwdev; in irdma_qp_rem_ref()
748 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
749 if (!refcount_dec_and_test(&iwqp->refcnt)) { in irdma_qp_rem_ref()
750 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
754 qp_num = iwqp->ibqp.qp_num; in irdma_qp_rem_ref()
755 iwdev->rf->qp_table[qp_num] = NULL; in irdma_qp_rem_ref()
756 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
757 complete(&iwqp->free_qp); in irdma_qp_rem_ref()
764 refcount_inc(&iwcq->refcnt); in irdma_cq_add_ref()
769 struct ib_device *ibdev = ibcq->device; in irdma_cq_rem_ref()
774 spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
775 if (!refcount_dec_and_test(&iwcq->refcnt)) { in irdma_cq_rem_ref()
776 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
780 iwdev->rf->cq_table[iwcq->cq_num] = NULL; in irdma_cq_rem_ref()
781 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags); in irdma_cq_rem_ref()
782 complete(&iwcq->free_cq); in irdma_cq_rem_ref()
787 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; in to_ibdev()
791 * irdma_get_qp - get qp address
799 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) in irdma_get_qp()
802 return &iwdev->rf->qp_table[qpn]->ibqp; in irdma_get_qp()
806 * irdma_remove_cqp_head - return head entry and remove
812 struct list_head *list = &dev->cqp_cmd_head; in irdma_remove_cqp_head()
817 entry = list->next; in irdma_remove_cqp_head()
824 * irdma_cqp_sds_cmd - create cqp command for sd
837 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_sds_cmd()
839 return -ENOMEM; in irdma_cqp_sds_cmd()
841 cqp_info = &cqp_request->info; in irdma_cqp_sds_cmd()
842 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, in irdma_cqp_sds_cmd()
843 sizeof(cqp_info->in.u.update_pe_sds.info)); in irdma_cqp_sds_cmd()
844 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; in irdma_cqp_sds_cmd()
845 cqp_info->post_sq = 1; in irdma_cqp_sds_cmd()
846 cqp_info->in.u.update_pe_sds.dev = dev; in irdma_cqp_sds_cmd()
847 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; in irdma_cqp_sds_cmd()
850 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_sds_cmd()
856 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
862 struct irdma_sc_dev *dev = qp->dev; in irdma_cqp_qp_suspend_resume()
864 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_cqp_qp_suspend_resume()
869 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_cqp_qp_suspend_resume()
871 return -ENOMEM; in irdma_cqp_qp_suspend_resume()
873 cqp_info = &cqp_request->info; in irdma_cqp_qp_suspend_resume()
874 cqp_info->cqp_cmd = op; in irdma_cqp_qp_suspend_resume()
875 cqp_info->in.u.suspend_resume.cqp = cqp; in irdma_cqp_qp_suspend_resume()
876 cqp_info->in.u.suspend_resume.qp = qp; in irdma_cqp_qp_suspend_resume()
877 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_suspend_resume()
880 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_suspend_resume()
886 * irdma_term_modify_qp - modify qp for term message
897 iwqp = qp->qp_uk.back_qp; in irdma_term_modify_qp()
902 * irdma_terminate_done - after terminate is completed
913 iwqp = qp->qp_uk.back_qp; in irdma_terminate_done()
914 spin_lock_irqsave(&iwqp->lock, flags); in irdma_terminate_done()
915 if (iwqp->hte_added) { in irdma_terminate_done()
916 iwqp->hte_added = 0; in irdma_terminate_done()
919 first_time = !(qp->term_flags & IRDMA_TERM_DONE); in irdma_terminate_done()
920 qp->term_flags |= IRDMA_TERM_DONE; in irdma_terminate_done()
921 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_terminate_done()
934 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_terminate_timeout()
937 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_timeout()
941 * irdma_terminate_start_timer - start terminate timeout
948 iwqp = qp->qp_uk.back_qp; in irdma_terminate_start_timer()
949 irdma_qp_add_ref(&iwqp->ibqp); in irdma_terminate_start_timer()
950 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); in irdma_terminate_start_timer()
951 iwqp->terminate_timer.expires = jiffies + HZ; in irdma_terminate_start_timer()
953 add_timer(&iwqp->terminate_timer); in irdma_terminate_start_timer()
957 * irdma_terminate_del_timer - delete terminate timeout
965 iwqp = qp->qp_uk.back_qp; in irdma_terminate_del_timer()
966 ret = timer_delete(&iwqp->terminate_timer); in irdma_terminate_del_timer()
968 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_del_timer()
972 * irdma_cqp_cq_create_cmd - create a cq for the cqp
979 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_cq_create_cmd()
986 return -ENOMEM; in irdma_cqp_cq_create_cmd()
988 cqp_info = &cqp_request->info; in irdma_cqp_cq_create_cmd()
989 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; in irdma_cqp_cq_create_cmd()
990 cqp_info->post_sq = 1; in irdma_cqp_cq_create_cmd()
991 cqp_info->in.u.cq_create.cq = cq; in irdma_cqp_cq_create_cmd()
992 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_cq_create_cmd()
1001 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1008 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_create_cmd()
1016 return -ENOMEM; in irdma_cqp_qp_create_cmd()
1018 cqp_info = &cqp_request->info; in irdma_cqp_qp_create_cmd()
1019 qp_info = &cqp_request->info.in.u.qp_create.info; in irdma_cqp_qp_create_cmd()
1021 qp_info->cq_num_valid = true; in irdma_cqp_qp_create_cmd()
1022 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; in irdma_cqp_qp_create_cmd()
1023 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; in irdma_cqp_qp_create_cmd()
1024 cqp_info->post_sq = 1; in irdma_cqp_qp_create_cmd()
1025 cqp_info->in.u.qp_create.qp = qp; in irdma_cqp_qp_create_cmd()
1026 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_create_cmd()
1035 * irdma_dealloc_push_page - free a push page for qp
1046 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) in irdma_dealloc_push_page()
1049 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_dealloc_push_page()
1053 cqp_info = &cqp_request->info; in irdma_dealloc_push_page()
1054 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; in irdma_dealloc_push_page()
1055 cqp_info->post_sq = 1; in irdma_dealloc_push_page()
1056 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; in irdma_dealloc_push_page()
1057 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; in irdma_dealloc_push_page()
1058 cqp_info->in.u.manage_push_page.info.free_page = 1; in irdma_dealloc_push_page()
1059 cqp_info->in.u.manage_push_page.info.push_page_type = 0; in irdma_dealloc_push_page()
1060 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; in irdma_dealloc_push_page()
1061 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; in irdma_dealloc_push_page()
1064 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; in irdma_dealloc_push_page()
1065 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_dealloc_push_page()
1069 * irdma_free_qp_rsrc - free up memory resources for qp
1074 struct irdma_device *iwdev = iwqp->iwdev; in irdma_free_qp_rsrc()
1075 struct irdma_pci_f *rf = iwdev->rf; in irdma_free_qp_rsrc()
1076 u32 qp_num = iwqp->ibqp.qp_num; in irdma_free_qp_rsrc()
1078 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); in irdma_free_qp_rsrc()
1079 irdma_dealloc_push_page(rf, &iwqp->sc_qp); in irdma_free_qp_rsrc()
1080 if (iwqp->sc_qp.vsi) { in irdma_free_qp_rsrc()
1081 irdma_qp_rem_qos(&iwqp->sc_qp); in irdma_free_qp_rsrc()
1082 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, in irdma_free_qp_rsrc()
1083 iwqp->sc_qp.user_pri); in irdma_free_qp_rsrc()
1087 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); in irdma_free_qp_rsrc()
1088 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size, in irdma_free_qp_rsrc()
1089 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa); in irdma_free_qp_rsrc()
1090 iwqp->q2_ctx_mem.va = NULL; in irdma_free_qp_rsrc()
1091 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size, in irdma_free_qp_rsrc()
1092 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa); in irdma_free_qp_rsrc()
1093 iwqp->kqp.dma_mem.va = NULL; in irdma_free_qp_rsrc()
1094 kfree(iwqp->kqp.sq_wrid_mem); in irdma_free_qp_rsrc()
1095 kfree(iwqp->kqp.rq_wrid_mem); in irdma_free_qp_rsrc()
1099 * irdma_cq_wq_destroy - send cq destroy cqp
1108 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cq_wq_destroy()
1112 cqp_info = &cqp_request->info; in irdma_cq_wq_destroy()
1113 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; in irdma_cq_wq_destroy()
1114 cqp_info->post_sq = 1; in irdma_cq_wq_destroy()
1115 cqp_info->in.u.cq_destroy.cq = cq; in irdma_cq_wq_destroy()
1116 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; in irdma_cq_wq_destroy()
1119 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cq_wq_destroy()
1123 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1131 cqp_info = &cqp_request->info; in irdma_hw_modify_qp_callback()
1132 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; in irdma_hw_modify_qp_callback()
1133 atomic_dec(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp_callback()
1134 wake_up(&iwqp->mod_qp_waitq); in irdma_hw_modify_qp_callback()
1138 * irdma_hw_modify_qp - setup cqp for modify qp
1148 struct irdma_pci_f *rf = iwdev->rf; in irdma_hw_modify_qp()
1153 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_hw_modify_qp()
1155 return -ENOMEM; in irdma_hw_modify_qp()
1158 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; in irdma_hw_modify_qp()
1159 atomic_inc(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp()
1161 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1162 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1164 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1165 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1166 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1167 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1169 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1171 if (rdma_protocol_roce(&iwdev->ibdev, 1)) in irdma_hw_modify_qp()
1174 switch (m_info->next_iwarp_state) { in irdma_hw_modify_qp()
1181 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) in irdma_hw_modify_qp()
1182 irdma_send_reset(iwqp->cm_node); in irdma_hw_modify_qp()
1184 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; in irdma_hw_modify_qp()
1188 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); in irdma_hw_modify_qp()
1190 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, in irdma_hw_modify_qp()
1193 return -ENOMEM; in irdma_hw_modify_qp()
1195 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1196 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1198 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1199 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1200 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1201 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1202 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; in irdma_hw_modify_qp()
1203 m_info->reset_tcp_conn = true; in irdma_hw_modify_qp()
1205 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1218 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1230 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1237 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_destroy_cmd()
1244 return -ENOMEM; in irdma_cqp_qp_destroy_cmd()
1246 cqp_info = &cqp_request->info; in irdma_cqp_qp_destroy_cmd()
1248 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; in irdma_cqp_qp_destroy_cmd()
1249 cqp_info->post_sq = 1; in irdma_cqp_qp_destroy_cmd()
1250 cqp_info->in.u.qp_destroy.qp = qp; in irdma_cqp_qp_destroy_cmd()
1251 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_destroy_cmd()
1252 cqp_info->in.u.qp_destroy.remove_hash_idx = true; in irdma_cqp_qp_destroy_cmd()
1255 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_destroy_cmd()
1261 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1270 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n"); in irdma_ieq_mpa_crc_ae()
1277 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1285 return -EINVAL; in irdma_ieq_check_mpacrc()
1291 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1300 struct irdma_device *iwdev = buf->vsi->back_vsi; in irdma_ieq_get_qp()
1305 struct iphdr *iph = (struct iphdr *)buf->iph; in irdma_ieq_get_qp()
1306 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_ieq_get_qp()
1308 if (iph->version == 4) { in irdma_ieq_get_qp()
1309 loc_addr[0] = ntohl(iph->daddr); in irdma_ieq_get_qp()
1310 rem_addr[0] = ntohl(iph->saddr); in irdma_ieq_get_qp()
1312 ip6h = (struct ipv6hdr *)buf->iph; in irdma_ieq_get_qp()
1313 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32); in irdma_ieq_get_qp()
1314 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32); in irdma_ieq_get_qp()
1316 loc_port = ntohs(tcph->dest); in irdma_ieq_get_qp()
1317 rem_port = ntohs(tcph->source); in irdma_ieq_get_qp()
1318 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, in irdma_ieq_get_qp()
1319 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); in irdma_ieq_get_qp()
1323 iwqp = cm_node->iwqp; in irdma_ieq_get_qp()
1326 return &iwqp->sc_qp; in irdma_ieq_get_qp()
1330 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1335 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; in irdma_send_ieq_ack()
1336 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; in irdma_send_ieq_ack()
1337 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_send_ieq_ack()
1339 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; in irdma_send_ieq_ack()
1340 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); in irdma_send_ieq_ack()
1346 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1353 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; in irdma_puda_ieq_get_ah_info()
1358 ah_info->do_lpbk = true; in irdma_puda_ieq_get_ah_info()
1359 ah_info->vlan_tag = buf->vlan_id; in irdma_puda_ieq_get_ah_info()
1360 ah_info->insert_vlan_tag = buf->vlan_valid; in irdma_puda_ieq_get_ah_info()
1361 ah_info->ipv4_valid = buf->ipv4; in irdma_puda_ieq_get_ah_info()
1362 ah_info->vsi = qp->vsi; in irdma_puda_ieq_get_ah_info()
1364 if (buf->smac_valid) in irdma_puda_ieq_get_ah_info()
1365 ether_addr_copy(ah_info->mac_addr, buf->smac); in irdma_puda_ieq_get_ah_info()
1367 if (buf->ipv4) { in irdma_puda_ieq_get_ah_info()
1368 ah_info->ipv4_valid = true; in irdma_puda_ieq_get_ah_info()
1369 iph = (struct iphdr *)buf->iph; in irdma_puda_ieq_get_ah_info()
1370 ah_info->hop_ttl = iph->ttl; in irdma_puda_ieq_get_ah_info()
1371 ah_info->tc_tos = iph->tos; in irdma_puda_ieq_get_ah_info()
1372 ah_info->dest_ip_addr[0] = ntohl(iph->daddr); in irdma_puda_ieq_get_ah_info()
1373 ah_info->src_ip_addr[0] = ntohl(iph->saddr); in irdma_puda_ieq_get_ah_info()
1375 ip6h = (struct ipv6hdr *)buf->iph; in irdma_puda_ieq_get_ah_info()
1376 ah_info->hop_ttl = ip6h->hop_limit; in irdma_puda_ieq_get_ah_info()
1377 ah_info->tc_tos = ip6h->priority; in irdma_puda_ieq_get_ah_info()
1378 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1379 ip6h->daddr.in6_u.u6_addr32); in irdma_puda_ieq_get_ah_info()
1380 irdma_copy_ip_ntohl(ah_info->src_ip_addr, in irdma_puda_ieq_get_ah_info()
1381 ip6h->saddr.in6_u.u6_addr32); in irdma_puda_ieq_get_ah_info()
1384 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), in irdma_puda_ieq_get_ah_info()
1385 ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1386 ah_info->ipv4_valid, in irdma_puda_ieq_get_ah_info()
1391 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1403 u8 *addr = buf->mem.va; in irdma_gen1_ieq_update_tcpip_info()
1405 iphlen = (buf->ipv4) ? 20 : 40; in irdma_gen1_ieq_update_tcpip_info()
1406 iph = (struct iphdr *)(addr + buf->maclen); in irdma_gen1_ieq_update_tcpip_info()
1407 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); in irdma_gen1_ieq_update_tcpip_info()
1408 pktsize = len + buf->tcphlen + iphlen; in irdma_gen1_ieq_update_tcpip_info()
1409 iph->tot_len = htons(pktsize); in irdma_gen1_ieq_update_tcpip_info()
1410 tcph->seq = htonl(seqnum); in irdma_gen1_ieq_update_tcpip_info()
1414 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1425 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_ieq_update_tcpip_info()
1428 addr = buf->mem.va; in irdma_ieq_update_tcpip_info()
1430 tcph->seq = htonl(seqnum); in irdma_ieq_update_tcpip_info()
1434 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1447 u8 *mem = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1448 struct ethhdr *ethh = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1450 if (ethh->h_proto == htons(0x8100)) { in irdma_gen1_puda_get_tcpip_info()
1451 info->vlan_valid = true; in irdma_gen1_puda_get_tcpip_info()
1452 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) & in irdma_gen1_puda_get_tcpip_info()
1456 buf->maclen = (info->vlan_valid) ? 18 : 14; in irdma_gen1_puda_get_tcpip_info()
1457 iphlen = (info->l3proto) ? 40 : 20; in irdma_gen1_puda_get_tcpip_info()
1458 buf->ipv4 = (info->l3proto) ? false : true; in irdma_gen1_puda_get_tcpip_info()
1459 buf->iph = mem + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1460 iph = (struct iphdr *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1461 buf->tcph = buf->iph + iphlen; in irdma_gen1_puda_get_tcpip_info()
1462 tcph = (struct tcphdr *)buf->tcph; in irdma_gen1_puda_get_tcpip_info()
1464 if (buf->ipv4) { in irdma_gen1_puda_get_tcpip_info()
1465 pkt_len = ntohs(iph->tot_len); in irdma_gen1_puda_get_tcpip_info()
1467 ip6h = (struct ipv6hdr *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1468 pkt_len = ntohs(ip6h->payload_len) + iphlen; in irdma_gen1_puda_get_tcpip_info()
1471 buf->totallen = pkt_len + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1473 if (info->payload_len < buf->totallen) { in irdma_gen1_puda_get_tcpip_info()
1474 ibdev_dbg(to_ibdev(buf->vsi->dev), in irdma_gen1_puda_get_tcpip_info()
1476 info->payload_len, buf->totallen); in irdma_gen1_puda_get_tcpip_info()
1477 return -EINVAL; in irdma_gen1_puda_get_tcpip_info()
1480 buf->tcphlen = tcph->doff << 2; in irdma_gen1_puda_get_tcpip_info()
1481 buf->datalen = pkt_len - iphlen - buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1482 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_gen1_puda_get_tcpip_info()
1483 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1484 buf->seqnum = ntohl(tcph->seq); in irdma_gen1_puda_get_tcpip_info()
1490 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1501 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_puda_get_tcpip_info()
1504 mem = buf->mem.va; in irdma_puda_get_tcpip_info()
1505 buf->vlan_valid = info->vlan_valid; in irdma_puda_get_tcpip_info()
1506 if (info->vlan_valid) in irdma_puda_get_tcpip_info()
1507 buf->vlan_id = info->vlan; in irdma_puda_get_tcpip_info()
1509 buf->ipv4 = info->ipv4; in irdma_puda_get_tcpip_info()
1510 if (buf->ipv4) in irdma_puda_get_tcpip_info()
1511 buf->iph = mem + IRDMA_IPV4_PAD; in irdma_puda_get_tcpip_info()
1513 buf->iph = mem; in irdma_puda_get_tcpip_info()
1515 buf->tcph = mem + IRDMA_TCP_OFFSET; in irdma_puda_get_tcpip_info()
1516 tcph = (struct tcphdr *)buf->tcph; in irdma_puda_get_tcpip_info()
1517 pkt_len = info->payload_len; in irdma_puda_get_tcpip_info()
1518 buf->totallen = pkt_len; in irdma_puda_get_tcpip_info()
1519 buf->tcphlen = tcph->doff << 2; in irdma_puda_get_tcpip_info()
1520 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; in irdma_puda_get_tcpip_info()
1521 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_puda_get_tcpip_info()
1522 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; in irdma_puda_get_tcpip_info()
1523 buf->seqnum = ntohl(tcph->seq); in irdma_puda_get_tcpip_info()
1525 if (info->smac_valid) { in irdma_puda_get_tcpip_info()
1526 ether_addr_copy(buf->smac, info->smac); in irdma_puda_get_tcpip_info()
1527 buf->smac_valid = true; in irdma_puda_get_tcpip_info()
1534 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1541 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; in irdma_hw_stats_timeout()
1543 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) in irdma_hw_stats_timeout()
1544 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); in irdma_hw_stats_timeout()
1546 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat); in irdma_hw_stats_timeout()
1548 mod_timer(&pf_devstat->stats_timer, in irdma_hw_stats_timeout()
1553 * irdma_hw_stats_start_timer - Start periodic stats timer
1558 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_start_timer()
1560 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); in irdma_hw_stats_start_timer()
1561 mod_timer(&devstat->stats_timer, in irdma_hw_stats_start_timer()
1566 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1571 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_stop_timer()
1573 timer_delete_sync(&devstat->stats_timer); in irdma_hw_stats_stop_timer()
1577 * irdma_process_stats - Checking for wrap and update stats
1582 sc_vsi_update_stats(pestat->vsi); in irdma_process_stats()
1586 * irdma_cqp_gather_stats_gen1 - Gather stats
1594 pestat->gather_info.gather_stats_va; in irdma_cqp_gather_stats_gen1()
1595 const struct irdma_hw_stat_map *map = dev->hw_stats_map; in irdma_cqp_gather_stats_gen1()
1596 u16 max_stats_idx = dev->hw_attrs.max_stat_idx; in irdma_cqp_gather_stats_gen1()
1602 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ? in irdma_cqp_gather_stats_gen1()
1603 pestat->gather_info.stats_inst_index : in irdma_cqp_gather_stats_gen1()
1604 pestat->hw->hmc.hmc_fn_id; in irdma_cqp_gather_stats_gen1()
1610 new_val = rd32(dev->hw, in irdma_cqp_gather_stats_gen1()
1611 dev->hw_stats_regs[i] + stats_inst_offset_32); in irdma_cqp_gather_stats_gen1()
1613 new_val = rd64(dev->hw, in irdma_cqp_gather_stats_gen1()
1614 dev->hw_stats_regs[i] + stats_inst_offset_64); in irdma_cqp_gather_stats_gen1()
1615 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val; in irdma_cqp_gather_stats_gen1()
1622 * irdma_process_cqp_stats - Checking for wrap and update stats
1627 struct irdma_vsi_pestat *pestat = cqp_request->param; in irdma_process_cqp_stats()
1633 * irdma_cqp_gather_stats_cmd - Gather stats
1643 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_gather_stats_cmd()
1650 return -ENOMEM; in irdma_cqp_gather_stats_cmd()
1652 cqp_info = &cqp_request->info; in irdma_cqp_gather_stats_cmd()
1654 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; in irdma_cqp_gather_stats_cmd()
1655 cqp_info->post_sq = 1; in irdma_cqp_gather_stats_cmd()
1656 cqp_info->in.u.stats_gather.info = pestat->gather_info; in irdma_cqp_gather_stats_cmd()
1657 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; in irdma_cqp_gather_stats_cmd()
1658 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; in irdma_cqp_gather_stats_cmd()
1659 cqp_request->param = pestat; in irdma_cqp_gather_stats_cmd()
1661 cqp_request->callback_fcn = irdma_process_cqp_stats; in irdma_cqp_gather_stats_cmd()
1665 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_gather_stats_cmd()
1671 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1679 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); in irdma_cqp_stats_inst_cmd()
1680 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_stats_inst_cmd()
1690 return -ENOMEM; in irdma_cqp_stats_inst_cmd()
1692 cqp_info = &cqp_request->info; in irdma_cqp_stats_inst_cmd()
1694 cqp_info->cqp_cmd = cmd; in irdma_cqp_stats_inst_cmd()
1695 cqp_info->post_sq = 1; in irdma_cqp_stats_inst_cmd()
1696 cqp_info->in.u.stats_manage.info = *stats_info; in irdma_cqp_stats_inst_cmd()
1697 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; in irdma_cqp_stats_inst_cmd()
1698 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; in irdma_cqp_stats_inst_cmd()
1701 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; in irdma_cqp_stats_inst_cmd()
1708 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1721 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_ceq_cmd()
1723 return -ENOMEM; in irdma_cqp_ceq_cmd()
1725 cqp_info = &cqp_request->info; in irdma_cqp_ceq_cmd()
1726 cqp_info->post_sq = 1; in irdma_cqp_ceq_cmd()
1727 cqp_info->cqp_cmd = op; in irdma_cqp_ceq_cmd()
1728 cqp_info->in.u.ceq_create.ceq = sc_ceq; in irdma_cqp_ceq_cmd()
1729 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_ceq_cmd()
1732 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ceq_cmd()
1738 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1751 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_aeq_cmd()
1753 return -ENOMEM; in irdma_cqp_aeq_cmd()
1755 cqp_info = &cqp_request->info; in irdma_cqp_aeq_cmd()
1756 cqp_info->post_sq = 1; in irdma_cqp_aeq_cmd()
1757 cqp_info->cqp_cmd = op; in irdma_cqp_aeq_cmd()
1758 cqp_info->in.u.aeq_create.aeq = sc_aeq; in irdma_cqp_aeq_cmd()
1759 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_aeq_cmd()
1762 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_aeq_cmd()
1768 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1777 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_ws_node_cmd()
1778 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; in irdma_cqp_ws_node_cmd()
1782 bool poll; in irdma_cqp_ws_node_cmd() local
1784 if (!rf->sc_dev.ceq_valid) in irdma_cqp_ws_node_cmd()
1785 poll = true; in irdma_cqp_ws_node_cmd()
1787 poll = false; in irdma_cqp_ws_node_cmd()
1789 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); in irdma_cqp_ws_node_cmd()
1791 return -ENOMEM; in irdma_cqp_ws_node_cmd()
1793 cqp_info = &cqp_request->info; in irdma_cqp_ws_node_cmd()
1795 cqp_info->cqp_cmd = cmd; in irdma_cqp_ws_node_cmd()
1796 cqp_info->post_sq = 1; in irdma_cqp_ws_node_cmd()
1797 cqp_info->in.u.ws_node.info = *node_info; in irdma_cqp_ws_node_cmd()
1798 cqp_info->in.u.ws_node.cqp = cqp; in irdma_cqp_ws_node_cmd()
1799 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; in irdma_cqp_ws_node_cmd()
1804 if (poll) { in irdma_cqp_ws_node_cmd()
1809 node_info->qs_handle = compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1810 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n", in irdma_cqp_ws_node_cmd()
1813 node_info->qs_handle = cqp_request->compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1817 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ws_node_cmd()
1823 * irdma_ah_cqp_op - perform an AH cqp operation
1843 return -EINVAL; in irdma_ah_cqp_op()
1845 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_ah_cqp_op()
1847 return -ENOMEM; in irdma_ah_cqp_op()
1849 cqp_info = &cqp_request->info; in irdma_ah_cqp_op()
1850 cqp_info->cqp_cmd = cmd; in irdma_ah_cqp_op()
1851 cqp_info->post_sq = 1; in irdma_ah_cqp_op()
1853 cqp_info->in.u.ah_create.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1854 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1855 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1857 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1858 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1859 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1863 cqp_request->callback_fcn = callback_fcn; in irdma_ah_cqp_op()
1864 cqp_request->param = cb_param; in irdma_ah_cqp_op()
1867 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_ah_cqp_op()
1870 return -ENOMEM; in irdma_ah_cqp_op()
1873 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); in irdma_ah_cqp_op()
1879 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1884 struct irdma_sc_qp *qp = cqp_request->param; in irdma_ieq_ah_cb()
1885 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; in irdma_ieq_ah_cb()
1888 spin_lock_irqsave(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
1889 if (!cqp_request->compl_info.op_ret_val) { in irdma_ieq_ah_cb()
1890 sc_ah->ah_info.ah_valid = true; in irdma_ieq_ah_cb()
1891 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); in irdma_ieq_ah_cb()
1893 sc_ah->ah_info.ah_valid = false; in irdma_ieq_ah_cb()
1894 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); in irdma_ieq_ah_cb()
1896 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
1900 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1905 struct irdma_cm_node *cm_node = cqp_request->param; in irdma_ilq_ah_cb()
1906 struct irdma_sc_ah *sc_ah = cm_node->ah; in irdma_ilq_ah_cb()
1908 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; in irdma_ilq_ah_cb()
1913 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
1934 return -ENOMEM; in irdma_puda_create_ah()
1936 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, in irdma_puda_create_ah()
1937 &ah_info->ah_idx, &rf->next_ah); in irdma_puda_create_ah()
1941 ah->dev = dev; in irdma_puda_create_ah()
1942 ah->ah_info = *ah_info; in irdma_puda_create_ah()
1956 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_create_ah()
1960 return -ENOMEM; in irdma_puda_create_ah()
1964 * irdma_puda_free_ah - free a puda address handle
1975 if (ah->ah_info.ah_valid) { in irdma_puda_free_ah()
1977 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_free_ah()
1984 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
1989 struct irdma_sc_ah *sc_ah = cqp_request->param; in irdma_gsi_ud_qp_ah_cb()
1991 if (!cqp_request->compl_info.op_ret_val) in irdma_gsi_ud_qp_ah_cb()
1992 sc_ah->ah_info.ah_valid = true; in irdma_gsi_ud_qp_ah_cb()
1994 sc_ah->ah_info.ah_valid = false; in irdma_gsi_ud_qp_ah_cb()
1998 * irdma_prm_add_pble_mem - add moemory to pble resources
2007 if (pchunk->size & 0xfff) in irdma_prm_add_pble_mem()
2008 return -EINVAL; in irdma_prm_add_pble_mem()
2010 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; in irdma_prm_add_pble_mem()
2012 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); in irdma_prm_add_pble_mem()
2013 if (!pchunk->bitmapbuf) in irdma_prm_add_pble_mem()
2014 return -ENOMEM; in irdma_prm_add_pble_mem()
2016 pchunk->sizeofbitmap = sizeofbitmap; in irdma_prm_add_pble_mem()
2018 pprm->total_pble_alloc += pchunk->size >> 3; in irdma_prm_add_pble_mem()
2019 pprm->free_pble_cnt += pchunk->size >> 3; in irdma_prm_add_pble_mem()
2025 * irdma_prm_get_pbles - get pble's from prm
2039 struct list_head *chunk_entry = pprm->clist.next; in irdma_prm_get_pbles()
2045 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); in irdma_prm_get_pbles()
2047 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2048 while (chunk_entry != &pprm->clist) { in irdma_prm_get_pbles()
2050 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, in irdma_prm_get_pbles()
2051 pchunk->sizeofbitmap, 0, in irdma_prm_get_pbles()
2053 if (bit_idx < pchunk->sizeofbitmap) in irdma_prm_get_pbles()
2057 chunk_entry = pchunk->list.next; in irdma_prm_get_pbles()
2060 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { in irdma_prm_get_pbles()
2061 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2062 return -ENOMEM; in irdma_prm_get_pbles()
2065 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); in irdma_prm_get_pbles()
2066 offset = bit_idx << pprm->pble_shift; in irdma_prm_get_pbles()
2067 *vaddr = pchunk->vaddr + offset; in irdma_prm_get_pbles()
2068 *fpm_addr = pchunk->fpm_addr + offset; in irdma_prm_get_pbles()
2070 chunkinfo->pchunk = pchunk; in irdma_prm_get_pbles()
2071 chunkinfo->bit_idx = bit_idx; in irdma_prm_get_pbles()
2072 chunkinfo->bits_used = bits_needed; in irdma_prm_get_pbles()
2074 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_get_pbles()
2075 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2081 * irdma_prm_return_pbles - return pbles back to prm
2090 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2091 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_return_pbles()
2092 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, in irdma_prm_return_pbles()
2093 chunkinfo->bits_used); in irdma_prm_return_pbles()
2094 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2110 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE, in irdma_map_vm_page_list()
2112 if (dma_mapping_error(hw->device, pg_dma[i])) in irdma_map_vm_page_list()
2122 return -ENOMEM; in irdma_map_vm_page_list()
2130 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); in irdma_unmap_vm_page_list()
2134 * irdma_pble_free_paged_mem - free virtual paged memory
2139 if (!chunk->pg_cnt) in irdma_pble_free_paged_mem()
2142 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, in irdma_pble_free_paged_mem()
2143 chunk->pg_cnt); in irdma_pble_free_paged_mem()
2146 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_free_paged_mem()
2147 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_free_paged_mem()
2148 vfree(chunk->vaddr); in irdma_pble_free_paged_mem()
2149 chunk->vaddr = NULL; in irdma_pble_free_paged_mem()
2150 chunk->type = 0; in irdma_pble_free_paged_mem()
2154 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2163 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); in irdma_pble_get_paged_mem()
2164 if (!chunk->dmainfo.dmaaddrs) in irdma_pble_get_paged_mem()
2165 return -ENOMEM; in irdma_pble_get_paged_mem()
2172 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, in irdma_pble_get_paged_mem()
2177 chunk->vaddr = va; in irdma_pble_get_paged_mem()
2178 chunk->size = size; in irdma_pble_get_paged_mem()
2179 chunk->pg_cnt = pg_cnt; in irdma_pble_get_paged_mem()
2180 chunk->type = PBLE_SD_PAGED; in irdma_pble_get_paged_mem()
2184 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_get_paged_mem()
2185 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_get_paged_mem()
2187 return -ENOMEM; in irdma_pble_get_paged_mem()
2191 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2200 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, in irdma_alloc_ws_node_id()
2208 * irdma_free_ws_node_id - Free a tx scheduler node ID
2216 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); in irdma_free_ws_node_id()
2220 * irdma_modify_qp_to_err - Modify a QP to error
2225 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; in irdma_modify_qp_to_err()
2228 if (qp->iwdev->rf->reset) in irdma_modify_qp_to_err()
2232 if (rdma_protocol_roce(qp->ibqp.device, 1)) in irdma_modify_qp_to_err()
2233 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2235 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2242 if (!iwqp->ibqp.event_handler) in irdma_ib_qp_event()
2256 ibevent.device = iwqp->ibqp.device; in irdma_ib_qp_event()
2257 ibevent.element.qp = &iwqp->ibqp; in irdma_ib_qp_event()
2258 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); in irdma_ib_qp_event()
2268 ukcq = &iwcq->sc_cq.cq_uk; in irdma_cq_empty()
2273 return polarity != ukcq->polarity; in irdma_cq_empty()
2281 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) { in irdma_remove_cmpls_list()
2283 list_del(&cmpl_node->list); in irdma_remove_cmpls_list()
2292 if (list_empty(&iwcq->cmpl_generated)) in irdma_generated_cmpls()
2293 return -ENOENT; in irdma_generated_cmpls()
2294 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); in irdma_generated_cmpls()
2295 list_del(&cmpl->list); in irdma_generated_cmpls()
2296 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); in irdma_generated_cmpls()
2299 ibdev_dbg(iwcq->ibcq.device, in irdma_generated_cmpls()
2300 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n", in irdma_generated_cmpls()
2301 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, in irdma_generated_cmpls()
2302 cq_poll_info->wr_id); in irdma_generated_cmpls()
2308 * irdma_set_cpi_common_values - fill in values for polling info struct
2316 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; in irdma_set_cpi_common_values()
2317 cpi->error = true; in irdma_set_cpi_common_values()
2318 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; in irdma_set_cpi_common_values()
2319 cpi->minor_err = FLUSH_GENERAL_ERR; in irdma_set_cpi_common_values()
2320 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp; in irdma_set_cpi_common_values()
2321 cpi->qp_id = qp_num; in irdma_set_cpi_common_values()
2326 if (!cq->ibcq.comp_handler) in irdma_comp_handler()
2328 if (atomic_cmpxchg(&cq->armed, 1, 0)) in irdma_comp_handler()
2329 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in irdma_comp_handler()
2334 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; in irdma_generate_flush_completions()
2335 struct irdma_ring *sq_ring = &qp->sq_ring; in irdma_generate_flush_completions()
2336 struct irdma_ring *rq_ring = &qp->rq_ring; in irdma_generate_flush_completions()
2344 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2345 if (irdma_cq_empty(iwqp->iwscq)) { in irdma_generate_flush_completions()
2348 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2352 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2353 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2357 wqe_idx = sq_ring->tail; in irdma_generate_flush_completions()
2358 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2360 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; in irdma_generate_flush_completions()
2361 sw_wqe = qp->sq_base[wqe_idx].elem; in irdma_generate_flush_completions()
2363 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); in irdma_generate_flush_completions()
2364 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; in irdma_generate_flush_completions()
2367 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); in irdma_generate_flush_completions()
2368 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { in irdma_generate_flush_completions()
2372 ibdev_dbg(iwqp->iwscq->ibcq.device, in irdma_generate_flush_completions()
2374 __func__, cmpl->cpi.wr_id, qp->qp_id); in irdma_generate_flush_completions()
2375 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); in irdma_generate_flush_completions()
2378 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2379 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2381 irdma_comp_handler(iwqp->iwscq); in irdma_generate_flush_completions()
2383 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2384 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_generate_flush_completions()
2388 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2389 if (irdma_cq_empty(iwqp->iwrcq)) { in irdma_generate_flush_completions()
2392 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2396 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2397 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2401 wqe_idx = rq_ring->tail; in irdma_generate_flush_completions()
2402 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2404 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; in irdma_generate_flush_completions()
2405 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; in irdma_generate_flush_completions()
2406 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; in irdma_generate_flush_completions()
2408 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); in irdma_generate_flush_completions()
2409 ibdev_dbg(iwqp->iwrcq->ibcq.device, in irdma_generate_flush_completions()
2411 __func__, cmpl->cpi.wr_id, qp->qp_id, in irdma_generate_flush_completions()
2413 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); in irdma_generate_flush_completions()
2417 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2418 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2420 irdma_comp_handler(iwqp->iwrcq); in irdma_generate_flush_completions()
2422 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2423 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, in irdma_generate_flush_completions()