Lines Matching +full:pa +full:- +full:stats
1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
4 * Copyright (c) 2015 - 2023 Intel Corporation
16 * - Redistributions of source code must retain the above
20 * - Redistributions in binary form must reproduce the above
41 {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"},
42 {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"},
47 {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"},
48 {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"},
65 "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"},
67 "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"},
71 … to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R…
79 "Denying a privileged operation on a non-privileged QP"},
82 "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"},
104 "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"},
186 * irdma_get_ae_desc - returns AE description
205 * irdma_arp_table -manage arp table
221 spin_lock_irqsave(&rf->arp_lock, flags); in irdma_arp_table()
222 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { in irdma_arp_table()
223 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) in irdma_arp_table()
229 if (arp_index != rf->arp_table_size) { in irdma_arp_table()
230 arp_index = -1; in irdma_arp_table()
235 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, in irdma_arp_table()
236 (u32 *)&arp_index, &rf->next_arp_index)) { in irdma_arp_table()
237 arp_index = -1; in irdma_arp_table()
241 memcpy(rf->arp_table[arp_index].ip_addr, ip, in irdma_arp_table()
242 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
243 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); in irdma_arp_table()
246 if (arp_index == rf->arp_table_size) in irdma_arp_table()
247 arp_index = -1; in irdma_arp_table()
250 if (arp_index == rf->arp_table_size) { in irdma_arp_table()
251 arp_index = -1; in irdma_arp_table()
255 memset(rf->arp_table[arp_index].ip_addr, 0, in irdma_arp_table()
256 sizeof(rf->arp_table[arp_index].ip_addr)); in irdma_arp_table()
257 eth_zero_addr(rf->arp_table[arp_index].mac_addr); in irdma_arp_table()
258 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); in irdma_arp_table()
261 arp_index = -1; in irdma_arp_table()
265 spin_unlock_irqrestore(&rf->arp_lock, flags); in irdma_arp_table()
270 * irdma_add_arp - add a new arp entry if needed
282 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) in irdma_add_arp()
285 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, in irdma_add_arp()
295 * irdma_netdevice_event - system notifier for netdev events
308 if (iwdev->netdev != netdev) in irdma_netdevice_event()
311 iwdev->iw_status = 1; in irdma_netdevice_event()
314 iwdev->iw_status = 0; in irdma_netdevice_event()
329 unregister_netdevice_notifier(&iwdev->nb_netdevice_event); in irdma_unregister_notifiers()
337 iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event; in irdma_register_notifiers()
338 ret = register_netdevice_notifier(&iwdev->nb_netdevice_event); in irdma_register_notifiers()
340 irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); in irdma_register_notifiers()
346 * irdma_alloc_and_get_cqp_request - get cqp struct
357 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
358 if (!list_empty(&cqp->cqp_avail_reqs)) { in irdma_alloc_and_get_cqp_request()
359 cqp_request = list_entry(cqp->cqp_avail_reqs.next, in irdma_alloc_and_get_cqp_request()
361 list_del_init(&cqp_request->list); in irdma_alloc_and_get_cqp_request()
363 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_alloc_and_get_cqp_request()
367 cqp_request->dynamic = true; in irdma_alloc_and_get_cqp_request()
369 init_waitqueue_head(&cqp_request->waitq); in irdma_alloc_and_get_cqp_request()
373 irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory"); in irdma_alloc_and_get_cqp_request()
377 cqp_request->waiting = wait; in irdma_alloc_and_get_cqp_request()
378 atomic_set(&cqp_request->refcnt, 1); in irdma_alloc_and_get_cqp_request()
379 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); in irdma_alloc_and_get_cqp_request()
385 * irdma_get_cqp_request - increase refcount for cqp_request
391 atomic_inc(&cqp_request->refcnt); in irdma_get_cqp_request()
395 * irdma_free_cqp_request - free cqp request
405 if (cqp_request->dynamic) { in irdma_free_cqp_request()
408 WRITE_ONCE(cqp_request->request_done, false); in irdma_free_cqp_request()
409 cqp_request->callback_fcn = NULL; in irdma_free_cqp_request()
410 cqp_request->waiting = false; in irdma_free_cqp_request()
412 spin_lock_irqsave(&cqp->req_lock, flags); in irdma_free_cqp_request()
413 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); in irdma_free_cqp_request()
414 spin_unlock_irqrestore(&cqp->req_lock, flags); in irdma_free_cqp_request()
416 wake_up(&cqp->remove_wq); in irdma_free_cqp_request()
420 * irdma_put_cqp_request - dec ref count and free if 0
428 if (atomic_dec_and_test(&cqp_request->refcnt)) in irdma_put_cqp_request()
433 * irdma_free_pending_cqp_request -free pending cqp request objs
441 cqp_request->compl_info.error = true; in irdma_free_pending_cqp_request()
442 WRITE_ONCE(cqp_request->request_done, true); in irdma_free_pending_cqp_request()
444 if (cqp_request->waiting) in irdma_free_pending_cqp_request()
445 wake_up(&cqp_request->waitq); in irdma_free_pending_cqp_request()
446 wait_event_timeout(cqp->remove_wq, in irdma_free_pending_cqp_request()
447 atomic_read(&cqp_request->refcnt) == 1, 1000); in irdma_free_pending_cqp_request()
452 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
459 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_cleanup_pending_cqp_op()
460 struct irdma_cqp *cqp = &rf->cqp; in irdma_cleanup_pending_cqp_op()
465 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
466 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
469 cqp->scratch_array[wqe_idx]; in irdma_cleanup_pending_cqp_op()
472 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); in irdma_cleanup_pending_cqp_op()
475 while (!list_empty(&dev->cqp_cmd_head)) { in irdma_cleanup_pending_cqp_op()
485 * irdma_wait_event - wait for completion
497 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); in irdma_wait_event()
499 int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms; in irdma_wait_event()
501 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); in irdma_wait_event()
502 if (wait_event_timeout(cqp_request->waitq, in irdma_wait_event()
503 READ_ONCE(cqp_request->request_done), in irdma_wait_event()
507 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); in irdma_wait_event()
512 if (!rf->reset) { in irdma_wait_event()
513 rf->reset = true; in irdma_wait_event()
514 rf->gen_ops.request_reset(rf); in irdma_wait_event()
516 return -ETIMEDOUT; in irdma_wait_event()
519 cqp_error = cqp_request->compl_info.error; in irdma_wait_event()
521 err_code = -EIO; in irdma_wait_event()
522 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { in irdma_wait_event()
523 if (cqp_request->compl_info.min_err_code == 0x8002) { in irdma_wait_event()
524 err_code = -EBUSY; in irdma_wait_event()
525 } else if (cqp_request->compl_info.min_err_code == 0x8029) { in irdma_wait_event()
526 if (!rf->reset) { in irdma_wait_event()
527 rf->reset = true; in irdma_wait_event()
528 rf->gen_ops.request_reset(rf); in irdma_wait_event()
552 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
578 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
597 * irdma_cqp_crit_err - check if CQP error is critical
624 * irdma_handle_cqp_op - process cqp command
632 struct irdma_sc_dev *dev = &rf->sc_dev; in irdma_handle_cqp_op()
633 struct cqp_cmds_info *info = &cqp_request->info; in irdma_handle_cqp_op()
637 if (rf->reset) in irdma_handle_cqp_op()
645 if (cqp_request->waiting) { in irdma_handle_cqp_op()
655 if (irdma_cqp_crit_err(dev, info->cqp_cmd, in irdma_handle_cqp_op()
656 cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
657 cqp_request->compl_info.min_err_code)) in irdma_handle_cqp_op()
658 irdma_dev_err(&rf->iwdev->ibdev, in irdma_handle_cqp_op()
660 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, in irdma_handle_cqp_op()
661 cqp_request->waiting, cqp_request->compl_info.error, in irdma_handle_cqp_op()
662 cqp_request->compl_info.maj_err_code, in irdma_handle_cqp_op()
663 cqp_request->compl_info.min_err_code); in irdma_handle_cqp_op()
666 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_handle_cqp_op()
676 atomic_inc(&iwqp->refcnt); in irdma_qp_add_ref()
683 struct irdma_device *iwdev = iwqp->iwdev; in irdma_qp_rem_ref()
686 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
687 if (!atomic_dec_and_test(&iwqp->refcnt)) { in irdma_qp_rem_ref()
688 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
692 iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL; in irdma_qp_rem_ref()
693 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); in irdma_qp_rem_ref()
694 complete(&iwqp->free_qp); in irdma_qp_rem_ref()
702 atomic_inc(&iwcq->refcnt); in irdma_cq_add_ref()
709 struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev); in irdma_cq_rem_ref()
712 spin_lock_irqsave(&rf->cqtable_lock, flags); in irdma_cq_rem_ref()
713 if (!atomic_dec_and_test(&iwcq->refcnt)) { in irdma_cq_rem_ref()
714 spin_unlock_irqrestore(&rf->cqtable_lock, flags); in irdma_cq_rem_ref()
718 rf->cq_table[iwcq->cq_num] = NULL; in irdma_cq_rem_ref()
719 spin_unlock_irqrestore(&rf->cqtable_lock, flags); in irdma_cq_rem_ref()
720 complete(&iwcq->free_cq); in irdma_cq_rem_ref()
726 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; in to_ibdev()
730 * irdma_get_qp - get qp address
739 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) in irdma_get_qp()
742 return &iwdev->rf->qp_table[qpn]->ibqp; in irdma_get_qp()
746 * irdma_remove_cqp_head - return head entry and remove
753 struct list_head *list = &dev->cqp_cmd_head; in irdma_remove_cqp_head()
758 entry = list->next; in irdma_remove_cqp_head()
765 * irdma_cqp_sds_cmd - create cqp command for sd
779 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_sds_cmd()
781 return -ENOMEM; in irdma_cqp_sds_cmd()
783 cqp_info = &cqp_request->info; in irdma_cqp_sds_cmd()
784 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, in irdma_cqp_sds_cmd()
785 sizeof(cqp_info->in.u.update_pe_sds.info)); in irdma_cqp_sds_cmd()
786 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; in irdma_cqp_sds_cmd()
787 cqp_info->post_sq = 1; in irdma_cqp_sds_cmd()
788 cqp_info->in.u.update_pe_sds.dev = dev; in irdma_cqp_sds_cmd()
789 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; in irdma_cqp_sds_cmd()
792 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_sds_cmd()
798 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
805 struct irdma_sc_dev *dev = qp->dev; in irdma_cqp_qp_suspend_resume()
807 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_cqp_qp_suspend_resume()
812 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_cqp_qp_suspend_resume()
814 return -ENOMEM; in irdma_cqp_qp_suspend_resume()
816 cqp_info = &cqp_request->info; in irdma_cqp_qp_suspend_resume()
817 cqp_info->cqp_cmd = op; in irdma_cqp_qp_suspend_resume()
818 cqp_info->in.u.suspend_resume.cqp = cqp; in irdma_cqp_qp_suspend_resume()
819 cqp_info->in.u.suspend_resume.qp = qp; in irdma_cqp_qp_suspend_resume()
820 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_suspend_resume()
823 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_suspend_resume()
829 * irdma_term_modify_qp - modify qp for term message
841 iwqp = qp->qp_uk.back_qp; in irdma_term_modify_qp()
846 * irdma_terminate_done - after terminate is completed
858 iwqp = qp->qp_uk.back_qp; in irdma_terminate_done()
859 spin_lock_irqsave(&iwqp->lock, flags); in irdma_terminate_done()
860 if (iwqp->hte_added) { in irdma_terminate_done()
861 iwqp->hte_added = 0; in irdma_terminate_done()
864 first_time = !(qp->term_flags & IRDMA_TERM_DONE); in irdma_terminate_done()
865 qp->term_flags |= IRDMA_TERM_DONE; in irdma_terminate_done()
866 spin_unlock_irqrestore(&iwqp->lock, flags); in irdma_terminate_done()
880 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_terminate_timeout()
883 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_timeout()
887 * irdma_terminate_start_timer - start terminate timeout
895 iwqp = qp->qp_uk.back_qp; in irdma_terminate_start_timer()
896 irdma_qp_add_ref(&iwqp->ibqp); in irdma_terminate_start_timer()
897 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); in irdma_terminate_start_timer()
898 iwqp->terminate_timer.expires = jiffies + HZ; in irdma_terminate_start_timer()
900 add_timer(&iwqp->terminate_timer); in irdma_terminate_start_timer()
904 * irdma_terminate_del_timer - delete terminate timeout
913 iwqp = qp->qp_uk.back_qp; in irdma_terminate_del_timer()
914 ret = irdma_del_timer_compat(&iwqp->terminate_timer); in irdma_terminate_del_timer()
916 irdma_qp_rem_ref(&iwqp->ibqp); in irdma_terminate_del_timer()
920 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
934 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_query_fpm_val_cmd()
936 return -ENOMEM; in irdma_cqp_query_fpm_val_cmd()
938 cqp_info = &cqp_request->info; in irdma_cqp_query_fpm_val_cmd()
939 cqp_request->param = NULL; in irdma_cqp_query_fpm_val_cmd()
940 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; in irdma_cqp_query_fpm_val_cmd()
941 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; in irdma_cqp_query_fpm_val_cmd()
942 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; in irdma_cqp_query_fpm_val_cmd()
943 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; in irdma_cqp_query_fpm_val_cmd()
944 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; in irdma_cqp_query_fpm_val_cmd()
945 cqp_info->post_sq = 1; in irdma_cqp_query_fpm_val_cmd()
946 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; in irdma_cqp_query_fpm_val_cmd()
949 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_query_fpm_val_cmd()
955 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
969 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_commit_fpm_val_cmd()
971 return -ENOMEM; in irdma_cqp_commit_fpm_val_cmd()
973 cqp_info = &cqp_request->info; in irdma_cqp_commit_fpm_val_cmd()
974 cqp_request->param = NULL; in irdma_cqp_commit_fpm_val_cmd()
975 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; in irdma_cqp_commit_fpm_val_cmd()
976 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; in irdma_cqp_commit_fpm_val_cmd()
977 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; in irdma_cqp_commit_fpm_val_cmd()
978 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; in irdma_cqp_commit_fpm_val_cmd()
979 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; in irdma_cqp_commit_fpm_val_cmd()
980 cqp_info->post_sq = 1; in irdma_cqp_commit_fpm_val_cmd()
981 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; in irdma_cqp_commit_fpm_val_cmd()
984 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_commit_fpm_val_cmd()
990 * irdma_cqp_cq_create_cmd - create a cq for the cqp
998 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_cq_create_cmd()
1005 return -ENOMEM; in irdma_cqp_cq_create_cmd()
1007 cqp_info = &cqp_request->info; in irdma_cqp_cq_create_cmd()
1008 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; in irdma_cqp_cq_create_cmd()
1009 cqp_info->post_sq = 1; in irdma_cqp_cq_create_cmd()
1010 cqp_info->in.u.cq_create.cq = cq; in irdma_cqp_cq_create_cmd()
1011 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_cq_create_cmd()
1020 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1028 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_create_cmd()
1036 return -ENOMEM; in irdma_cqp_qp_create_cmd()
1038 cqp_info = &cqp_request->info; in irdma_cqp_qp_create_cmd()
1039 qp_info = &cqp_request->info.in.u.qp_create.info; in irdma_cqp_qp_create_cmd()
1041 qp_info->cq_num_valid = true; in irdma_cqp_qp_create_cmd()
1042 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; in irdma_cqp_qp_create_cmd()
1043 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; in irdma_cqp_qp_create_cmd()
1044 cqp_info->post_sq = 1; in irdma_cqp_qp_create_cmd()
1045 cqp_info->in.u.qp_create.qp = qp; in irdma_cqp_qp_create_cmd()
1046 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_create_cmd()
1055 * irdma_dealloc_push_page - free a push page for qp
1066 struct irdma_sc_qp *qp = &iwqp->sc_qp; in irdma_dealloc_push_page()
1068 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) in irdma_dealloc_push_page()
1071 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); in irdma_dealloc_push_page()
1075 cqp_info = &cqp_request->info; in irdma_dealloc_push_page()
1076 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; in irdma_dealloc_push_page()
1077 cqp_info->post_sq = 1; in irdma_dealloc_push_page()
1078 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; in irdma_dealloc_push_page()
1079 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; in irdma_dealloc_push_page()
1080 cqp_info->in.u.manage_push_page.info.free_page = 1; in irdma_dealloc_push_page()
1081 cqp_info->in.u.manage_push_page.info.push_page_type = 0; in irdma_dealloc_push_page()
1082 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; in irdma_dealloc_push_page()
1083 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; in irdma_dealloc_push_page()
1086 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; in irdma_dealloc_push_page()
1087 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_dealloc_push_page()
1091 * irdma_cq_wq_destroy - send cq destroy cqp
1101 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cq_wq_destroy()
1105 cqp_info = &cqp_request->info; in irdma_cq_wq_destroy()
1106 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; in irdma_cq_wq_destroy()
1107 cqp_info->post_sq = 1; in irdma_cq_wq_destroy()
1108 cqp_info->in.u.cq_destroy.cq = cq; in irdma_cq_wq_destroy()
1109 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; in irdma_cq_wq_destroy()
1112 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cq_wq_destroy()
1116 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1125 cqp_info = &cqp_request->info; in irdma_hw_modify_qp_callback()
1126 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; in irdma_hw_modify_qp_callback()
1127 atomic_dec(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp_callback()
1128 wake_up(&iwqp->mod_qp_waitq); in irdma_hw_modify_qp_callback()
1132 * irdma_hw_modify_qp - setup cqp for modify qp
1143 struct irdma_pci_f *rf = iwdev->rf; in irdma_hw_modify_qp()
1148 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_hw_modify_qp()
1150 return -ENOMEM; in irdma_hw_modify_qp()
1153 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; in irdma_hw_modify_qp()
1154 atomic_inc(&iwqp->hw_mod_qp_pend); in irdma_hw_modify_qp()
1156 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1157 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1159 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1160 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1161 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1162 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1164 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1166 if (rdma_protocol_roce(&iwdev->ibdev, 1)) in irdma_hw_modify_qp()
1169 switch (m_info->next_iwarp_state) { in irdma_hw_modify_qp()
1176 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) in irdma_hw_modify_qp()
1177 irdma_send_reset(iwqp->cm_node); in irdma_hw_modify_qp()
1179 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; in irdma_hw_modify_qp()
1183 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); in irdma_hw_modify_qp()
1185 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, in irdma_hw_modify_qp()
1188 return -ENOMEM; in irdma_hw_modify_qp()
1190 cqp_info = &cqp_request->info; in irdma_hw_modify_qp()
1191 m_info = &cqp_info->in.u.qp_modify.info; in irdma_hw_modify_qp()
1193 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; in irdma_hw_modify_qp()
1194 cqp_info->post_sq = 1; in irdma_hw_modify_qp()
1195 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; in irdma_hw_modify_qp()
1196 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; in irdma_hw_modify_qp()
1197 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; in irdma_hw_modify_qp()
1198 m_info->reset_tcp_conn = true; in irdma_hw_modify_qp()
1200 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_hw_modify_qp()
1213 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1226 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1234 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_qp_destroy_cmd()
1241 return -ENOMEM; in irdma_cqp_qp_destroy_cmd()
1243 cqp_info = &cqp_request->info; in irdma_cqp_qp_destroy_cmd()
1245 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; in irdma_cqp_qp_destroy_cmd()
1246 cqp_info->post_sq = 1; in irdma_cqp_qp_destroy_cmd()
1247 cqp_info->in.u.qp_destroy.qp = qp; in irdma_cqp_qp_destroy_cmd()
1248 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; in irdma_cqp_qp_destroy_cmd()
1249 cqp_info->in.u.qp_destroy.remove_hash_idx = true; in irdma_cqp_qp_destroy_cmd()
1252 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_qp_destroy_cmd()
1258 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1268 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); in irdma_ieq_mpa_crc_ae()
1275 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1285 struct irdma_device *iwdev = buf->vsi->back_vsi; in irdma_ieq_get_qp()
1290 struct ip *iph = (struct ip *)buf->iph; in irdma_ieq_get_qp()
1291 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_ieq_get_qp()
1293 if (iph->ip_v == 4) { in irdma_ieq_get_qp()
1294 loc_addr[0] = ntohl(iph->ip_dst.s_addr); in irdma_ieq_get_qp()
1295 rem_addr[0] = ntohl(iph->ip_src.s_addr); in irdma_ieq_get_qp()
1297 ip6h = (struct ip6_hdr *)buf->iph; in irdma_ieq_get_qp()
1298 irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32); in irdma_ieq_get_qp()
1299 irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32); in irdma_ieq_get_qp()
1301 loc_port = ntohs(tcph->th_dport); in irdma_ieq_get_qp()
1302 rem_port = ntohs(tcph->th_sport); in irdma_ieq_get_qp()
1303 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, in irdma_ieq_get_qp()
1304 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); in irdma_ieq_get_qp()
1308 iwqp = cm_node->iwqp; in irdma_ieq_get_qp()
1311 return &iwqp->sc_qp; in irdma_ieq_get_qp()
1315 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1321 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; in irdma_send_ieq_ack()
1322 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; in irdma_send_ieq_ack()
1323 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; in irdma_send_ieq_ack()
1325 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; in irdma_send_ieq_ack()
1326 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); in irdma_send_ieq_ack()
1332 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1340 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; in irdma_puda_ieq_get_ah_info()
1345 ah_info->do_lpbk = true; in irdma_puda_ieq_get_ah_info()
1346 ah_info->vlan_tag = buf->vlan_id; in irdma_puda_ieq_get_ah_info()
1347 ah_info->insert_vlan_tag = buf->vlan_valid; in irdma_puda_ieq_get_ah_info()
1348 ah_info->ipv4_valid = buf->ipv4; in irdma_puda_ieq_get_ah_info()
1349 ah_info->vsi = qp->vsi; in irdma_puda_ieq_get_ah_info()
1351 if (buf->smac_valid) in irdma_puda_ieq_get_ah_info()
1352 ether_addr_copy(ah_info->mac_addr, buf->smac); in irdma_puda_ieq_get_ah_info()
1354 if (buf->ipv4) { in irdma_puda_ieq_get_ah_info()
1355 ah_info->ipv4_valid = true; in irdma_puda_ieq_get_ah_info()
1356 iph = (struct ip *)buf->iph; in irdma_puda_ieq_get_ah_info()
1357 ah_info->hop_ttl = iph->ip_ttl; in irdma_puda_ieq_get_ah_info()
1358 ah_info->tc_tos = iph->ip_tos; in irdma_puda_ieq_get_ah_info()
1359 ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr); in irdma_puda_ieq_get_ah_info()
1360 ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr); in irdma_puda_ieq_get_ah_info()
1362 ip6h = (struct ip6_hdr *)buf->iph; in irdma_puda_ieq_get_ah_info()
1363 ah_info->hop_ttl = ip6h->ip6_hops; in irdma_puda_ieq_get_ah_info()
1364 ah_info->tc_tos = ip6h->ip6_vfc; in irdma_puda_ieq_get_ah_info()
1365 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1366 ip6h->ip6_dst.__u6_addr.__u6_addr32); in irdma_puda_ieq_get_ah_info()
1367 irdma_copy_ip_ntohl(ah_info->src_ip_addr, in irdma_puda_ieq_get_ah_info()
1368 ip6h->ip6_src.__u6_addr.__u6_addr32); in irdma_puda_ieq_get_ah_info()
1371 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), in irdma_puda_ieq_get_ah_info()
1372 ah_info->dest_ip_addr, in irdma_puda_ieq_get_ah_info()
1377 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1390 u8 *addr = buf->mem.va; in irdma_gen1_ieq_update_tcpip_info()
1392 iphlen = (buf->ipv4) ? 20 : 40; in irdma_gen1_ieq_update_tcpip_info()
1393 iph = (struct ip *)(addr + buf->maclen); in irdma_gen1_ieq_update_tcpip_info()
1394 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); in irdma_gen1_ieq_update_tcpip_info()
1395 pktsize = len + buf->tcphlen + iphlen; in irdma_gen1_ieq_update_tcpip_info()
1396 iph->ip_len = htons(pktsize); in irdma_gen1_ieq_update_tcpip_info()
1397 tcph->th_seq = htonl(seqnum); in irdma_gen1_ieq_update_tcpip_info()
1401 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1413 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_ieq_update_tcpip_info()
1416 addr = buf->mem.va; in irdma_ieq_update_tcpip_info()
1418 tcph->th_seq = htonl(seqnum); in irdma_ieq_update_tcpip_info()
1422 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1436 u8 *mem = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1437 struct ether_header *ethh = buf->mem.va; in irdma_gen1_puda_get_tcpip_info()
1439 if (ethh->ether_type == htons(0x8100)) { in irdma_gen1_puda_get_tcpip_info()
1440 info->vlan_valid = true; in irdma_gen1_puda_get_tcpip_info()
1441 buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) & in irdma_gen1_puda_get_tcpip_info()
1445 buf->maclen = (info->vlan_valid) ? 18 : 14; in irdma_gen1_puda_get_tcpip_info()
1446 iphlen = (info->l3proto) ? 40 : 20; in irdma_gen1_puda_get_tcpip_info()
1447 buf->ipv4 = (info->l3proto) ? false : true; in irdma_gen1_puda_get_tcpip_info()
1448 buf->iph = mem + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1449 iph = (struct ip *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1450 buf->tcph = buf->iph + iphlen; in irdma_gen1_puda_get_tcpip_info()
1451 tcph = (struct tcphdr *)buf->tcph; in irdma_gen1_puda_get_tcpip_info()
1453 if (buf->ipv4) { in irdma_gen1_puda_get_tcpip_info()
1454 pkt_len = ntohs(iph->ip_len); in irdma_gen1_puda_get_tcpip_info()
1456 ip6h = (struct ip6_hdr *)buf->iph; in irdma_gen1_puda_get_tcpip_info()
1457 pkt_len = ntohs(ip6h->ip6_plen) + iphlen; in irdma_gen1_puda_get_tcpip_info()
1460 buf->totallen = pkt_len + buf->maclen; in irdma_gen1_puda_get_tcpip_info()
1462 if (info->payload_len < buf->totallen) { in irdma_gen1_puda_get_tcpip_info()
1463 irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR, in irdma_gen1_puda_get_tcpip_info()
1465 info->payload_len, buf->totallen); in irdma_gen1_puda_get_tcpip_info()
1466 return -EINVAL; in irdma_gen1_puda_get_tcpip_info()
1469 buf->tcphlen = tcph->th_off << 2; in irdma_gen1_puda_get_tcpip_info()
1470 buf->datalen = pkt_len - iphlen - buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1471 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_gen1_puda_get_tcpip_info()
1472 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; in irdma_gen1_puda_get_tcpip_info()
1473 buf->seqnum = ntohl(tcph->th_seq); in irdma_gen1_puda_get_tcpip_info()
1479 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1491 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) in irdma_puda_get_tcpip_info()
1494 mem = buf->mem.va; in irdma_puda_get_tcpip_info()
1495 buf->vlan_valid = info->vlan_valid; in irdma_puda_get_tcpip_info()
1496 if (info->vlan_valid) in irdma_puda_get_tcpip_info()
1497 buf->vlan_id = info->vlan; in irdma_puda_get_tcpip_info()
1499 buf->ipv4 = info->ipv4; in irdma_puda_get_tcpip_info()
1500 if (buf->ipv4) in irdma_puda_get_tcpip_info()
1501 buf->iph = mem + IRDMA_IPV4_PAD; in irdma_puda_get_tcpip_info()
1503 buf->iph = mem; in irdma_puda_get_tcpip_info()
1505 buf->tcph = mem + IRDMA_TCP_OFFSET; in irdma_puda_get_tcpip_info()
1506 tcph = (struct tcphdr *)buf->tcph; in irdma_puda_get_tcpip_info()
1507 pkt_len = info->payload_len; in irdma_puda_get_tcpip_info()
1508 buf->totallen = pkt_len; in irdma_puda_get_tcpip_info()
1509 buf->tcphlen = tcph->th_off << 2; in irdma_puda_get_tcpip_info()
1510 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; in irdma_puda_get_tcpip_info()
1511 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; in irdma_puda_get_tcpip_info()
1512 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; in irdma_puda_get_tcpip_info()
1513 buf->seqnum = ntohl(tcph->th_seq); in irdma_puda_get_tcpip_info()
1515 if (info->smac_valid) { in irdma_puda_get_tcpip_info()
1516 ether_addr_copy(buf->smac, info->smac); in irdma_puda_get_tcpip_info()
1517 buf->smac_valid = true; in irdma_puda_get_tcpip_info()
1524 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1532 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; in irdma_hw_stats_timeout()
1534 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) in irdma_hw_stats_timeout()
1535 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); in irdma_hw_stats_timeout()
1537 mod_timer(&pf_devstat->stats_timer, in irdma_hw_stats_timeout()
1542 * irdma_hw_stats_start_timer - Start periodic stats timer
1548 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_start_timer()
1550 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); in irdma_hw_stats_start_timer()
1551 mod_timer(&devstat->stats_timer, in irdma_hw_stats_start_timer()
1556 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1562 struct irdma_vsi_pestat *devstat = vsi->pestat; in irdma_hw_stats_stop_timer()
1564 del_timer_sync(&devstat->stats_timer); in irdma_hw_stats_stop_timer()
1568 * irdma_process_cqp_stats - Checking for wrap and update stats
1574 struct irdma_vsi_pestat *pestat = cqp_request->param; in irdma_process_cqp_stats()
1576 sc_vsi_update_stats(pestat->vsi); in irdma_process_cqp_stats()
1580 * irdma_cqp_gather_stats_cmd - Gather stats
1582 * @pestat: pointer to stats info
1583 * @wait: flag to wait or not wait for stats
1591 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_gather_stats_cmd()
1598 return -ENOMEM; in irdma_cqp_gather_stats_cmd()
1600 cqp_info = &cqp_request->info; in irdma_cqp_gather_stats_cmd()
1602 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; in irdma_cqp_gather_stats_cmd()
1603 cqp_info->post_sq = 1; in irdma_cqp_gather_stats_cmd()
1604 cqp_info->in.u.stats_gather.info = pestat->gather_info; in irdma_cqp_gather_stats_cmd()
1605 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; in irdma_cqp_gather_stats_cmd()
1606 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; in irdma_cqp_gather_stats_cmd()
1607 cqp_request->param = pestat; in irdma_cqp_gather_stats_cmd()
1609 cqp_request->callback_fcn = irdma_process_cqp_stats; in irdma_cqp_gather_stats_cmd()
1612 sc_vsi_update_stats(pestat->vsi); in irdma_cqp_gather_stats_cmd()
1613 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_gather_stats_cmd()
1619 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1622 * @stats_info: pointer to allocate stats info
1628 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); in irdma_cqp_stats_inst_cmd()
1629 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_stats_inst_cmd()
1639 return -ENOMEM; in irdma_cqp_stats_inst_cmd()
1641 cqp_info = &cqp_request->info; in irdma_cqp_stats_inst_cmd()
1643 cqp_info->cqp_cmd = cmd; in irdma_cqp_stats_inst_cmd()
1644 cqp_info->post_sq = 1; in irdma_cqp_stats_inst_cmd()
1645 cqp_info->in.u.stats_manage.info = *stats_info; in irdma_cqp_stats_inst_cmd()
1646 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; in irdma_cqp_stats_inst_cmd()
1647 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; in irdma_cqp_stats_inst_cmd()
1650 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; in irdma_cqp_stats_inst_cmd()
1657 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1671 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_ceq_cmd()
1673 return -ENOMEM; in irdma_cqp_ceq_cmd()
1675 cqp_info = &cqp_request->info; in irdma_cqp_ceq_cmd()
1676 cqp_info->post_sq = 1; in irdma_cqp_ceq_cmd()
1677 cqp_info->cqp_cmd = op; in irdma_cqp_ceq_cmd()
1678 cqp_info->in.u.ceq_create.ceq = sc_ceq; in irdma_cqp_ceq_cmd()
1679 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_ceq_cmd()
1682 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ceq_cmd()
1688 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1702 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); in irdma_cqp_aeq_cmd()
1704 return -ENOMEM; in irdma_cqp_aeq_cmd()
1706 cqp_info = &cqp_request->info; in irdma_cqp_aeq_cmd()
1707 cqp_info->post_sq = 1; in irdma_cqp_aeq_cmd()
1708 cqp_info->cqp_cmd = op; in irdma_cqp_aeq_cmd()
1709 cqp_info->in.u.aeq_create.aeq = sc_aeq; in irdma_cqp_aeq_cmd()
1710 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; in irdma_cqp_aeq_cmd()
1713 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_aeq_cmd()
1719 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1729 struct irdma_cqp *iwcqp = &rf->cqp; in irdma_cqp_ws_node_cmd()
1730 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; in irdma_cqp_ws_node_cmd()
1736 if (!rf->sc_dev.ceq_valid) in irdma_cqp_ws_node_cmd()
1743 return -ENOMEM; in irdma_cqp_ws_node_cmd()
1745 cqp_info = &cqp_request->info; in irdma_cqp_ws_node_cmd()
1747 cqp_info->cqp_cmd = cmd; in irdma_cqp_ws_node_cmd()
1748 cqp_info->post_sq = 1; in irdma_cqp_ws_node_cmd()
1749 cqp_info->in.u.ws_node.info = *node_info; in irdma_cqp_ws_node_cmd()
1750 cqp_info->in.u.ws_node.cqp = cqp; in irdma_cqp_ws_node_cmd()
1751 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; in irdma_cqp_ws_node_cmd()
1761 node_info->qs_handle = compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1762 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, in irdma_cqp_ws_node_cmd()
1766 node_info->qs_handle = cqp_request->compl_info.op_ret_val; in irdma_cqp_ws_node_cmd()
1770 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_cqp_ws_node_cmd()
1776 * irdma_ah_cqp_op - perform an AH cqp operation
1797 return -EINVAL; in irdma_ah_cqp_op()
1799 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); in irdma_ah_cqp_op()
1801 return -ENOMEM; in irdma_ah_cqp_op()
1803 cqp_info = &cqp_request->info; in irdma_ah_cqp_op()
1804 cqp_info->cqp_cmd = cmd; in irdma_ah_cqp_op()
1805 cqp_info->post_sq = 1; in irdma_ah_cqp_op()
1809 sc_ah->ah_info.cqp_request = cqp_request; in irdma_ah_cqp_op()
1811 cqp_info->in.u.ah_create.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1812 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1813 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1815 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; in irdma_ah_cqp_op()
1816 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; in irdma_ah_cqp_op()
1817 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; in irdma_ah_cqp_op()
1821 cqp_request->callback_fcn = callback_fcn; in irdma_ah_cqp_op()
1822 cqp_request->param = cb_param; in irdma_ah_cqp_op()
1825 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_ah_cqp_op()
1828 return -ENOMEM; in irdma_ah_cqp_op()
1831 sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY); in irdma_ah_cqp_op()
1837 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1843 struct irdma_sc_qp *qp = cqp_request->param; in irdma_ieq_ah_cb()
1844 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; in irdma_ieq_ah_cb()
1847 spin_lock_irqsave(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
1848 if (!cqp_request->compl_info.op_ret_val) { in irdma_ieq_ah_cb()
1849 sc_ah->ah_info.ah_valid = true; in irdma_ieq_ah_cb()
1850 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); in irdma_ieq_ah_cb()
1852 sc_ah->ah_info.ah_valid = false; in irdma_ieq_ah_cb()
1853 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); in irdma_ieq_ah_cb()
1855 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); in irdma_ieq_ah_cb()
1859 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1865 struct irdma_cm_node *cm_node = cqp_request->param; in irdma_ilq_ah_cb()
1866 struct irdma_sc_ah *sc_ah = cm_node->ah; in irdma_ilq_ah_cb()
1868 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; in irdma_ilq_ah_cb()
1873 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
1895 return -ENOMEM; in irdma_puda_create_ah()
1897 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, in irdma_puda_create_ah()
1898 &ah_info->ah_idx, &rf->next_ah); in irdma_puda_create_ah()
1902 ah->dev = dev; in irdma_puda_create_ah()
1903 ah->ah_info = *ah_info; in irdma_puda_create_ah()
1917 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_create_ah()
1921 return -ENOMEM; in irdma_puda_create_ah()
1925 * irdma_puda_free_ah - free a puda address handle
1937 if (ah->ah_info.ah_valid) { in irdma_puda_free_ah()
1939 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); in irdma_puda_free_ah()
1946 * irdma_prm_add_pble_mem - add moemory to pble resources
1956 if (pchunk->size & 0xfff) in irdma_prm_add_pble_mem()
1957 return -EINVAL; in irdma_prm_add_pble_mem()
1959 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; in irdma_prm_add_pble_mem()
1961 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); in irdma_prm_add_pble_mem()
1962 if (!pchunk->bitmapbuf) in irdma_prm_add_pble_mem()
1963 return -ENOMEM; in irdma_prm_add_pble_mem()
1965 pchunk->sizeofbitmap = sizeofbitmap; in irdma_prm_add_pble_mem()
1967 pprm->total_pble_alloc += pchunk->size >> 3; in irdma_prm_add_pble_mem()
1968 pprm->free_pble_cnt += pchunk->size >> 3; in irdma_prm_add_pble_mem()
1974 * irdma_prm_get_pbles - get pble's from prm
1989 struct list_head *chunk_entry = (&pprm->clist)->next; in irdma_prm_get_pbles()
1996 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); in irdma_prm_get_pbles()
1998 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
1999 while (chunk_entry != &pprm->clist) { in irdma_prm_get_pbles()
2001 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, in irdma_prm_get_pbles()
2002 pchunk->sizeofbitmap, 0, in irdma_prm_get_pbles()
2004 if (bit_idx < pchunk->sizeofbitmap) in irdma_prm_get_pbles()
2008 chunk_entry = (&pchunk->list)->next; in irdma_prm_get_pbles()
2011 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { in irdma_prm_get_pbles()
2012 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2013 return -ENOMEM; in irdma_prm_get_pbles()
2016 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); in irdma_prm_get_pbles()
2017 offset = bit_idx << pprm->pble_shift; in irdma_prm_get_pbles()
2018 *vaddr = (u64 *)((u8 *)pchunk->vaddr + offset); in irdma_prm_get_pbles()
2019 *fpm_addr = pchunk->fpm_addr + offset; in irdma_prm_get_pbles()
2021 chunkinfo->pchunk = pchunk; in irdma_prm_get_pbles()
2022 chunkinfo->bit_idx = bit_idx; in irdma_prm_get_pbles()
2023 chunkinfo->bits_used = bits_needed; in irdma_prm_get_pbles()
2025 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_get_pbles()
2026 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_get_pbles()
2032 * irdma_prm_return_pbles - return pbles back to prm
2042 spin_lock_irqsave(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2043 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); in irdma_prm_return_pbles()
2044 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, in irdma_prm_return_pbles()
2045 chunkinfo->bits_used); in irdma_prm_return_pbles()
2046 spin_unlock_irqrestore(&pprm->prm_lock, flags); in irdma_prm_return_pbles()
2074 return -ENOMEM; in irdma_map_vm_page_list()
2087 * irdma_pble_free_paged_mem - free virtual paged memory
2093 if (!chunk->pg_cnt) in irdma_pble_free_paged_mem()
2096 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, in irdma_pble_free_paged_mem()
2097 chunk->pg_cnt); in irdma_pble_free_paged_mem()
2100 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_free_paged_mem()
2101 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_free_paged_mem()
2102 vfree(chunk->vaddr); in irdma_pble_free_paged_mem()
2103 chunk->vaddr = NULL; in irdma_pble_free_paged_mem()
2104 chunk->type = 0; in irdma_pble_free_paged_mem()
2108 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2118 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); in irdma_pble_get_paged_mem()
2119 if (!chunk->dmainfo.dmaaddrs) in irdma_pble_get_paged_mem()
2120 return -ENOMEM; in irdma_pble_get_paged_mem()
2127 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, in irdma_pble_get_paged_mem()
2132 chunk->vaddr = va; in irdma_pble_get_paged_mem()
2133 chunk->size = size; in irdma_pble_get_paged_mem()
2134 chunk->pg_cnt = pg_cnt; in irdma_pble_get_paged_mem()
2135 chunk->type = PBLE_SD_PAGED; in irdma_pble_get_paged_mem()
2139 kfree(chunk->dmainfo.dmaaddrs); in irdma_pble_get_paged_mem()
2140 chunk->dmainfo.dmaaddrs = NULL; in irdma_pble_get_paged_mem()
2142 return -ENOMEM; in irdma_pble_get_paged_mem()
2146 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2156 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, in irdma_alloc_ws_node_id()
2164 * irdma_free_ws_node_id - Free a tx scheduler node ID
2173 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); in irdma_free_ws_node_id()
2177 * irdma_modify_qp_to_err - Modify a QP to error
2183 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; in irdma_modify_qp_to_err()
2186 if (qp->iwdev->rf->reset) in irdma_modify_qp_to_err()
2190 if (rdma_protocol_roce(qp->ibqp.device, 1)) in irdma_modify_qp_to_err()
2191 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2193 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); in irdma_modify_qp_to_err()
2201 if (!iwqp->ibqp.event_handler) in irdma_ib_qp_event()
2215 ibevent.device = iwqp->ibqp.device; in irdma_ib_qp_event()
2216 ibevent.element.qp = &iwqp->ibqp; in irdma_ib_qp_event()
2217 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); in irdma_ib_qp_event()
2241 * irdma_upload_qp_context - upload raw QP context
2260 rf = iwqp->iwdev->rf; in irdma_upload_qp_context()
2262 return -EINVAL; in irdma_upload_qp_context()
2264 qp = &iwqp->sc_qp; in irdma_upload_qp_context()
2265 dev = &rf->sc_dev; in irdma_upload_qp_context()
2266 iwcqp = &rf->cqp; in irdma_upload_qp_context()
2270 return -EINVAL; in irdma_upload_qp_context()
2272 cqp_info = &cqp_request->info; in irdma_upload_qp_context()
2273 info = &cqp_info->in.u.qp_upload_context.info; in irdma_upload_qp_context()
2275 cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT; in irdma_upload_qp_context()
2276 cqp_info->post_sq = 1; in irdma_upload_qp_context()
2277 cqp_info->in.u.qp_upload_context.dev = dev; in irdma_upload_qp_context()
2278 cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request; in irdma_upload_qp_context()
2281 dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE); in irdma_upload_qp_context()
2283 irdma_put_cqp_request(&rf->cqp, cqp_request); in irdma_upload_qp_context()
2284 return -ENOMEM; in irdma_upload_qp_context()
2288 info->buf_pa = dma_mem.pa; in irdma_upload_qp_context()
2289 info->raw_format = raw; in irdma_upload_qp_context()
2290 info->freeze_qp = freeze; in irdma_upload_qp_context()
2291 info->qp_type = qp->qp_uk.qp_type; /* 1 is iWARP and 2 UDA */ in irdma_upload_qp_context()
2292 info->qp_id = qp->qp_uk.qp_id; in irdma_upload_qp_context()
2296 irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id); in irdma_upload_qp_context()
2308 irdma_free_dma_mem(dev->hw, &dma_mem); in irdma_upload_qp_context()
2321 ukcq = &iwcq->sc_cq.cq_uk; in irdma_cq_empty()
2326 return polarity != ukcq->polarity; in irdma_cq_empty()
2335 list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) { in irdma_remove_cmpls_list()
2337 list_del(&cmpl_node->list); in irdma_remove_cmpls_list()
2347 if (list_empty(&iwcq->cmpl_generated)) in irdma_generated_cmpls()
2348 return -ENOENT; in irdma_generated_cmpls()
2349 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); in irdma_generated_cmpls()
2350 list_del(&cmpl->list); in irdma_generated_cmpls()
2351 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); in irdma_generated_cmpls()
2354 irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS, in irdma_generated_cmpls()
2356 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, in irdma_generated_cmpls()
2357 cq_poll_info->wr_id); in irdma_generated_cmpls()
2363 * irdma_set_cpi_common_values - fill in values for polling info struct
2372 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; in irdma_set_cpi_common_values()
2373 cpi->error = 1; in irdma_set_cpi_common_values()
2374 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; in irdma_set_cpi_common_values()
2375 cpi->minor_err = FLUSH_GENERAL_ERR; in irdma_set_cpi_common_values()
2376 cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp; in irdma_set_cpi_common_values()
2377 cpi->qp_id = qp_num; in irdma_set_cpi_common_values()
2383 if (!cq->ibcq.comp_handler) in irdma_comp_handler()
2386 if (atomic_cmpxchg(&cq->armed, 1, 0)) in irdma_comp_handler()
2387 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); in irdma_comp_handler()
2391 * irdma_generate_flush_completions - generate completion from WRs
2397 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; in irdma_generate_flush_completions()
2398 struct irdma_ring *sq_ring = &qp->sq_ring; in irdma_generate_flush_completions()
2399 struct irdma_ring *rq_ring = &qp->rq_ring; in irdma_generate_flush_completions()
2407 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2408 if (irdma_cq_empty(iwqp->iwscq)) { in irdma_generate_flush_completions()
2411 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2415 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2416 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2420 wqe_idx = sq_ring->tail; in irdma_generate_flush_completions()
2421 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2423 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; in irdma_generate_flush_completions()
2424 cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled; in irdma_generate_flush_completions()
2425 sw_wqe = qp->sq_base[wqe_idx].elem; in irdma_generate_flush_completions()
2427 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); in irdma_generate_flush_completions()
2428 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; in irdma_generate_flush_completions()
2431 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); in irdma_generate_flush_completions()
2433 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { in irdma_generate_flush_completions()
2437 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, in irdma_generate_flush_completions()
2439 __func__, cmpl->cpi.wr_id, qp->qp_id); in irdma_generate_flush_completions()
2440 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); in irdma_generate_flush_completions()
2443 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2444 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2446 irdma_comp_handler(iwqp->iwscq); in irdma_generate_flush_completions()
2450 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); in irdma_generate_flush_completions()
2454 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2455 if (irdma_cq_empty(iwqp->iwrcq)) { in irdma_generate_flush_completions()
2458 spin_lock_irqsave(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2462 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2463 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2467 wqe_idx = rq_ring->tail; in irdma_generate_flush_completions()
2468 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); in irdma_generate_flush_completions()
2470 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; in irdma_generate_flush_completions()
2471 cmpl->cpi.signaled = 1; in irdma_generate_flush_completions()
2472 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; in irdma_generate_flush_completions()
2473 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; in irdma_generate_flush_completions()
2475 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); in irdma_generate_flush_completions()
2476 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, in irdma_generate_flush_completions()
2478 __func__, cmpl->cpi.wr_id, qp->qp_id, in irdma_generate_flush_completions()
2481 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); in irdma_generate_flush_completions()
2485 spin_unlock_irqrestore(&iwqp->lock, flags2); in irdma_generate_flush_completions()
2486 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2488 irdma_comp_handler(iwqp->iwrcq); in irdma_generate_flush_completions()
2490 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); in irdma_generate_flush_completions()
2496 * irdma_udqp_qs_change - change qs for UD QP in a worker thread
2504 irdma_qp_rem_qos(&iwqp->sc_qp); in irdma_udqp_qs_change()
2506 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri); in irdma_udqp_qs_change()
2508 iwqp->ctx_info.user_pri = user_prio; in irdma_udqp_qs_change()
2509 iwqp->sc_qp.user_pri = user_prio; in irdma_udqp_qs_change()
2512 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio)) in irdma_udqp_qs_change()
2513 irdma_dev_warn(&iwqp->iwdev->ibdev, in irdma_udqp_qs_change()
2515 __func__, iwqp->ibqp.qp_num, user_prio); in irdma_udqp_qs_change()
2516 irdma_qp_add_qos(&iwqp->sc_qp); in irdma_udqp_qs_change()
2524 irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change); in irdma_udqp_qs_worker()
2525 if (udqs_work->qs_change) in irdma_udqp_qs_worker()
2526 irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME); in irdma_udqp_qs_worker()
2527 irdma_qp_rem_ref(&udqs_work->iwqp->ibqp); in irdma_udqp_qs_worker()