2587 	struct irdma_sc_ceq *ceq;  in irdma_sc_cq_create()  local
2597 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_create()
2598 	if (ceq && ceq->reg_cq) {  in irdma_sc_cq_create()
2599 		ret_code = irdma_sc_add_cq_ctx(ceq, cq);  in irdma_sc_cq_create()
2606 		if (ceq && ceq->reg_cq)  in irdma_sc_cq_create()
2607 			irdma_sc_remove_cq_ctx(ceq, cq);  in irdma_sc_cq_create()
2662 	struct irdma_sc_ceq *ceq;  in irdma_sc_cq_destroy()  local
2669 	ceq = cq->dev->ceq[cq->ceq_id];  in irdma_sc_cq_destroy()
2670 	if (ceq && ceq->reg_cq)  in irdma_sc_cq_destroy()
2671 		irdma_sc_remove_cq_ctx(ceq, cq);  in irdma_sc_cq_destroy()
3102 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,  in irdma_sc_find_reg_cq()  argument
3106 	for (i = 0; i < ceq->reg_cq_size; i++) {  in irdma_sc_find_reg_cq()
3107 		if (cq == ceq->reg_cq[i])  in irdma_sc_find_reg_cq()
3120 irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)  in irdma_sc_add_cq_ctx()  argument
3124 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3126 	if (ceq->reg_cq_size == ceq->elem_cnt) {  in irdma_sc_add_cq_ctx()
3127 		spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3131 	ceq->reg_cq[ceq->reg_cq_size++] = cq;  in irdma_sc_add_cq_ctx()
3133 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_add_cq_ctx()
3144 irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)  in irdma_sc_remove_cq_ctx()  argument
3149 	spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3150 	cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);  in irdma_sc_remove_cq_ctx()
3154 	ceq->reg_cq_size--;  in irdma_sc_remove_cq_ctx()
3155 	if (cq_ctx_idx != ceq->reg_cq_size)  in irdma_sc_remove_cq_ctx()
3156 		ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];  in irdma_sc_remove_cq_ctx()
3157 	ceq->reg_cq[ceq->reg_cq_size] = NULL;  in irdma_sc_remove_cq_ctx()
3160 	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_remove_cq_ctx()
3719 irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,  in irdma_sc_ceq_init()  argument
3735 	ceq->size = sizeof(*ceq);  in irdma_sc_ceq_init()
3736 	ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;  in irdma_sc_ceq_init()
3737 	ceq->ceq_id = info->ceq_id;  in irdma_sc_ceq_init()
3738 	ceq->dev = info->dev;  in irdma_sc_ceq_init()
3739 	ceq->elem_cnt = info->elem_cnt;  in irdma_sc_ceq_init()
3740 	ceq->ceq_elem_pa = info->ceqe_pa;  in irdma_sc_ceq_init()
3741 	ceq->virtual_map = info->virtual_map;  in irdma_sc_ceq_init()
3742 	ceq->itr_no_expire = info->itr_no_expire;  in irdma_sc_ceq_init()
3743 	ceq->reg_cq = info->reg_cq;  in irdma_sc_ceq_init()
3744 	ceq->reg_cq_size = 0;  in irdma_sc_ceq_init()
3745 	spin_lock_init(&ceq->req_cq_lock);  in irdma_sc_ceq_init()
3746 	ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);  in irdma_sc_ceq_init()
3747 	ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);  in irdma_sc_ceq_init()
3748 	ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);  in irdma_sc_ceq_init()
3749 	ceq->tph_en = info->tph_en;  in irdma_sc_ceq_init()
3750 	ceq->tph_val = info->tph_val;  in irdma_sc_ceq_init()
3751 	ceq->vsi = info->vsi;  in irdma_sc_ceq_init()
3752 	ceq->polarity = 1;  in irdma_sc_ceq_init()
3753 	IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);  in irdma_sc_ceq_init()
3754 	ceq->dev->ceq[info->ceq_id] = ceq;  in irdma_sc_ceq_init()
3766 irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,  in irdma_sc_ceq_create()  argument
3773 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_create()
3777 	set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt);  in irdma_sc_ceq_create()
3779 		      (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));  in irdma_sc_ceq_create()
3781 		      (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));  in irdma_sc_ceq_create()
3783 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |  in irdma_sc_ceq_create()
3784 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));  in irdma_sc_ceq_create()
3785 	hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |  in irdma_sc_ceq_create()
3787 	    FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_create()
3788 	    FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_create()
3789 	    FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |  in irdma_sc_ceq_create()
3790 	    FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_create()
3809 irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)  in irdma_sc_cceq_create_done()  argument
3813 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_create_done()
3823 irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)  in irdma_sc_cceq_destroy_done()  argument
3827 	if (ceq->reg_cq)  in irdma_sc_cceq_destroy_done()
3828 		irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_destroy_done()
3829 	cqp = ceq->dev->cqp;  in irdma_sc_cceq_destroy_done()
3842 irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)  in irdma_sc_cceq_create()  argument
3845 	struct irdma_sc_dev *dev = ceq->dev;  in irdma_sc_cceq_create()
3847 	dev->ccq->vsi = ceq->vsi;  in irdma_sc_cceq_create()
3848 	if (ceq->reg_cq) {  in irdma_sc_cceq_create()
3849 		ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);  in irdma_sc_cceq_create()
3853 	ret_code = irdma_sc_ceq_create(ceq, scratch, true);  in irdma_sc_cceq_create()
3855 		return irdma_sc_cceq_create_done(ceq);  in irdma_sc_cceq_create()
3867 irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq)  in irdma_sc_ceq_destroy()  argument
3873 	cqp = ceq->dev->cqp;  in irdma_sc_ceq_destroy()
3878 	set_64bit_val(wqe, IRDMA_BYTE_16, ceq->elem_cnt);  in irdma_sc_ceq_destroy()
3879 	set_64bit_val(wqe, IRDMA_BYTE_48, ceq->first_pm_pbl_idx);  in irdma_sc_ceq_destroy()
3880 	hdr = ceq->ceq_id |  in irdma_sc_ceq_destroy()
3882 	    FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |  in irdma_sc_ceq_destroy()
3883 	    FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |  in irdma_sc_ceq_destroy()
3884 	    FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |  in irdma_sc_ceq_destroy()
3892 	ceq->dev->ceq[ceq->ceq_id] = NULL;  in irdma_sc_ceq_destroy()
3908 irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)  in irdma_sc_process_ceq()  argument
3920 		ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);  in irdma_sc_process_ceq()
3923 		if (polarity != ceq->polarity)  in irdma_sc_process_ceq()
3929 			IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3931 			if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3932 				ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3937 		if (ceq->reg_cq) {  in irdma_sc_process_ceq()
3938 			spin_lock_irqsave(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3939 			cq_idx = irdma_sc_find_reg_cq(ceq, cq);  in irdma_sc_process_ceq()
3940 			spin_unlock_irqrestore(&ceq->req_cq_lock, flags);  in irdma_sc_process_ceq()
3942 		IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);  in irdma_sc_process_ceq()
3943 		if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))  in irdma_sc_process_ceq()
3944 			ceq->polarity ^= 1;  in irdma_sc_process_ceq()
3962 irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)  in irdma_sc_cleanup_ceqes()  argument
3965 	u8 ceq_polarity = ceq->polarity;  in irdma_sc_cleanup_ceqes()
3972 	next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);  in irdma_sc_cleanup_ceqes()
3974 	for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) {  in irdma_sc_cleanup_ceqes()
3975 		ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);  in irdma_sc_cleanup_ceqes()
3986 		next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);  in irdma_sc_cleanup_ceqes()
5106 		status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,  in irdma_exec_cqp_cmd()
5116 		status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,  in irdma_exec_cqp_cmd()