766 static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe)  in ionic_next_eqe()  argument
771 	qeqe = ionic_queue_at_prod(&eq->q);  in ionic_next_eqe()
775 	if (eq->q.cons != color)  in ionic_next_eqe()
781 	ibdev_dbg(&eq->dev->ibdev, "poll eq prod %u\n", eq->q.prod);  in ionic_next_eqe()
783 			     qeqe, BIT(eq->q.stride_log2), true);  in ionic_next_eqe()
893 static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget)  in ionic_poll_eq()  argument
895 	struct ionic_ibdev *dev = eq->dev;  in ionic_poll_eq()
902 		if (!ionic_next_eqe(eq, &eqe))  in ionic_poll_eq()
905 		ionic_queue_produce(&eq->q);  in ionic_poll_eq()
908 		eq->q.cons = ionic_color_wrap(eq->q.prod, eq->q.cons);  in ionic_poll_eq()
937 	struct ionic_eq *eq = container_of(work, struct ionic_eq, work);  in ionic_poll_eq_work()  local
940 	if (unlikely(!eq->enable) || WARN_ON(eq->armed))  in ionic_poll_eq_work()
943 	npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET);  in ionic_poll_eq_work()
945 		ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,  in ionic_poll_eq_work()
947 		queue_work(ionic_evt_workq, &eq->work);  in ionic_poll_eq_work()
949 		xchg(&eq->armed, 1);  in ionic_poll_eq_work()
950 		ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,  in ionic_poll_eq_work()
957 	struct ionic_eq *eq = eqptr;  in ionic_poll_eq_isr()  local
961 	was_armed = xchg(&eq->armed, 0);  in ionic_poll_eq_isr()
963 	if (unlikely(!eq->enable) || !was_armed)  in ionic_poll_eq_isr()
966 	npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET);  in ionic_poll_eq_isr()
968 		ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,  in ionic_poll_eq_isr()
970 		queue_work(ionic_evt_workq, &eq->work);  in ionic_poll_eq_isr()
972 		xchg(&eq->armed, 1);  in ionic_poll_eq_isr()
973 		ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr,  in ionic_poll_eq_isr()
983 	struct ionic_eq *eq;  in ionic_create_eq()  local
986 	eq = kzalloc_obj(*eq);  in ionic_create_eq()
987 	if (!eq)  in ionic_create_eq()
990 	eq->dev = dev;  in ionic_create_eq()
992 	rc = ionic_queue_init(&eq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH,  in ionic_create_eq()
997 	eq->eqid = eqid;  in ionic_create_eq()
999 	eq->armed = true;  in ionic_create_eq()
1000 	eq->enable = false;  in ionic_create_eq()
1001 	INIT_WORK(&eq->work, ionic_poll_eq_work);  in ionic_create_eq()
1007 	eq->irq = intr_obj.vector;  in ionic_create_eq()
1008 	eq->intr = intr_obj.index;  in ionic_create_eq()
1010 	ionic_queue_dbell_init(&eq->q, eq->eqid);  in ionic_create_eq()
1013 	eq->q.cons = true;  in ionic_create_eq()
1015 	snprintf(eq->name, sizeof(eq->name), "%s-%d-%d-eq",  in ionic_create_eq()
1016 		 "ionr", dev->lif_cfg.lif_index, eq->eqid);  in ionic_create_eq()
1018 	ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET);  in ionic_create_eq()
1019 	ionic_intr_mask_assert(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET);  in ionic_create_eq()
1020 	ionic_intr_coal_init(dev->lif_cfg.intr_ctrl, eq->intr, 0);  in ionic_create_eq()
1021 	ionic_intr_clean(dev->lif_cfg.intr_ctrl, eq->intr);  in ionic_create_eq()
1023 	eq->enable = true;  in ionic_create_eq()
1025 	rc = request_irq(eq->irq, ionic_poll_eq_isr, 0, eq->name, eq);  in ionic_create_eq()
1029 	rc = ionic_rdma_queue_devcmd(dev, &eq->q, eq->eqid, eq->intr,  in ionic_create_eq()
1034 	ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_CLEAR);  in ionic_create_eq()
1036 	return eq;  in ionic_create_eq()
1039 	eq->enable = false;  in ionic_create_eq()
1040 	free_irq(eq->irq, eq);  in ionic_create_eq()
1041 	flush_work(&eq->work);  in ionic_create_eq()
1043 	ionic_intr_free(dev->lif_cfg.lif, eq->intr);  in ionic_create_eq()
1045 	ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev);  in ionic_create_eq()
1047 	kfree(eq);  in ionic_create_eq()
1052 static void ionic_destroy_eq(struct ionic_eq *eq)  in ionic_destroy_eq()  argument
1054 	struct ionic_ibdev *dev = eq->dev;  in ionic_destroy_eq()
1056 	eq->enable = false;  in ionic_destroy_eq()
1057 	free_irq(eq->irq, eq);  in ionic_destroy_eq()
1058 	flush_work(&eq->work);  in ionic_destroy_eq()
1060 	ionic_intr_free(dev->lif_cfg.lif, eq->intr);  in ionic_destroy_eq()
1061 	ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev);  in ionic_destroy_eq()
1062 	kfree(eq);  in ionic_destroy_eq()
1070 	struct ionic_eq *eq;  in ionic_create_rdma_admin()  local
1105 		eq = ionic_create_eq(dev, eq_i + dev->lif_cfg.eq_base);  in ionic_create_rdma_admin()
1106 		if (IS_ERR(eq)) {  in ionic_create_rdma_admin()
1107 			rc = PTR_ERR(eq);  in ionic_create_rdma_admin()
1111 					  "fail create eq %pe\n", eq);  in ionic_create_rdma_admin()
1117 				  eq_i, dev->lif_cfg.eq_count, eq);  in ionic_create_rdma_admin()
1123 		dev->eq_vec[eq_i] = eq;  in ionic_create_rdma_admin()
1192 	struct ionic_eq *eq;  in ionic_destroy_rdma_admin()  local
1221 			eq = dev->eq_vec[--dev->lif_cfg.eq_count];  in ionic_destroy_rdma_admin()
1222 			ionic_destroy_eq(eq);  in ionic_destroy_rdma_admin()