31 static void ionic_admin_timedout(struct ionic_aq *aq)  in ionic_admin_timedout()  argument
33 	struct ionic_ibdev *dev = aq->dev;  in ionic_admin_timedout()
37 	spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_timedout()
38 	if (ionic_queue_empty(&aq->q))  in ionic_admin_timedout()
42 	if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED)  in ionic_admin_timedout()
46 		  aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp));  in ionic_admin_timedout()
48 	pos = (aq->q.prod - 1) & aq->q.mask;  in ionic_admin_timedout()
49 	if (pos == aq->q.cons)  in ionic_admin_timedout()
54 		       ionic_queue_at(&aq->q, pos),  in ionic_admin_timedout()
55 		       BIT(aq->q.stride_log2), true);  in ionic_admin_timedout()
58 	spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_timedout()
70 static void ionic_admin_reset_wdog(struct ionic_aq *aq)  in ionic_admin_reset_wdog()  argument
72 	if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED)  in ionic_admin_reset_wdog()
75 	aq->stamp = jiffies;  in ionic_admin_reset_wdog()
76 	ionic_admin_reset_dwork(aq->dev);  in ionic_admin_reset_wdog()
94 static void ionic_admin_poll_locked(struct ionic_aq *aq)  in ionic_admin_poll_locked()  argument
96 	struct ionic_cq *cq = &aq->vcq->cq[0];  in ionic_admin_poll_locked()
98 	struct ionic_ibdev *dev = aq->dev;  in ionic_admin_poll_locked()
105 	lockdep_assert_held(&aq->lock);  in ionic_admin_poll_locked()
107 	if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) {  in ionic_admin_poll_locked()
108 		list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) {  in ionic_admin_poll_locked()
110 			aq->q_wr[wr->status].wr = NULL;  in ionic_admin_poll_locked()
111 			wr->status = atomic_read(&aq->admin_state);  in ionic_admin_poll_locked()
114 		INIT_LIST_HEAD(&aq->wr_prod);  in ionic_admin_poll_locked()
116 		list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) {  in ionic_admin_poll_locked()
118 			wr->status = atomic_read(&aq->admin_state);  in ionic_admin_poll_locked()
121 		INIT_LIST_HEAD(&aq->wr_post);  in ionic_admin_poll_locked()
139 		if (unlikely(qid != aq->aqid)) {  in ionic_admin_poll_locked()
145 		if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) {  in ionic_admin_poll_locked()
149 					       aq->q.cons, qid);  in ionic_admin_poll_locked()
153 		if (unlikely(ionic_queue_empty(&aq->q))) {  in ionic_admin_poll_locked()
159 		wr = aq->q_wr[aq->q.cons].wr;  in ionic_admin_poll_locked()
161 			aq->q_wr[aq->q.cons].wr = NULL;  in ionic_admin_poll_locked()
165 			wr->status = atomic_read(&aq->admin_state);  in ionic_admin_poll_locked()
169 		ionic_queue_consume_entries(&aq->q,  in ionic_admin_poll_locked()
170 					    aq->q_wr[aq->q.cons].wqe_strides);  in ionic_admin_poll_locked()
178 		ionic_admin_reset_wdog(aq);  in ionic_admin_poll_locked()
182 		queue_work(ionic_evt_workq, &aq->work);  in ionic_admin_poll_locked()
183 	} else if (!aq->armed) {  in ionic_admin_poll_locked()
184 		aq->armed = true;  in ionic_admin_poll_locked()
189 		queue_work(ionic_evt_workq, &aq->work);  in ionic_admin_poll_locked()
192 	if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE)  in ionic_admin_poll_locked()
195 	old_prod = aq->q.prod;  in ionic_admin_poll_locked()
197 	if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post))  in ionic_admin_poll_locked()
198 		ionic_admin_reset_wdog(aq);  in ionic_admin_poll_locked()
200 	if (list_empty(&aq->wr_post))  in ionic_admin_poll_locked()
208 		wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr,  in ionic_admin_poll_locked()
211 			     (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2;  in ionic_admin_poll_locked()
212 		avlbl_strides = ionic_queue_length_remaining(&aq->q);  in ionic_admin_poll_locked()
217 		list_move(&wr->aq_ent, &aq->wr_prod);  in ionic_admin_poll_locked()
218 		wr->status = aq->q.prod;  in ionic_admin_poll_locked()
219 		aq->q_wr[aq->q.prod].wr = wr;  in ionic_admin_poll_locked()
220 		aq->q_wr[aq->q.prod].wqe_strides = wr_strides;  in ionic_admin_poll_locked()
226 		memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe,  in ionic_admin_poll_locked()
231 		memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN,  in ionic_admin_poll_locked()
234 			  aq->q.prod, wr_strides);  in ionic_admin_poll_locked()
236 				     ionic_queue_at_prod(&aq->q),  in ionic_admin_poll_locked()
237 				     BIT(aq->q.stride_log2), true);  in ionic_admin_poll_locked()
238 		ionic_queue_produce(&aq->q);  in ionic_admin_poll_locked()
247 			memcpy(ionic_queue_at_prod(&aq->q), src + i,  in ionic_admin_poll_locked()
250 					     ionic_queue_at_prod(&aq->q),  in ionic_admin_poll_locked()
251 					     BIT(aq->q.stride_log2), true);  in ionic_admin_poll_locked()
252 			ionic_queue_produce(&aq->q);  in ionic_admin_poll_locked()
254 	} while (!list_empty(&aq->wr_post));  in ionic_admin_poll_locked()
256 	if (old_prod != aq->q.prod)  in ionic_admin_poll_locked()
258 				 ionic_queue_dbell_val(&aq->q));  in ionic_admin_poll_locked()
265 	struct ionic_aq *aq, *bad_aq = NULL;  in ionic_admin_dwork()  local
273 		aq = dev->aq_vec[i];  in ionic_admin_dwork()
275 		spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_dwork()
277 		if (ionic_queue_empty(&aq->q))  in ionic_admin_dwork()
283 		if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN))  in ionic_admin_dwork()
288 		pos = aq->q.cons;  in ionic_admin_dwork()
289 		ionic_admin_poll_locked(aq);  in ionic_admin_dwork()
290 		if (pos != aq->q.cons) {  in ionic_admin_dwork()
292 				  "missed event for acq %d\n", aq->cqid);  in ionic_admin_dwork()
296 		if (time_is_after_eq_jiffies(aq->stamp +  in ionic_admin_dwork()
300 				  (u32)jiffies_to_msecs(jiffies - aq->stamp));  in ionic_admin_dwork()
305 		bad_aq = aq;  in ionic_admin_dwork()
308 		spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_dwork()
321 	struct ionic_aq *aq = container_of(ws, struct ionic_aq, work);  in ionic_admin_work()  local
324 	spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_work()
325 	ionic_admin_poll_locked(aq);  in ionic_admin_work()
326 	spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_work()
329 static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr)  in ionic_admin_post_aq()  argument
335 	wr->aq = aq;  in ionic_admin_post_aq()
337 	spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_post_aq()
338 	poll = list_empty(&aq->wr_post);  in ionic_admin_post_aq()
339 	list_add(&wr->aq_ent, &aq->wr_post);  in ionic_admin_post_aq()
341 		ionic_admin_poll_locked(aq);  in ionic_admin_post_aq()
342 	spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_post_aq()
356 	struct ionic_aq *aq = wr->aq;  in ionic_admin_cancel()  local
359 	spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_cancel()
364 			aq->q_wr[wr->status].wr = NULL;  in ionic_admin_cancel()
367 	spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_cancel()
372 	struct ionic_aq *aq = wr->aq;  in ionic_admin_busy_wait()  local
382 		spin_lock_irqsave(&aq->lock, irqflags);  in ionic_admin_busy_wait()
383 		ionic_admin_poll_locked(aq);  in ionic_admin_busy_wait()
384 		spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_admin_busy_wait()
391 	ionic_admin_timedout(aq);  in ionic_admin_busy_wait()
494 	struct ionic_aq *aq = cq_context;  in ionic_rdma_admincq_comp()  local
497 	spin_lock_irqsave(&aq->lock, irqflags);  in ionic_rdma_admincq_comp()
498 	aq->armed = false;  in ionic_rdma_admincq_comp()
499 	if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED)  in ionic_rdma_admincq_comp()
500 		queue_work(ionic_evt_workq, &aq->work);  in ionic_rdma_admincq_comp()
501 	spin_unlock_irqrestore(&aq->lock, irqflags);  in ionic_rdma_admincq_comp()
506 	struct ionic_aq *aq = cq_context;  in ionic_rdma_admincq_event()  local
508 	ibdev_err(&aq->dev->ibdev, "admincq event %d\n", event->event);  in ionic_rdma_admincq_event()
558 	struct ionic_aq *aq;  in __ionic_create_rdma_adminq()  local
561 	aq = kzalloc_obj(*aq);  in __ionic_create_rdma_adminq()
562 	if (!aq)  in __ionic_create_rdma_adminq()
565 	atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED);  in __ionic_create_rdma_adminq()
566 	aq->dev = dev;  in __ionic_create_rdma_adminq()
567 	aq->aqid = aqid;  in __ionic_create_rdma_adminq()
568 	aq->cqid = cqid;  in __ionic_create_rdma_adminq()
569 	spin_lock_init(&aq->lock);  in __ionic_create_rdma_adminq()
571 	rc = ionic_queue_init(&aq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH,  in __ionic_create_rdma_adminq()
576 	ionic_queue_dbell_init(&aq->q, aq->aqid);  in __ionic_create_rdma_adminq()
578 	aq->q_wr = kzalloc_objs(*aq->q_wr, (u32)aq->q.mask + 1);  in __ionic_create_rdma_adminq()
579 	if (!aq->q_wr) {  in __ionic_create_rdma_adminq()
584 	INIT_LIST_HEAD(&aq->wr_prod);  in __ionic_create_rdma_adminq()
585 	INIT_LIST_HEAD(&aq->wr_post);  in __ionic_create_rdma_adminq()
587 	INIT_WORK(&aq->work, ionic_admin_work);  in __ionic_create_rdma_adminq()
588 	aq->armed = false;  in __ionic_create_rdma_adminq()
590 	return aq;  in __ionic_create_rdma_adminq()
593 	ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev);  in __ionic_create_rdma_adminq()
595 	kfree(aq);  in __ionic_create_rdma_adminq()
601 					struct ionic_aq *aq)  in __ionic_destroy_rdma_adminq()  argument
603 	kfree(aq->q_wr);  in __ionic_destroy_rdma_adminq()
604 	ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev);  in __ionic_destroy_rdma_adminq()
605 	kfree(aq);  in __ionic_destroy_rdma_adminq()
611 	struct ionic_aq *aq;  in ionic_create_rdma_adminq()  local
614 	aq = __ionic_create_rdma_adminq(dev, aqid, cqid);  in ionic_create_rdma_adminq()
615 	if (IS_ERR(aq))  in ionic_create_rdma_adminq()
616 		return aq;  in ionic_create_rdma_adminq()
618 	rc = ionic_rdma_queue_devcmd(dev, &aq->q, aq->aqid, aq->cqid,  in ionic_create_rdma_adminq()
623 	return aq;  in ionic_create_rdma_adminq()
626 	__ionic_destroy_rdma_adminq(dev, aq);  in ionic_create_rdma_adminq()
680 		struct ionic_aq *aq = dev->aq_vec[i];  in ionic_kill_ibdev()  local
682 		spin_lock(&aq->lock);  in ionic_kill_ibdev()
683 		if (atomic_read(&aq->admin_state) != IONIC_ADMIN_KILLED) {  in ionic_kill_ibdev()
684 			atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED);  in ionic_kill_ibdev()
686 			ionic_admin_poll_locked(aq);  in ionic_kill_ibdev()
689 		spin_unlock(&aq->lock);  in ionic_kill_ibdev()
739 		struct ionic_aq *aq = dev->aq_vec[i];  in ionic_kill_rdma_admin()  local
741 		spin_lock(&aq->lock);  in ionic_kill_rdma_admin()
743 		if (atomic_read(&aq->admin_state) == IONIC_ADMIN_ACTIVE)  in ionic_kill_rdma_admin()
744 			atomic_set(&aq->admin_state, IONIC_ADMIN_PAUSED);  in ionic_kill_rdma_admin()
745 		spin_unlock(&aq->lock);  in ionic_kill_rdma_admin()
1069 	struct ionic_aq *aq;  in ionic_create_rdma_admin()  local
1152 		aq = ionic_create_rdma_adminq(dev, aq_i + dev->lif_cfg.aq_base,  in ionic_create_rdma_admin()
1154 		if (IS_ERR(aq)) {  in ionic_create_rdma_admin()
1159 			rc = PTR_ERR(aq);  in ionic_create_rdma_admin()
1163 					  "failed to create aq %pe\n", aq);  in ionic_create_rdma_admin()
1169 				  aq_i, dev->lif_cfg.aq_count, aq);  in ionic_create_rdma_admin()
1173 		vcq->ibcq.cq_context = aq;  in ionic_create_rdma_admin()
1174 		aq->vcq = vcq;  in ionic_create_rdma_admin()
1176 		atomic_set(&aq->admin_state, IONIC_ADMIN_ACTIVE);  in ionic_create_rdma_admin()
1177 		dev->aq_vec[aq_i] = aq;  in ionic_create_rdma_admin()
1191 	struct ionic_aq *aq;  in ionic_destroy_rdma_admin()  local
1204 			aq = dev->aq_vec[--dev->lif_cfg.aq_count];  in ionic_destroy_rdma_admin()
1205 			vcq = aq->vcq;  in ionic_destroy_rdma_admin()
1207 			cancel_work_sync(&aq->work);  in ionic_destroy_rdma_admin()
1209 			__ionic_destroy_rdma_adminq(dev, aq);  in ionic_destroy_rdma_admin()