569 static inline void write_gts(struct t4_cq *cq, u32 val)  in write_gts()  argument
571 	writel(val | V_INGRESSQID(cq->bar2_qid),  in write_gts()
572 		       (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));  in write_gts()
575 static inline int t4_clear_cq_armed(struct t4_cq *cq)  in t4_clear_cq_armed()  argument
577 	return test_and_clear_bit(CQ_ARMED, &cq->flags);  in t4_clear_cq_armed()
580 static inline int t4_arm_cq(struct t4_cq *cq, int se)  in t4_arm_cq()  argument
584 	set_bit(CQ_ARMED, &cq->flags);  in t4_arm_cq()
585 	while (cq->cidx_inc > CIDXINC_MASK) {  in t4_arm_cq()
587 		writel(val | V_INGRESSQID(cq->bar2_qid),  in t4_arm_cq()
588 		       (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));  in t4_arm_cq()
589 		cq->cidx_inc -= CIDXINC_MASK;  in t4_arm_cq()
591 	val = SEINTARM(se) | CIDXINC(cq->cidx_inc) | TIMERREG(6);  in t4_arm_cq()
592 	writel(val | V_INGRESSQID(cq->bar2_qid),  in t4_arm_cq()
593 		       (void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));  in t4_arm_cq()
594 	cq->cidx_inc = 0;  in t4_arm_cq()
598 static inline void t4_swcq_produce(struct t4_cq *cq)  in t4_swcq_produce()  argument
600 	cq->sw_in_use++;  in t4_swcq_produce()
601 	if (cq->sw_in_use == cq->size) {  in t4_swcq_produce()
603 			 __func__, cq->cqid);  in t4_swcq_produce()
604 		cq->error = 1;  in t4_swcq_produce()
607 	if (++cq->sw_pidx == cq->size)  in t4_swcq_produce()
608 		cq->sw_pidx = 0;  in t4_swcq_produce()
611 static inline void t4_swcq_consume(struct t4_cq *cq)  in t4_swcq_consume()  argument
613 	BUG_ON(cq->sw_in_use < 1);  in t4_swcq_consume()
614 	cq->sw_in_use--;  in t4_swcq_consume()
615 	if (++cq->sw_cidx == cq->size)  in t4_swcq_consume()
616 		cq->sw_cidx = 0;  in t4_swcq_consume()
619 static inline void t4_hwcq_consume(struct t4_cq *cq)  in t4_hwcq_consume()  argument
621 	cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts;  in t4_hwcq_consume()
622 	if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == M_CIDXINC) {  in t4_hwcq_consume()
625 		val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7);  in t4_hwcq_consume()
626 		write_gts(cq, val);  in t4_hwcq_consume()
627 		cq->cidx_inc = 0;  in t4_hwcq_consume()
629 	if (++cq->cidx == cq->size) {  in t4_hwcq_consume()
630 		cq->cidx = 0;  in t4_hwcq_consume()
631 		cq->gen ^= 1;  in t4_hwcq_consume()
635 static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)  in t4_valid_cqe()  argument
637 	return (CQE_GENBIT(cqe) == cq->gen);  in t4_valid_cqe()
640 static inline int t4_cq_notempty(struct t4_cq *cq)  in t4_cq_notempty()  argument
642 	return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);  in t4_cq_notempty()
645 static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)  in t4_next_hw_cqe()  argument
650 	if (cq->cidx == 0)  in t4_next_hw_cqe()
651 		prev_cidx = cq->size - 1;  in t4_next_hw_cqe()
653 		prev_cidx = cq->cidx - 1;  in t4_next_hw_cqe()
655 	if (cq->queue[prev_cidx].bits_type_ts != cq->bits_type_ts) {  in t4_next_hw_cqe()
657 		cq->error = 1;  in t4_next_hw_cqe()
658 		printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);  in t4_next_hw_cqe()
660 	} else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {  in t4_next_hw_cqe()
664 		*cqe = &cq->queue[cq->cidx];  in t4_next_hw_cqe()
671 static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq)  in t4_next_sw_cqe()  argument
673 	if (cq->sw_in_use == cq->size) {  in t4_next_sw_cqe()
675 			 __func__, cq->cqid);  in t4_next_sw_cqe()
676 		cq->error = 1;  in t4_next_sw_cqe()
680 	if (cq->sw_in_use)  in t4_next_sw_cqe()
681 		return &cq->sw_queue[cq->sw_cidx];  in t4_next_sw_cqe()
685 static inline int t4_next_cqe(struct t4_cq *cq, struct t4_cqe **cqe)  in t4_next_cqe()  argument
689 	if (cq->error)  in t4_next_cqe()
691 	else if (cq->sw_in_use)  in t4_next_cqe()
692 		*cqe = &cq->sw_queue[cq->sw_cidx];  in t4_next_cqe()
694 		ret = t4_next_hw_cqe(cq, cqe);  in t4_next_cqe()
698 static inline int t4_cq_in_error(struct t4_cq *cq)  in t4_cq_in_error()  argument
700 	return ((struct t4_status_page *)&cq->queue[cq->size])->qp_err;  in t4_cq_in_error()
703 static inline void t4_set_cq_in_error(struct t4_cq *cq)  in t4_set_cq_in_error()  argument
705 	((struct t4_status_page *)&cq->queue[cq->size])->qp_err = 1;  in t4_set_cq_in_error()