49 int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,  in __check_cmdq_stall()  argument
54 	cmdq = &rcfw->cmdq;  in __check_cmdq_stall()
98 static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie)  in bnxt_re_is_fw_stalled()  argument
103 	crsqe = &rcfw->crsqe_tbl[cookie];  in bnxt_re_is_fw_stalled()
104 	cmdq = &rcfw->cmdq;  in bnxt_re_is_fw_stalled()
107 		      (rcfw->max_timeout * HZ))) {  in bnxt_re_is_fw_stalled()
108 		dev_warn_ratelimited(&rcfw->pdev->dev,  in bnxt_re_is_fw_stalled()
112 				     rcfw->max_timeout * 1000,  in bnxt_re_is_fw_stalled()
130 static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)  in __wait_for_resp()  argument
137 	cmdq = &rcfw->cmdq;  in __wait_for_resp()
139 	crsqe = &rcfw->crsqe_tbl[cookie];  in __wait_for_resp()
142 		if (RCFW_NO_FW_ACCESS(rcfw))  in __wait_for_resp()
150 					 RCFW_NO_FW_ACCESS(rcfw),  in __wait_for_resp()
151 					 msecs_to_jiffies(rcfw->max_timeout * 1000));  in __wait_for_resp()
158 		bnxt_qplib_service_creq((unsigned long)rcfw);  in __wait_for_resp()
159 		dev_warn_ratelimited(&rcfw->pdev->dev,  in __wait_for_resp()
168 		ret = bnxt_re_is_fw_stalled(rcfw, cookie);  in __wait_for_resp()
187 static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)  in __block_for_resp()  argument
189 	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;  in __block_for_resp()
194 	crsqe = &rcfw->crsqe_tbl[cookie];  in __block_for_resp()
197 		if (RCFW_NO_FW_ACCESS(rcfw))  in __block_for_resp()
207 		bnxt_qplib_service_creq((unsigned long)rcfw);  in __block_for_resp()
213 	dev_warn_ratelimited(&rcfw->pdev->dev,  in __block_for_resp()
238 static  void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw,  in __send_message_no_waiter()  argument
241 	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;  in __send_message_no_waiter()
251 	crsqe = &rcfw->crsqe_tbl[cookie];  in __send_message_no_waiter()
279 	atomic_inc(&rcfw->timeout_send);  in __send_message_no_waiter()
286 static int __send_message(struct bnxt_qplib_rcfw *rcfw,  in __send_message()  argument
301 	cmdq = &rcfw->cmdq;  in __send_message()
303 	pdev = rcfw->pdev;  in __send_message()
312 	crsqe = &rcfw->crsqe_tbl[cookie];  in __send_message()
318 		rcfw->cmdq_full_dbg++;  in __send_message()
411 static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)  in __poll_for_resp()  argument
413 	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;  in __poll_for_resp()
419 	crsqe = &rcfw->crsqe_tbl[cookie];  in __poll_for_resp()
422 		if (RCFW_NO_FW_ACCESS(rcfw))  in __poll_for_resp()
429 		bnxt_qplib_service_creq((unsigned long)rcfw);  in __poll_for_resp()
434 		    (rcfw->max_timeout * 1000)) {  in __poll_for_resp()
435 			dev_warn_ratelimited(&rcfw->pdev->dev,  in __poll_for_resp()
439 			ret = bnxt_re_is_fw_stalled(rcfw, cookie);  in __poll_for_resp()
447 static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw,  in __send_message_basic_sanity()  argument
452 	cmdq = &rcfw->cmdq;  in __send_message_basic_sanity()
455 	if (RCFW_NO_FW_ACCESS(rcfw))  in __send_message_basic_sanity()
463 		dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n");  in __send_message_basic_sanity()
471 		dev_err(&rcfw->pdev->dev,  in __send_message_basic_sanity()
481 static  void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw,  in __destroy_timedout_ah()  argument
492 	__send_message_no_waiter(rcfw, &msg);  in __destroy_timedout_ah()
493 	dev_warn_ratelimited(&rcfw->pdev->dev,  in __destroy_timedout_ah()
496 		atomic_read(&rcfw->timeout_send));  in __destroy_timedout_ah()
512 static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,  in __bnxt_qplib_rcfw_send_message()  argument
524 	rc = __send_message_basic_sanity(rcfw, msg, opcode);  in __bnxt_qplib_rcfw_send_message()
528 	rc = __send_message(rcfw, msg);  in __bnxt_qplib_rcfw_send_message()
537 		rc = __block_for_resp(rcfw, cookie);  in __bnxt_qplib_rcfw_send_message()
538 	else if (atomic_read(&rcfw->rcfw_intr_enabled))  in __bnxt_qplib_rcfw_send_message()
539 		rc = __wait_for_resp(rcfw, cookie);  in __bnxt_qplib_rcfw_send_message()
541 		rc = __poll_for_resp(rcfw, cookie);  in __bnxt_qplib_rcfw_send_message()
547 		spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags);  in __bnxt_qplib_rcfw_send_message()
548 		crsqe = &rcfw->crsqe_tbl[cookie];  in __bnxt_qplib_rcfw_send_message()
551 			set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags);  in __bnxt_qplib_rcfw_send_message()
552 		spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags);  in __bnxt_qplib_rcfw_send_message()
560 		dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n",  in __bnxt_qplib_rcfw_send_message()
604 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,  in bnxt_qplib_rcfw_send_message()  argument
610 		down(&rcfw->rcfw_inflight);  in bnxt_qplib_rcfw_send_message()
611 		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);  in bnxt_qplib_rcfw_send_message()
612 		up(&rcfw->rcfw_inflight);  in bnxt_qplib_rcfw_send_message()
614 		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);  in bnxt_qplib_rcfw_send_message()
620 static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw,  in bnxt_re_add_perf_stats()  argument
626 	latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen -  in bnxt_re_add_perf_stats()
629 		rcfw->rcfw_lat_slab_sec[latency_msec/1000]++;  in bnxt_re_add_perf_stats()
631 	if (!rcfw->sp_perf_stats_enabled)  in bnxt_re_add_perf_stats()
635 		rcfw->rcfw_lat_slab_msec[latency_msec]++;  in bnxt_re_add_perf_stats()
639 		dest_stats_id = rcfw->qp_create_stats_id++;  in bnxt_re_add_perf_stats()
641 		dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id];  in bnxt_re_add_perf_stats()
644 		dest_stats_id = rcfw->qp_destroy_stats_id++;  in bnxt_re_add_perf_stats()
646 		dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id];  in bnxt_re_add_perf_stats()
649 		dest_stats_id = rcfw->mr_create_stats_id++;  in bnxt_re_add_perf_stats()
651 		dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id];  in bnxt_re_add_perf_stats()
655 		dest_stats_id = rcfw->mr_destroy_stats_id++;  in bnxt_re_add_perf_stats()
657 		dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id];  in bnxt_re_add_perf_stats()
662 		dest_stats_id = rcfw->qp_modify_stats_id++;  in bnxt_re_add_perf_stats()
664 		dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id];  in bnxt_re_add_perf_stats()
671 				(rcfw->cmdq.last_seen - crsqe->send_timestamp), 1);  in bnxt_re_add_perf_stats()
676 static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,  in bnxt_qplib_process_qp_event()  argument
680 	struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq;  in bnxt_qplib_process_qp_event()
696 	pdev = rcfw->pdev;  in bnxt_qplib_process_qp_event()
699 		tbl = &rcfw->res->reftbl.qpref;  in bnxt_qplib_process_qp_event()
710 		rc = rcfw->creq.aeq_handler(rcfw, event, qp);  in bnxt_qplib_process_qp_event()
723 		tbl = &rcfw->res->reftbl.cqref;  in bnxt_qplib_process_qp_event()
732 		rc = rcfw->creq.aeq_handler(rcfw, event, cq);  in bnxt_qplib_process_qp_event()
751 		crsqe = &rcfw->crsqe_tbl[cookie];  in bnxt_qplib_process_qp_event()
753 		bnxt_re_add_perf_stats(rcfw, crsqe);  in bnxt_qplib_process_qp_event()
756 				       &rcfw->cmdq.flags),  in bnxt_qplib_process_qp_event()
760 				(long)jiffies_to_msecs(rcfw->cmdq.last_seen -  in bnxt_qplib_process_qp_event()
768 			atomic_dec(&rcfw->timeout_send);  in bnxt_qplib_process_qp_event()
804 			__destroy_timedout_ah(rcfw,  in bnxt_qplib_process_qp_event()
817 	struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;  in bnxt_qplib_service_creq()  local
818 	struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;  in bnxt_qplib_service_creq()
828 	pdev = rcfw->pdev;  in bnxt_qplib_service_creq()
829 	res = rcfw->res;  in bnxt_qplib_service_creq()
833 		if (RCFW_NO_FW_ACCESS(rcfw)) {  in bnxt_qplib_service_creq()
845 		rcfw->cmdq.last_seen = jiffies;  in bnxt_qplib_service_creq()
850 				(rcfw,(struct creq_qp_event *)creqe,  in bnxt_qplib_service_creq()
855 			rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL);  in bnxt_qplib_service_creq()
897 		wake_up_all(&rcfw->cmdq.waitq);  in bnxt_qplib_service_creq()
902 	struct bnxt_qplib_rcfw *rcfw = dev_instance;  in bnxt_qplib_creq_irq()  local
904 	bnxt_qplib_service_creq((unsigned long)rcfw);  in bnxt_qplib_creq_irq()
909 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)  in bnxt_qplib_deinit_rcfw()  argument
920 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_deinit_rcfw()
923 	clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);  in bnxt_qplib_deinit_rcfw()
927 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn)  in bnxt_qplib_init_rcfw()  argument
938 	res = rcfw->res;  in bnxt_qplib_init_rcfw()
1001 	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);  in bnxt_qplib_init_rcfw()
1004 	set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);  in bnxt_qplib_init_rcfw()
1011 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_free_rcfw_channel()  local
1013 	vfree(rcfw->rcfw_lat_slab_msec);  in bnxt_qplib_free_rcfw_channel()
1014 	rcfw->rcfw_lat_slab_msec = NULL;  in bnxt_qplib_free_rcfw_channel()
1015 	vfree(rcfw->qp_create_stats);  in bnxt_qplib_free_rcfw_channel()
1016 	rcfw->qp_create_stats = NULL;  in bnxt_qplib_free_rcfw_channel()
1017 	vfree(rcfw->qp_destroy_stats);  in bnxt_qplib_free_rcfw_channel()
1018 	rcfw->qp_destroy_stats = NULL;  in bnxt_qplib_free_rcfw_channel()
1019 	vfree(rcfw->mr_create_stats);  in bnxt_qplib_free_rcfw_channel()
1020 	rcfw->mr_create_stats = NULL;  in bnxt_qplib_free_rcfw_channel()
1021 	vfree(rcfw->mr_destroy_stats);  in bnxt_qplib_free_rcfw_channel()
1022 	rcfw->mr_destroy_stats = NULL;  in bnxt_qplib_free_rcfw_channel()
1023 	vfree(rcfw->qp_modify_stats);  in bnxt_qplib_free_rcfw_channel()
1024 	rcfw->qp_modify_stats = NULL;  in bnxt_qplib_free_rcfw_channel()
1025 	rcfw->sp_perf_stats_enabled = false;  in bnxt_qplib_free_rcfw_channel()
1027 	kfree(rcfw->crsqe_tbl);  in bnxt_qplib_free_rcfw_channel()
1028 	rcfw->crsqe_tbl = NULL;  in bnxt_qplib_free_rcfw_channel()
1030 	bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);  in bnxt_qplib_free_rcfw_channel()
1031 	bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);  in bnxt_qplib_free_rcfw_channel()
1032 	rcfw->pdev = NULL;  in bnxt_qplib_free_rcfw_channel()
1038 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;  in bnxt_qplib_alloc_rcfw_channel()  local
1043 	rcfw->pdev = res->pdev;  in bnxt_qplib_alloc_rcfw_channel()
1044 	rcfw->res = res;  in bnxt_qplib_alloc_rcfw_channel()
1045 	cmdq = &rcfw->cmdq;  in bnxt_qplib_alloc_rcfw_channel()
1046 	creq = &rcfw->creq;  in bnxt_qplib_alloc_rcfw_channel()
1052 	hwq_attr.res = rcfw->res;  in bnxt_qplib_alloc_rcfw_channel()
1058 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_alloc_rcfw_channel()
1068 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_alloc_rcfw_channel()
1073 	rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements,  in bnxt_qplib_alloc_rcfw_channel()
1074 			sizeof(*rcfw->crsqe_tbl), GFP_KERNEL);  in bnxt_qplib_alloc_rcfw_channel()
1075 	if (!rcfw->crsqe_tbl) {  in bnxt_qplib_alloc_rcfw_channel()
1076 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_alloc_rcfw_channel()
1081 	rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout;  in bnxt_qplib_alloc_rcfw_channel()
1083 	rcfw->sp_perf_stats_enabled = false;  in bnxt_qplib_alloc_rcfw_channel()
1084 	rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) *  in bnxt_qplib_alloc_rcfw_channel()
1086 	rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);  in bnxt_qplib_alloc_rcfw_channel()
1087 	rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);  in bnxt_qplib_alloc_rcfw_channel()
1088 	rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);  in bnxt_qplib_alloc_rcfw_channel()
1089 	rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);  in bnxt_qplib_alloc_rcfw_channel()
1090 	rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);  in bnxt_qplib_alloc_rcfw_channel()
1092 	if (rcfw->rcfw_lat_slab_msec &&  in bnxt_qplib_alloc_rcfw_channel()
1093 	    rcfw->qp_create_stats &&  in bnxt_qplib_alloc_rcfw_channel()
1094 	    rcfw->qp_destroy_stats &&  in bnxt_qplib_alloc_rcfw_channel()
1095 	    rcfw->mr_create_stats &&  in bnxt_qplib_alloc_rcfw_channel()
1096 	    rcfw->mr_destroy_stats &&  in bnxt_qplib_alloc_rcfw_channel()
1097 	    rcfw->qp_modify_stats)  in bnxt_qplib_alloc_rcfw_channel()
1098 		rcfw->sp_perf_stats_enabled = true;  in bnxt_qplib_alloc_rcfw_channel()
1102 	bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);  in bnxt_qplib_alloc_rcfw_channel()
1104 	bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);  in bnxt_qplib_alloc_rcfw_channel()
1108 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)  in bnxt_qplib_rcfw_stop_irq()  argument
1113 	creq = &rcfw->creq;  in bnxt_qplib_rcfw_stop_irq()
1114 	res = rcfw->res;  in bnxt_qplib_rcfw_stop_irq()
1124 	free_irq(creq->msix_vec, rcfw);  in bnxt_qplib_rcfw_stop_irq()
1130 	if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) {  in bnxt_qplib_rcfw_stop_irq()
1131 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_rcfw_stop_irq()
1133 			atomic_read(&rcfw->rcfw_intr_enabled));  in bnxt_qplib_rcfw_stop_irq()
1135 	atomic_set(&rcfw->rcfw_intr_enabled, 0);  in bnxt_qplib_rcfw_stop_irq()
1136 	rcfw->num_irq_stopped++;  in bnxt_qplib_rcfw_stop_irq()
1139 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)  in bnxt_qplib_disable_rcfw_channel()  argument
1144 	creq = &rcfw->creq;  in bnxt_qplib_disable_rcfw_channel()
1145 	cmdq = &rcfw->cmdq;  in bnxt_qplib_disable_rcfw_channel()
1147 	bnxt_qplib_rcfw_stop_irq(rcfw, true);  in bnxt_qplib_disable_rcfw_channel()
1163 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,  in bnxt_qplib_rcfw_start_irq()  argument
1170 	creq = &rcfw->creq;  in bnxt_qplib_rcfw_start_irq()
1171 	res = rcfw->res;  in bnxt_qplib_rcfw_start_irq()
1184 			 creq->irq_name, rcfw);  in bnxt_qplib_rcfw_start_irq()
1194 	rcfw->num_irq_started++;  in bnxt_qplib_rcfw_start_irq()
1198 	if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) {  in bnxt_qplib_rcfw_start_irq()
1199 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_rcfw_start_irq()
1201 			atomic_read(&rcfw->rcfw_intr_enabled));  in bnxt_qplib_rcfw_start_irq()
1203 	atomic_inc(&rcfw->rcfw_intr_enabled);  in bnxt_qplib_rcfw_start_irq()
1207 static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw)  in bnxt_qplib_map_cmdq_mbox()  argument
1213 	pdev = rcfw->pdev;  in bnxt_qplib_map_cmdq_mbox()
1214 	mbox = &rcfw->cmdq.cmdq_mbox;  in bnxt_qplib_map_cmdq_mbox()
1243 static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)  in bnxt_qplib_map_creq_db()  argument
1249 	res = rcfw->res;  in bnxt_qplib_map_creq_db()
1250 	creq_db = &rcfw->creq.creq_db;  in bnxt_qplib_map_creq_db()
1260 	creq_db->dbinfo.hwq = &rcfw->creq.hwq;  in bnxt_qplib_map_creq_db()
1261 	creq_db->dbinfo.xid = rcfw->creq.ring_id;  in bnxt_qplib_map_creq_db()
1262 	creq_db->dbinfo.seed = rcfw->creq.ring_id;  in bnxt_qplib_map_creq_db()
1266 	creq_db->dbinfo.res = rcfw->res;  in bnxt_qplib_map_creq_db()
1271 static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)  in bnxt_qplib_start_rcfw()  argument
1278 	cmdq = &rcfw->cmdq;  in bnxt_qplib_start_rcfw()
1279 	creq = &rcfw->creq;  in bnxt_qplib_start_rcfw()
1293 int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,  in bnxt_qplib_enable_rcfw_channel()  argument
1302 	cmdq = &rcfw->cmdq;  in bnxt_qplib_enable_rcfw_channel()
1303 	creq = &rcfw->creq;  in bnxt_qplib_enable_rcfw_channel()
1314 	rc = bnxt_qplib_map_cmdq_mbox(rcfw);  in bnxt_qplib_enable_rcfw_channel()
1318 	rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);  in bnxt_qplib_enable_rcfw_channel()
1322 	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);  in bnxt_qplib_enable_rcfw_channel()
1324 		dev_err(&rcfw->pdev->dev,  in bnxt_qplib_enable_rcfw_channel()
1326 		bnxt_qplib_disable_rcfw_channel(rcfw);  in bnxt_qplib_enable_rcfw_channel()
1330 	rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd,  in bnxt_qplib_enable_rcfw_channel()
1332 	sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd);  in bnxt_qplib_enable_rcfw_channel()
1333 	dev_dbg(&rcfw->pdev->dev,  in bnxt_qplib_enable_rcfw_channel()
1334 		"Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd);  in bnxt_qplib_enable_rcfw_channel()
1335 	bnxt_qplib_start_rcfw(rcfw);  in bnxt_qplib_enable_rcfw_channel()