71 static void smc_ibdev_cnt_inc(struct smc_link *lnk)  in smc_ibdev_cnt_inc()  argument
73 	atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_inc()
76 static void smc_ibdev_cnt_dec(struct smc_link *lnk)  in smc_ibdev_cnt_dec()  argument
78 	atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);  in smc_ibdev_cnt_dec()
130 	conn->lnk = NULL;	/* reset conn->lnk first */  in smcr_lgr_conn_assign_link()
132 		struct smc_link *lnk = &conn->lgr->lnk[i];  in smcr_lgr_conn_assign_link()  local
134 		if (lnk->state != expected || lnk->link_is_asym)  in smcr_lgr_conn_assign_link()
137 			conn->lnk = lnk; /* temporary, SMC server assigns link*/  in smcr_lgr_conn_assign_link()
144 				lnk2 = &conn->lgr->lnk[j];  in smcr_lgr_conn_assign_link()
147 					conn->lnk = lnk2;  in smcr_lgr_conn_assign_link()
152 		if (!conn->lnk)  in smcr_lgr_conn_assign_link()
153 			conn->lnk = lnk;  in smcr_lgr_conn_assign_link()
156 	if (!conn->lnk)  in smcr_lgr_conn_assign_link()
158 	atomic_inc(&conn->lnk->conn_cnt);  in smcr_lgr_conn_assign_link()
202 	if (conn->lnk)  in __smc_lgr_unregister_conn()
203 		atomic_dec(&conn->lnk->conn_cnt);  in __smc_lgr_unregister_conn()
498 		if (!smc_link_usable(&lgr->lnk[i]))  in smc_nl_handle_lgr()
500 		if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb))  in smc_nl_handle_lgr()
703 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_lgr_link_deactivate_all()  local
705 		if (smc_link_sendable(lnk))  in smcr_lgr_link_deactivate_all()
706 			lnk->state = SMC_LNK_INACTIVE;  in smcr_lgr_link_deactivate_all()
770 			if (smc_link_usable(&lgr->lnk[i]) &&  in smcr_next_link_id()
771 			    lgr->lnk[i].link_id == link_id)  in smcr_next_link_id()
788 int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,  in smcr_link_init()  argument
796 		lnk->smcibdev = ini->smcrv2.ib_dev_v2;  in smcr_link_init()
797 		lnk->ibport = ini->smcrv2.ib_port_v2;  in smcr_link_init()
799 		lnk->smcibdev = ini->ib_dev;  in smcr_link_init()
800 		lnk->ibport = ini->ib_port;  in smcr_link_init()
802 	get_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
803 	atomic_inc(&lnk->smcibdev->lnk_cnt);  in smcr_link_init()
804 	refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */  in smcr_link_init()
805 	lnk->clearing = 0;  in smcr_link_init()
806 	lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu;  in smcr_link_init()
807 	lnk->link_id = smcr_next_link_id(lgr);  in smcr_link_init()
808 	lnk->lgr = lgr;  in smcr_link_init()
810 	lnk->link_idx = link_idx;  in smcr_link_init()
811 	lnk->wr_rx_id_compl = 0;  in smcr_link_init()
812 	smc_ibdev_cnt_inc(lnk);  in smcr_link_init()
813 	smcr_copy_dev_info_to_link(lnk);  in smcr_link_init()
814 	atomic_set(&lnk->conn_cnt, 0);  in smcr_link_init()
815 	smc_llc_link_set_uid(lnk);  in smcr_link_init()
816 	INIT_WORK(&lnk->link_down_wrk, smc_link_down_work);  in smcr_link_init()
817 	if (!lnk->smcibdev->initialized) {  in smcr_link_init()
818 		rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev);  in smcr_link_init()
823 	lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +  in smcr_link_init()
825 	rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,  in smcr_link_init()
826 				  ini->vlan_id, lnk->gid, &lnk->sgid_index,  in smcr_link_init()
831 	rc = smc_llc_link_init(lnk);  in smcr_link_init()
834 	rc = smc_wr_alloc_link_mem(lnk);  in smcr_link_init()
837 	rc = smc_ib_create_protection_domain(lnk);  in smcr_link_init()
840 	rc = smc_ib_create_queue_pair(lnk);  in smcr_link_init()
843 	rc = smc_wr_create_link(lnk);  in smcr_link_init()
846 	lnk->state = SMC_LNK_ACTIVATING;  in smcr_link_init()
850 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_init()
852 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_init()
854 	smc_wr_free_link_mem(lnk);  in smcr_link_init()
856 	smc_llc_link_clear(lnk, false);  in smcr_link_init()
858 	smc_ibdev_cnt_dec(lnk);  in smcr_link_init()
859 	put_device(&lnk->smcibdev->ibdev->dev);  in smcr_link_init()
860 	smcibdev = lnk->smcibdev;  in smcr_link_init()
861 	memset(lnk, 0, sizeof(struct smc_link));  in smcr_link_init()
862 	lnk->state = SMC_LNK_UNUSED;  in smcr_link_init()
875 	struct smc_link *lnk;  in smc_lgr_create()  local
965 		lnk = &lgr->lnk[link_idx];  in smc_lgr_create()
966 		rc = smcr_link_init(lgr, lnk, link_idx, ini);  in smc_lgr_create()
971 		lgr->net = smc_ib_net(lnk->smcibdev);  in smc_lgr_create()
1058 		smc_wr_tx_put_slot(conn->lnk,  in smc_switch_cursor()
1067 	atomic_dec(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1069 	smcr_link_put(conn->lnk);  in smc_switch_link_and_count()
1070 	conn->lnk = to_lnk;  in smc_switch_link_and_count()
1071 	atomic_inc(&conn->lnk->conn_cnt);  in smc_switch_link_and_count()
1073 	smcr_link_hold(conn->lnk);  in smc_switch_link_and_count()
1091 		if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx)  in smc_switch_conns()
1093 		if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev &&  in smc_switch_conns()
1094 		    from_lnk->ibport == lgr->lnk[i].ibport) {  in smc_switch_conns()
1097 		to_lnk = &lgr->lnk[i];  in smc_switch_conns()
1108 		if (conn->lnk != from_lnk)  in smc_switch_conns()
1111 		/* conn->lnk not yet set in SMC_INIT state */  in smc_switch_conns()
1271 		smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */  in smc_conn_free()
1277 				struct smc_link *lnk)  in smcr_buf_unmap_link()  argument
1280 		buf_desc->is_reg_mr[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1281 	if (!buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_unmap_link()
1285 	    buf_desc->mr[lnk->link_idx]) {  in smcr_buf_unmap_link()
1286 		smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]);  in smcr_buf_unmap_link()
1287 		buf_desc->mr[lnk->link_idx] = NULL;  in smcr_buf_unmap_link()
1290 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE);  in smcr_buf_unmap_link()
1292 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE);  in smcr_buf_unmap_link()
1294 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_unmap_link()
1295 	buf_desc->is_map_ib[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1299 static void smcr_buf_unmap_lgr(struct smc_link *lnk)  in smcr_buf_unmap_lgr()  argument
1301 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_unmap_lgr()
1308 			smcr_buf_unmap_link(buf_desc, true, lnk);  in smcr_buf_unmap_lgr()
1314 			smcr_buf_unmap_link(buf_desc, false, lnk);  in smcr_buf_unmap_lgr()
1319 static void smcr_rtoken_clear_link(struct smc_link *lnk)  in smcr_rtoken_clear_link()  argument
1321 	struct smc_link_group *lgr = lnk->lgr;  in smcr_rtoken_clear_link()
1325 		lgr->rtokens[i][lnk->link_idx].rkey = 0;  in smcr_rtoken_clear_link()
1326 		lgr->rtokens[i][lnk->link_idx].dma_addr = 0;  in smcr_rtoken_clear_link()
1330 static void __smcr_link_clear(struct smc_link *lnk)  in __smcr_link_clear()  argument
1332 	struct smc_link_group *lgr = lnk->lgr;  in __smcr_link_clear()
1335 	smc_wr_free_link_mem(lnk);  in __smcr_link_clear()
1336 	smc_ibdev_cnt_dec(lnk);  in __smcr_link_clear()
1337 	put_device(&lnk->smcibdev->ibdev->dev);  in __smcr_link_clear()
1338 	smcibdev = lnk->smcibdev;  in __smcr_link_clear()
1339 	memset(lnk, 0, sizeof(struct smc_link));  in __smcr_link_clear()
1340 	lnk->state = SMC_LNK_UNUSED;  in __smcr_link_clear()
1347 void smcr_link_clear(struct smc_link *lnk, bool log)  in smcr_link_clear()  argument
1349 	if (!lnk->lgr || lnk->clearing ||  in smcr_link_clear()
1350 	    lnk->state == SMC_LNK_UNUSED)  in smcr_link_clear()
1352 	lnk->clearing = 1;  in smcr_link_clear()
1353 	lnk->peer_qpn = 0;  in smcr_link_clear()
1354 	smc_llc_link_clear(lnk, log);  in smcr_link_clear()
1355 	smcr_buf_unmap_lgr(lnk);  in smcr_link_clear()
1356 	smcr_rtoken_clear_link(lnk);  in smcr_link_clear()
1357 	smc_ib_modify_qp_error(lnk);  in smcr_link_clear()
1358 	smc_wr_free_link(lnk);  in smcr_link_clear()
1359 	smc_ib_destroy_queue_pair(lnk);  in smcr_link_clear()
1360 	smc_ib_dealloc_protection_domain(lnk);  in smcr_link_clear()
1361 	smcr_link_put(lnk); /* theoretically last link_put */  in smcr_link_clear()
1364 void smcr_link_hold(struct smc_link *lnk)  in smcr_link_hold()  argument
1366 	refcount_inc(&lnk->refcnt);  in smcr_link_hold()
1369 void smcr_link_put(struct smc_link *lnk)  in smcr_link_put()  argument
1371 	if (refcount_dec_and_test(&lnk->refcnt))  in smcr_link_put()
1372 		__smcr_link_clear(lnk);  in smcr_link_put()
1381 		smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]);  in smcr_buf_free()
1462 			if (lgr->lnk[i].state != SMC_LNK_UNUSED)  in smc_lgr_free()
1463 				smcr_link_clear(&lgr->lnk[i], false);  in smc_lgr_free()
1658 				if (lgr->lnk[i].smcibdev == smcibdev)  in smc_smcr_terminate_all()
1659 					smcr_link_down_cond_sched(&lgr->lnk[i]);  in smc_smcr_terminate_all()
1688 		if (smc_link_usable(&lgr->lnk[i]))  in smcr_lgr_set_type()
1689 			lgr->lnk[i].link_is_asym = false;  in smcr_lgr_set_type()
1721 	lgr->lnk[asym_lnk_idx].link_is_asym = true;  in smcr_lgr_set_type_asym()
1767 static void smcr_link_down(struct smc_link *lnk)  in smcr_link_down()  argument
1769 	struct smc_link_group *lgr = lnk->lgr;  in smcr_link_down()
1773 	if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))  in smcr_link_down()
1776 	to_lnk = smc_switch_conns(lgr, lnk, true);  in smcr_link_down()
1778 		smcr_link_clear(lnk, true);  in smcr_link_down()
1782 	del_link_id = lnk->link_id;  in smcr_link_down()
1801 			smcr_link_clear(lnk, true);  in smcr_link_down()
1808 void smcr_link_down_cond(struct smc_link *lnk)  in smcr_link_down_cond()  argument
1810 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond()
1811 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond()
1812 		smcr_link_down(lnk);  in smcr_link_down_cond()
1817 void smcr_link_down_cond_sched(struct smc_link *lnk)  in smcr_link_down_cond_sched()  argument
1819 	if (smc_link_downing(&lnk->state)) {  in smcr_link_down_cond_sched()
1820 		trace_smcr_link_down(lnk, __builtin_return_address(0));  in smcr_link_down_cond_sched()
1821 		schedule_work(&lnk->link_down_wrk);  in smcr_link_down_cond_sched()
1837 			struct smc_link *lnk = &lgr->lnk[i];  in smcr_port_err()  local
1839 			if (smc_link_usable(lnk) &&  in smcr_port_err()
1840 			    lnk->smcibdev == smcibdev && lnk->ibport == ibport)  in smcr_port_err()
1841 				smcr_link_down_cond_sched(lnk);  in smcr_port_err()
1915 	struct smc_link *lnk;  in smcr_lgr_match()  local
1923 		lnk = &lgr->lnk[i];  in smcr_lgr_match()
1925 		if (!smc_link_active(lnk))  in smcr_lgr_match()
1928 		if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net))  in smcr_lgr_match()
1930 		if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) &&  in smcr_lgr_match()
1931 		    !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) &&  in smcr_lgr_match()
1933 		     !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN)))  in smcr_lgr_match()
2032 		smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */  in smc_conn_create()
2121 			     struct smc_link *lnk)  in smcr_buf_map_link()  argument
2127 	if (buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_map_link()
2139 	rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL);  in smcr_buf_map_link()
2145 		for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) {  in smcr_buf_map_link()
2154 		sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl,  in smcr_buf_map_link()
2159 	rc = smc_ib_buf_map_sg(lnk, buf_desc,  in smcr_buf_map_link()
2168 		smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx;  in smcr_buf_map_link()
2176 		rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags,  in smcr_buf_map_link()
2177 					      buf_desc, lnk->link_idx);  in smcr_buf_map_link()
2180 		smc_ib_sync_sg_for_device(lnk, buf_desc,  in smcr_buf_map_link()
2183 	buf_desc->is_map_ib[lnk->link_idx] = true;  in smcr_buf_map_link()
2187 	smc_ib_buf_unmap_sg(lnk, buf_desc,  in smcr_buf_map_link()
2190 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_map_link()
2215 static int _smcr_buf_map_lgr(struct smc_link *lnk, struct rw_semaphore *lock,  in _smcr_buf_map_lgr()  argument
2225 		rc = smcr_buf_map_link(buf_desc, is_rmb, lnk);  in _smcr_buf_map_lgr()
2235 int smcr_buf_map_lgr(struct smc_link *lnk)  in smcr_buf_map_lgr()  argument
2237 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_map_lgr()
2241 		rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock,  in smcr_buf_map_lgr()
2245 		rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock,  in smcr_buf_map_lgr()
2256 int smcr_buf_reg_lgr(struct smc_link *lnk)  in smcr_buf_reg_lgr()  argument
2258 	struct smc_link_group *lgr = lnk->lgr;  in smcr_buf_reg_lgr()
2268 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2286 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2353 		struct smc_link *lnk = &lgr->lnk[i];  in smcr_buf_map_usable_links()  local
2355 		if (!smc_link_usable(lnk))  in smcr_buf_map_usable_links()
2357 		if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) {  in smcr_buf_map_usable_links()
2499 	    !smc_link_active(conn->lnk))  in smc_sndbuf_sync_sg_for_device()
2501 	smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE);  in smc_sndbuf_sync_sg_for_device()
2513 		if (!smc_link_active(&conn->lgr->lnk[i]))  in smc_rmb_sync_sg_for_cpu()
2515 		smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc,  in smc_rmb_sync_sg_for_cpu()
2635 		if (lgr->lnk[link_idx].link_id == link_id) {  in smc_rtoken_set2()
2647 int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey)  in smc_rtoken_add()  argument
2649 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_add()
2655 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_add()
2656 		    lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr &&  in smc_rtoken_add()
2665 	lgr->rtokens[i][lnk->link_idx].rkey = rkey;  in smc_rtoken_add()
2666 	lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr;  in smc_rtoken_add()
2671 int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey)  in smc_rtoken_delete()  argument
2673 	struct smc_link_group *lgr = smc_get_lgr(lnk);  in smc_rtoken_delete()
2678 		if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&  in smc_rtoken_delete()
2693 			    struct smc_link *lnk,  in smc_rmb_rtoken_handling()  argument
2696 	conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr,  in smc_rmb_rtoken_handling()