1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 20336ba09fSKarsten Graul #include "smc_pnet.h" 219bf9abeaSUrsula Braun 220f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 230f627126SStefan Raspl 240f627126SStefan Raspl struct smc_llc_hdr { 250f627126SStefan Raspl struct smc_wr_rx_hdr common; 260f627126SStefan Raspl u8 length; /* 44 */ 2752bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 2852bedf37SKarsten Graul u8 reserved:4, 2952bedf37SKarsten Graul add_link_rej_rsn:4; 3052bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3152bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3252bedf37SKarsten Graul reserved:4; 3352bedf37SKarsten Graul #endif 340f627126SStefan Raspl u8 flags; 350f627126SStefan Raspl }; 360f627126SStefan Raspl 3775d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 3875d320d6SKarsten Graul 390f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 400f627126SStefan Raspl struct smc_llc_hdr hd; 410f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 420f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 430f627126SStefan Raspl u8 sender_qp_num[3]; 440f627126SStefan Raspl u8 link_num; 450f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 460f627126SStefan Raspl u8 max_links; 470f627126SStefan Raspl u8 reserved[9]; 480f627126SStefan Raspl }; 490f627126SStefan Raspl 5052bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5152bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 5252bedf37SKarsten Graul 5352bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 5452bedf37SKarsten Graul 5552bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 5652bedf37SKarsten Graul struct smc_llc_hdr hd; 5752bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 5852bedf37SKarsten Graul u8 reserved2[2]; 5952bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 6052bedf37SKarsten Graul u8 sender_qp_num[3]; 6152bedf37SKarsten Graul u8 link_num; 62fbed3b37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 63fbed3b37SKarsten Graul u8 reserved3 : 4, 64fbed3b37SKarsten Graul qp_mtu : 4; 65fbed3b37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 66fbed3b37SKarsten Graul u8 qp_mtu : 4, 67fbed3b37SKarsten Graul reserved3 : 4; 68fbed3b37SKarsten Graul #endif 6952bedf37SKarsten Graul u8 initial_psn[3]; 7052bedf37SKarsten Graul u8 reserved[8]; 7152bedf37SKarsten Graul }; 7252bedf37SKarsten Graul 7387f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt { 7487f88cdaSKarsten Graul __be32 rmb_key; 7587f88cdaSKarsten Graul __be32 rmb_key_new; 7687f88cdaSKarsten Graul __be64 rmb_vaddr_new; 7787f88cdaSKarsten Graul }; 7887f88cdaSKarsten Graul 7987f88cdaSKarsten Graul #define SMC_LLC_RKEYS_PER_CONT_MSG 2 8087f88cdaSKarsten Graul 8187f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont { /* type 0x03 */ 8287f88cdaSKarsten Graul struct smc_llc_hdr hd; 8387f88cdaSKarsten Graul u8 link_num; 8487f88cdaSKarsten Graul u8 num_rkeys; 8587f88cdaSKarsten Graul u8 reserved2[2]; 8687f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 8787f88cdaSKarsten Graul u8 reserved[4]; 8887f88cdaSKarsten Graul } __packed; /* format defined in RFC7609 */ 8987f88cdaSKarsten Graul 9052bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 9152bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 9252bedf37SKarsten Graul 9352bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 9452bedf37SKarsten Graul struct smc_llc_hdr hd; 9552bedf37SKarsten Graul u8 link_num; 9652bedf37SKarsten Graul __be32 reason; 9752bedf37SKarsten Graul u8 reserved[35]; 9852bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 9952bedf37SKarsten Graul 100313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 101313164daSKarsten Graul struct smc_llc_hdr hd; 102313164daSKarsten Graul u8 user_data[16]; 103313164daSKarsten Graul u8 reserved[24]; 104313164daSKarsten Graul }; 105313164daSKarsten Graul 1064ed75de5SKarsten Graul struct smc_rmb_rtoken { 1074ed75de5SKarsten Graul union { 1084ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 1094ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 1104ed75de5SKarsten Graul /* rtoken is always for the current link */ 1114ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 1124ed75de5SKarsten Graul }; 1134ed75de5SKarsten Graul __be32 rmb_key; 1144ed75de5SKarsten Graul __be64 rmb_vaddr; 1154ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 1164ed75de5SKarsten Graul 1174ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 1184ed75de5SKarsten Graul 1194ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 1204ed75de5SKarsten Graul struct smc_llc_hdr hd; 1214ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1224ed75de5SKarsten Graul u8 reserved; 1234ed75de5SKarsten Graul }; 1244ed75de5SKarsten Graul 1254ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 1263bc67e09SKarsten Graul #define SMC_LLC_FLAG_RKEY_RETRY 0x10 1274ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1284ed75de5SKarsten Graul 1294ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1304ed75de5SKarsten Graul struct smc_llc_hdr hd; 1314ed75de5SKarsten Graul u8 num_rkeys; 1324ed75de5SKarsten Graul u8 err_mask; 1334ed75de5SKarsten Graul u8 reserved[2]; 1344ed75de5SKarsten Graul __be32 rkey[8]; 1354ed75de5SKarsten Graul u8 reserved2[4]; 1364ed75de5SKarsten Graul }; 1374ed75de5SKarsten Graul 1380f627126SStefan Raspl union smc_llc_msg { 1390f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 14052bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 14187f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont add_link_cont; 14252bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1434ed75de5SKarsten Graul 1444ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1454ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1464ed75de5SKarsten Graul 147313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1480f627126SStefan Raspl struct { 1490f627126SStefan Raspl struct smc_llc_hdr hdr; 1500f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1510f627126SStefan Raspl } raw; 1520f627126SStefan Raspl }; 1530f627126SStefan Raspl 1540f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1550f627126SStefan Raspl 1566c8968c4SKarsten Graul struct smc_llc_qentry { 1576c8968c4SKarsten Graul struct list_head list; 1586c8968c4SKarsten Graul struct smc_link *link; 1596c8968c4SKarsten Graul union smc_llc_msg msg; 1606c8968c4SKarsten Graul }; 1616c8968c4SKarsten Graul 1624dadd151SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 1634dadd151SKarsten Graul 164555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 165555da9afSKarsten Graul { 166555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 167555da9afSKarsten Graul 168555da9afSKarsten Graul flow->qentry = NULL; 169555da9afSKarsten Graul return qentry; 170555da9afSKarsten Graul } 171555da9afSKarsten Graul 172555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 173555da9afSKarsten Graul { 174555da9afSKarsten Graul struct smc_llc_qentry *qentry; 175555da9afSKarsten Graul 176555da9afSKarsten Graul if (flow->qentry) { 177555da9afSKarsten Graul qentry = flow->qentry; 178555da9afSKarsten Graul flow->qentry = NULL; 179555da9afSKarsten Graul kfree(qentry); 180555da9afSKarsten Graul } 181555da9afSKarsten Graul } 182555da9afSKarsten Graul 183555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 184555da9afSKarsten Graul struct smc_llc_qentry *qentry) 185555da9afSKarsten Graul { 186555da9afSKarsten Graul flow->qentry = qentry; 187555da9afSKarsten Graul } 188555da9afSKarsten Graul 189*6778a6beSKarsten Graul static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 190*6778a6beSKarsten Graul struct smc_llc_qentry *qentry) 191*6778a6beSKarsten Graul { 192*6778a6beSKarsten Graul u8 msg_type = qentry->msg.raw.hdr.common.type; 193*6778a6beSKarsten Graul 194*6778a6beSKarsten Graul if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 195*6778a6beSKarsten Graul flow_type != msg_type && !lgr->delayed_event) { 196*6778a6beSKarsten Graul lgr->delayed_event = qentry; 197*6778a6beSKarsten Graul return; 198*6778a6beSKarsten Graul } 199*6778a6beSKarsten Graul /* drop parallel or already-in-progress llc requests */ 200*6778a6beSKarsten Graul if (flow_type != msg_type) 201*6778a6beSKarsten Graul pr_warn_once("smc: SMC-R lg %*phN dropped parallel " 202*6778a6beSKarsten Graul "LLC msg: msg %d flow %d role %d\n", 203*6778a6beSKarsten Graul SMC_LGR_ID_SIZE, &lgr->id, 204*6778a6beSKarsten Graul qentry->msg.raw.hdr.common.type, 205*6778a6beSKarsten Graul flow_type, lgr->role); 206*6778a6beSKarsten Graul kfree(qentry); 207*6778a6beSKarsten Graul } 208*6778a6beSKarsten Graul 209555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 210555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 211555da9afSKarsten Graul struct smc_llc_qentry *qentry) 212555da9afSKarsten Graul { 213555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 214555da9afSKarsten Graul 215555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 216555da9afSKarsten Graul if (flow->type) { 217555da9afSKarsten Graul /* a flow is already active */ 218*6778a6beSKarsten Graul smc_llc_flow_parallel(lgr, flow->type, qentry); 219555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 220555da9afSKarsten Graul return false; 221555da9afSKarsten Graul } 222555da9afSKarsten Graul switch (qentry->msg.raw.hdr.common.type) { 223555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 224555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 225555da9afSKarsten Graul break; 226555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 227555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 228555da9afSKarsten Graul break; 229555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 230555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 231555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 232555da9afSKarsten Graul break; 233555da9afSKarsten Graul default: 234555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 235555da9afSKarsten Graul } 236555da9afSKarsten Graul if (qentry == lgr->delayed_event) 237555da9afSKarsten Graul lgr->delayed_event = NULL; 238555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 239*6778a6beSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 240555da9afSKarsten Graul return true; 241555da9afSKarsten Graul } 242555da9afSKarsten Graul 243555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 244555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 245555da9afSKarsten Graul enum smc_llc_flowtype type) 246555da9afSKarsten Graul { 247555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 248555da9afSKarsten Graul int rc; 249555da9afSKarsten Graul 250555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 251555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 252555da9afSKarsten Graul */ 253555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 254555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 255555da9afSKarsten Graul again: 256555da9afSKarsten Graul if (list_empty(&lgr->list)) 257555da9afSKarsten Graul return -ENODEV; 258555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 259555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 260555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 261555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 262555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 263555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 264555da9afSKarsten Graul return 0; 265555da9afSKarsten Graul } 266555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 267*6778a6beSKarsten Graul rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 268555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 269555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 270*6778a6beSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote))), 271*6778a6beSKarsten Graul SMC_LLC_WAIT_TIME * 10); 272555da9afSKarsten Graul if (!rc) 273555da9afSKarsten Graul return -ETIMEDOUT; 274555da9afSKarsten Graul goto again; 275555da9afSKarsten Graul } 276555da9afSKarsten Graul 277555da9afSKarsten Graul /* finish the current llc flow */ 278555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 279555da9afSKarsten Graul { 280555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 281555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 282555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 283555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 284555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 285555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 286555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 287555da9afSKarsten Graul else 288*6778a6beSKarsten Graul wake_up(&lgr->llc_flow_waiter); 289555da9afSKarsten Graul } 290555da9afSKarsten Graul 291555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 292555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 293555da9afSKarsten Graul */ 294555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 295555da9afSKarsten Graul struct smc_link *lnk, 296555da9afSKarsten Graul int time_out, u8 exp_msg) 297555da9afSKarsten Graul { 298555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 299*6778a6beSKarsten Graul u8 rcv_msg; 300555da9afSKarsten Graul 301*6778a6beSKarsten Graul wait_event_timeout(lgr->llc_msg_waiter, 302555da9afSKarsten Graul (flow->qentry || 303555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 304555da9afSKarsten Graul list_empty(&lgr->list)), 305555da9afSKarsten Graul time_out); 306555da9afSKarsten Graul if (!flow->qentry || 307555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 308555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 309555da9afSKarsten Graul goto out; 310555da9afSKarsten Graul } 311*6778a6beSKarsten Graul rcv_msg = flow->qentry->msg.raw.hdr.common.type; 312*6778a6beSKarsten Graul if (exp_msg && rcv_msg != exp_msg) { 313555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 314*6778a6beSKarsten Graul rcv_msg == SMC_LLC_DELETE_LINK) { 315555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 316555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 317555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 318555da9afSKarsten Graul return NULL; 319555da9afSKarsten Graul } 320*6778a6beSKarsten Graul pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: " 321*6778a6beSKarsten Graul "msg %d exp %d flow %d role %d flags %x\n", 322*6778a6beSKarsten Graul SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg, 323*6778a6beSKarsten Graul flow->type, lgr->role, 324*6778a6beSKarsten Graul flow->qentry->msg.raw.hdr.flags); 325555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 326555da9afSKarsten Graul } 327555da9afSKarsten Graul out: 328555da9afSKarsten Graul return flow->qentry; 329555da9afSKarsten Graul } 330555da9afSKarsten Graul 3319bf9abeaSUrsula Braun /********************************** send *************************************/ 3329bf9abeaSUrsula Braun 3339bf9abeaSUrsula Braun struct smc_llc_tx_pend { 3349bf9abeaSUrsula Braun }; 3359bf9abeaSUrsula Braun 3369bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 3379bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 3389bf9abeaSUrsula Braun struct smc_link *link, 3399bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3409bf9abeaSUrsula Braun { 3419bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3429bf9abeaSUrsula Braun } 3439bf9abeaSUrsula Braun 3449bf9abeaSUrsula Braun /** 3459bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3469bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3479bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3489bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3499bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3509bf9abeaSUrsula Braun * 3519bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3529bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3539bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3549bf9abeaSUrsula Braun * 3559bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 3569bf9abeaSUrsula Braun */ 3579bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 3589bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 3599bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 3609bf9abeaSUrsula Braun { 3619bf9abeaSUrsula Braun int rc; 3629bf9abeaSUrsula Braun 363ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 364ad6f317fSUrsula Braun pend); 3659bf9abeaSUrsula Braun if (rc < 0) 3669bf9abeaSUrsula Braun return rc; 3679bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3689bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 3699bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 3709bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3719bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 3729bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 3739bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3749bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 3759bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 3769bf9abeaSUrsula Braun return 0; 3779bf9abeaSUrsula Braun } 3789bf9abeaSUrsula Braun 3799bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 380947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 3819bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 3829bf9abeaSUrsula Braun { 3839bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 3849bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 3859bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 3869bf9abeaSUrsula Braun int rc; 3879bf9abeaSUrsula Braun 3889bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 3899bf9abeaSUrsula Braun if (rc) 3909bf9abeaSUrsula Braun return rc; 3919bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 3929bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 3939bf9abeaSUrsula Braun confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 3949bf9abeaSUrsula Braun confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 39575d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 3969bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 3979bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 398947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 399947541f3SUrsula Braun ETH_ALEN); 4007005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 4019bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 4022be922f3SKarsten Graul confllc->link_num = link->link_id; 40345fa8da0SKarsten Graul memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 404b1570a87SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 40552bedf37SKarsten Graul /* send llc message */ 40652bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 40752bedf37SKarsten Graul return rc; 40852bedf37SKarsten Graul } 40952bedf37SKarsten Graul 41044aa81ceSKarsten Graul /* send LLC confirm rkey request */ 4113d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 41244aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 41344aa81ceSKarsten Graul { 41444aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 41544aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 41644aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 4173d88a21bSKarsten Graul struct smc_link *link; 4183d88a21bSKarsten Graul int i, rc, rtok_ix; 41944aa81ceSKarsten Graul 4203d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 42144aa81ceSKarsten Graul if (rc) 42244aa81ceSKarsten Graul return rc; 42344aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 42444aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 42544aa81ceSKarsten Graul rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 42644aa81ceSKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 4273d88a21bSKarsten Graul 4283d88a21bSKarsten Graul rtok_ix = 1; 4293d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 4303d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 4313d88a21bSKarsten Graul if (link->state == SMC_LNK_ACTIVE && link != send_link) { 4323d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 4333d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 434387707fdSKarsten Graul htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 4353d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 4363d88a21bSKarsten Graul (u64)sg_dma_address( 4373d88a21bSKarsten Graul rmb_desc->sgt[link->link_idx].sgl)); 4383d88a21bSKarsten Graul rtok_ix++; 4393d88a21bSKarsten Graul } 4403d88a21bSKarsten Graul } 4413d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 4423d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 4433d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 4443d88a21bSKarsten Graul htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 44544aa81ceSKarsten Graul rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 4463d88a21bSKarsten Graul (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 44744aa81ceSKarsten Graul /* send llc message */ 4483d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 44944aa81ceSKarsten Graul return rc; 45044aa81ceSKarsten Graul } 45144aa81ceSKarsten Graul 45260e03c62SKarsten Graul /* send LLC delete rkey request */ 45360e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 45460e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 45560e03c62SKarsten Graul { 45660e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 45760e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 45860e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 45960e03c62SKarsten Graul int rc; 46060e03c62SKarsten Graul 46160e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 46260e03c62SKarsten Graul if (rc) 46360e03c62SKarsten Graul return rc; 46460e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 46560e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 46660e03c62SKarsten Graul rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 46760e03c62SKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 46860e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 469387707fdSKarsten Graul rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 47060e03c62SKarsten Graul /* send llc message */ 47160e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 47260e03c62SKarsten Graul return rc; 47360e03c62SKarsten Graul } 47460e03c62SKarsten Graul 47552bedf37SKarsten Graul /* send ADD LINK request or response */ 4767005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 477fbed3b37SKarsten Graul struct smc_link *link_new, 47852bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 47952bedf37SKarsten Graul { 48052bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 48152bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 48252bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 48352bedf37SKarsten Graul int rc; 48452bedf37SKarsten Graul 48552bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 48652bedf37SKarsten Graul if (rc) 48752bedf37SKarsten Graul return rc; 48852bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 489fbed3b37SKarsten Graul 490fbed3b37SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 491fbed3b37SKarsten Graul addllc->hd.common.type = SMC_LLC_ADD_LINK; 492fbed3b37SKarsten Graul addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 493fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 494fbed3b37SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 495fbed3b37SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 496fbed3b37SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 497fbed3b37SKarsten Graul if (link_new) { 498fbed3b37SKarsten Graul addllc->link_num = link_new->link_id; 499fbed3b37SKarsten Graul hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 500fbed3b37SKarsten Graul hton24(addllc->initial_psn, link_new->psn_initial); 501fbed3b37SKarsten Graul if (reqresp == SMC_LLC_REQ) 502fbed3b37SKarsten Graul addllc->qp_mtu = link_new->path_mtu; 503fbed3b37SKarsten Graul else 504fbed3b37SKarsten Graul addllc->qp_mtu = min(link_new->path_mtu, 505fbed3b37SKarsten Graul link_new->peer_mtu); 506fbed3b37SKarsten Graul } 50752bedf37SKarsten Graul /* send llc message */ 50852bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 50952bedf37SKarsten Graul return rc; 51052bedf37SKarsten Graul } 51152bedf37SKarsten Graul 51252bedf37SKarsten Graul /* send DELETE LINK request or response */ 513fbed3b37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 514fbed3b37SKarsten Graul enum smc_llc_reqresp reqresp, bool orderly, 515fbed3b37SKarsten Graul u32 reason) 51652bedf37SKarsten Graul { 51752bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 51852bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 51952bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 52052bedf37SKarsten Graul int rc; 52152bedf37SKarsten Graul 52252bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 52352bedf37SKarsten Graul if (rc) 52452bedf37SKarsten Graul return rc; 52552bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 526fbed3b37SKarsten Graul 527fbed3b37SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 528fbed3b37SKarsten Graul delllc->hd.common.type = SMC_LLC_DELETE_LINK; 529fbed3b37SKarsten Graul delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 530fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 531fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 532fbed3b37SKarsten Graul if (orderly) 533fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 534fbed3b37SKarsten Graul if (link_del_id) 535fbed3b37SKarsten Graul delllc->link_num = link_del_id; 536fbed3b37SKarsten Graul else 537fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 538fbed3b37SKarsten Graul delllc->reason = htonl(reason); 5399bf9abeaSUrsula Braun /* send llc message */ 5409bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 5419bf9abeaSUrsula Braun return rc; 5429bf9abeaSUrsula Braun } 5439bf9abeaSUrsula Braun 544d97935faSKarsten Graul /* send LLC test link request */ 545d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 546313164daSKarsten Graul { 547313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 548313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 549313164daSKarsten Graul struct smc_wr_buf *wr_buf; 550313164daSKarsten Graul int rc; 551313164daSKarsten Graul 552313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 553313164daSKarsten Graul if (rc) 554313164daSKarsten Graul return rc; 555313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 556313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 557313164daSKarsten Graul testllc->hd.common.type = SMC_LLC_TEST_LINK; 558313164daSKarsten Graul testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 559313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 560313164daSKarsten Graul /* send llc message */ 561313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 562313164daSKarsten Graul return rc; 563313164daSKarsten Graul } 564313164daSKarsten Graul 5656c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 5666c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 5674ed75de5SKarsten Graul { 5684ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 5694ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 5704ed75de5SKarsten Graul int rc; 5714ed75de5SKarsten Graul 5726c8968c4SKarsten Graul if (!smc_link_usable(link)) 5736c8968c4SKarsten Graul return -ENOLINK; 5746c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 5754ed75de5SKarsten Graul if (rc) 5766c8968c4SKarsten Graul return rc; 5776c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 5786c8968c4SKarsten Graul return smc_wr_tx_send(link, pend); 5794ed75de5SKarsten Graul } 5804ed75de5SKarsten Graul 581f3811fd7SKarsten Graul /* schedule an llc send on link, may wait for buffers, 582f3811fd7SKarsten Graul * and wait for send completion notification. 583f3811fd7SKarsten Graul * @return 0 on success 584f3811fd7SKarsten Graul */ 585f3811fd7SKarsten Graul static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 586f3811fd7SKarsten Graul { 587f3811fd7SKarsten Graul struct smc_wr_tx_pend_priv *pend; 588f3811fd7SKarsten Graul struct smc_wr_buf *wr_buf; 589f3811fd7SKarsten Graul int rc; 590f3811fd7SKarsten Graul 591f3811fd7SKarsten Graul if (!smc_link_usable(link)) 592f3811fd7SKarsten Graul return -ENOLINK; 593f3811fd7SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 594f3811fd7SKarsten Graul if (rc) 595f3811fd7SKarsten Graul return rc; 596f3811fd7SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 597f3811fd7SKarsten Graul return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 598f3811fd7SKarsten Graul } 599f3811fd7SKarsten Graul 6009bf9abeaSUrsula Braun /********************************* receive ***********************************/ 6019bf9abeaSUrsula Braun 602336ba09fSKarsten Graul static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 603336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t) 604336ba09fSKarsten Graul { 605336ba09fSKarsten Graul int i; 606336ba09fSKarsten Graul 607336ba09fSKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 608336ba09fSKarsten Graul (lgr->type != SMC_LGR_SINGLE && 609336ba09fSKarsten Graul (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 610336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 611336ba09fSKarsten Graul return -EMLINK; 612336ba09fSKarsten Graul 613336ba09fSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 614336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 615336ba09fSKarsten Graul for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 616336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 617336ba09fSKarsten Graul return i; 618336ba09fSKarsten Graul } else { 619336ba09fSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 620336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 621336ba09fSKarsten Graul return i; 622336ba09fSKarsten Graul } 623336ba09fSKarsten Graul return -EMLINK; 624336ba09fSKarsten Graul } 625336ba09fSKarsten Graul 62687f88cdaSKarsten Graul /* return first buffer from any of the next buf lists */ 62787f88cdaSKarsten Graul static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 62887f88cdaSKarsten Graul int *buf_lst) 62987f88cdaSKarsten Graul { 63087f88cdaSKarsten Graul struct smc_buf_desc *buf_pos; 63187f88cdaSKarsten Graul 63287f88cdaSKarsten Graul while (*buf_lst < SMC_RMBE_SIZES) { 63387f88cdaSKarsten Graul buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 63487f88cdaSKarsten Graul struct smc_buf_desc, list); 63587f88cdaSKarsten Graul if (buf_pos) 63687f88cdaSKarsten Graul return buf_pos; 63787f88cdaSKarsten Graul (*buf_lst)++; 63887f88cdaSKarsten Graul } 63987f88cdaSKarsten Graul return NULL; 64087f88cdaSKarsten Graul } 64187f88cdaSKarsten Graul 64287f88cdaSKarsten Graul /* return next rmb from buffer lists */ 64387f88cdaSKarsten Graul static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 64487f88cdaSKarsten Graul int *buf_lst, 64587f88cdaSKarsten Graul struct smc_buf_desc *buf_pos) 64687f88cdaSKarsten Graul { 64787f88cdaSKarsten Graul struct smc_buf_desc *buf_next; 64887f88cdaSKarsten Graul 64987f88cdaSKarsten Graul if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 65087f88cdaSKarsten Graul (*buf_lst)++; 65187f88cdaSKarsten Graul return _smc_llc_get_next_rmb(lgr, buf_lst); 65287f88cdaSKarsten Graul } 65387f88cdaSKarsten Graul buf_next = list_next_entry(buf_pos, list); 65487f88cdaSKarsten Graul return buf_next; 65587f88cdaSKarsten Graul } 65687f88cdaSKarsten Graul 65787f88cdaSKarsten Graul static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 65887f88cdaSKarsten Graul int *buf_lst) 65987f88cdaSKarsten Graul { 66087f88cdaSKarsten Graul *buf_lst = 0; 66187f88cdaSKarsten Graul return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 66287f88cdaSKarsten Graul } 66387f88cdaSKarsten Graul 66487f88cdaSKarsten Graul /* send one add_link_continue msg */ 66587f88cdaSKarsten Graul static int smc_llc_add_link_cont(struct smc_link *link, 66687f88cdaSKarsten Graul struct smc_link *link_new, u8 *num_rkeys_todo, 66787f88cdaSKarsten Graul int *buf_lst, struct smc_buf_desc **buf_pos) 66887f88cdaSKarsten Graul { 66987f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 67087f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 67187f88cdaSKarsten Graul int prim_lnk_idx, lnk_idx, i, rc; 67287f88cdaSKarsten Graul struct smc_wr_tx_pend_priv *pend; 67387f88cdaSKarsten Graul struct smc_wr_buf *wr_buf; 67487f88cdaSKarsten Graul struct smc_buf_desc *rmb; 67587f88cdaSKarsten Graul u8 n; 67687f88cdaSKarsten Graul 67787f88cdaSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 67887f88cdaSKarsten Graul if (rc) 67987f88cdaSKarsten Graul return rc; 68087f88cdaSKarsten Graul addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 68187f88cdaSKarsten Graul memset(addc_llc, 0, sizeof(*addc_llc)); 68287f88cdaSKarsten Graul 68387f88cdaSKarsten Graul prim_lnk_idx = link->link_idx; 68487f88cdaSKarsten Graul lnk_idx = link_new->link_idx; 68587f88cdaSKarsten Graul addc_llc->link_num = link_new->link_id; 68687f88cdaSKarsten Graul addc_llc->num_rkeys = *num_rkeys_todo; 68787f88cdaSKarsten Graul n = *num_rkeys_todo; 68887f88cdaSKarsten Graul for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 68987f88cdaSKarsten Graul if (!*buf_pos) { 69087f88cdaSKarsten Graul addc_llc->num_rkeys = addc_llc->num_rkeys - 69187f88cdaSKarsten Graul *num_rkeys_todo; 69287f88cdaSKarsten Graul *num_rkeys_todo = 0; 69387f88cdaSKarsten Graul break; 69487f88cdaSKarsten Graul } 69587f88cdaSKarsten Graul rmb = *buf_pos; 69687f88cdaSKarsten Graul 69787f88cdaSKarsten Graul addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 69887f88cdaSKarsten Graul addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 69987f88cdaSKarsten Graul addc_llc->rt[i].rmb_vaddr_new = 70087f88cdaSKarsten Graul cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 70187f88cdaSKarsten Graul 70287f88cdaSKarsten Graul (*num_rkeys_todo)--; 70387f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 70487f88cdaSKarsten Graul while (*buf_pos && !(*buf_pos)->used) 70587f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 70687f88cdaSKarsten Graul } 70787f88cdaSKarsten Graul addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; 70887f88cdaSKarsten Graul addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 70987f88cdaSKarsten Graul if (lgr->role == SMC_CLNT) 71087f88cdaSKarsten Graul addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 71187f88cdaSKarsten Graul return smc_wr_tx_send(link, pend); 71287f88cdaSKarsten Graul } 71387f88cdaSKarsten Graul 71487f88cdaSKarsten Graul static int smc_llc_cli_rkey_exchange(struct smc_link *link, 71587f88cdaSKarsten Graul struct smc_link *link_new) 71687f88cdaSKarsten Graul { 71787f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 71887f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 71987f88cdaSKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 72087f88cdaSKarsten Graul struct smc_llc_qentry *qentry; 72187f88cdaSKarsten Graul struct smc_buf_desc *buf_pos; 72287f88cdaSKarsten Graul int buf_lst; 72387f88cdaSKarsten Graul int rc = 0; 72487f88cdaSKarsten Graul int i; 72587f88cdaSKarsten Graul 72687f88cdaSKarsten Graul mutex_lock(&lgr->rmbs_lock); 72787f88cdaSKarsten Graul num_rkeys_send = lgr->conns_num; 72887f88cdaSKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 72987f88cdaSKarsten Graul do { 73087f88cdaSKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 73187f88cdaSKarsten Graul SMC_LLC_ADD_LINK_CONT); 73287f88cdaSKarsten Graul if (!qentry) { 73387f88cdaSKarsten Graul rc = -ETIMEDOUT; 73487f88cdaSKarsten Graul break; 73587f88cdaSKarsten Graul } 73687f88cdaSKarsten Graul addc_llc = &qentry->msg.add_link_cont; 73787f88cdaSKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 73887f88cdaSKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 73987f88cdaSKarsten Graul for (i = 0; i < max; i++) { 74087f88cdaSKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 74187f88cdaSKarsten Graul addc_llc->rt[i].rmb_key, 74287f88cdaSKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 74387f88cdaSKarsten Graul addc_llc->rt[i].rmb_key_new); 74487f88cdaSKarsten Graul num_rkeys_recv--; 74587f88cdaSKarsten Graul } 74687f88cdaSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 74787f88cdaSKarsten Graul rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 74887f88cdaSKarsten Graul &buf_lst, &buf_pos); 74987f88cdaSKarsten Graul if (rc) 75087f88cdaSKarsten Graul break; 75187f88cdaSKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 75287f88cdaSKarsten Graul 75387f88cdaSKarsten Graul mutex_unlock(&lgr->rmbs_lock); 75487f88cdaSKarsten Graul return rc; 75587f88cdaSKarsten Graul } 75687f88cdaSKarsten Graul 757336ba09fSKarsten Graul /* prepare and send an add link reject response */ 758336ba09fSKarsten Graul static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 759336ba09fSKarsten Graul { 760336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 761336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 762336ba09fSKarsten Graul qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 763336ba09fSKarsten Graul return smc_llc_send_message(qentry->link, &qentry->msg); 764336ba09fSKarsten Graul } 765336ba09fSKarsten Graul 766b1570a87SKarsten Graul static int smc_llc_cli_conf_link(struct smc_link *link, 767b1570a87SKarsten Graul struct smc_init_info *ini, 768b1570a87SKarsten Graul struct smc_link *link_new, 769b1570a87SKarsten Graul enum smc_lgr_type lgr_new_t) 770b1570a87SKarsten Graul { 771b1570a87SKarsten Graul struct smc_link_group *lgr = link->lgr; 772b1570a87SKarsten Graul struct smc_llc_qentry *qentry = NULL; 773b1570a87SKarsten Graul int rc = 0; 774b1570a87SKarsten Graul 775b1570a87SKarsten Graul /* receive CONFIRM LINK request over RoCE fabric */ 776b1570a87SKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 777b1570a87SKarsten Graul if (!qentry) { 778b1570a87SKarsten Graul rc = smc_llc_send_delete_link(link, link_new->link_id, 779b1570a87SKarsten Graul SMC_LLC_REQ, false, 780b1570a87SKarsten Graul SMC_LLC_DEL_LOST_PATH); 781b1570a87SKarsten Graul return -ENOLINK; 782b1570a87SKarsten Graul } 783b1570a87SKarsten Graul if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 784b1570a87SKarsten Graul /* received DELETE_LINK instead */ 785b1570a87SKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 786b1570a87SKarsten Graul smc_llc_send_message(link, &qentry->msg); 787b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 788b1570a87SKarsten Graul return -ENOLINK; 789b1570a87SKarsten Graul } 790649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 791b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 792b1570a87SKarsten Graul 793b1570a87SKarsten Graul rc = smc_ib_modify_qp_rts(link_new); 794b1570a87SKarsten Graul if (rc) { 795b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 796b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 797b1570a87SKarsten Graul return -ENOLINK; 798b1570a87SKarsten Graul } 799b1570a87SKarsten Graul smc_wr_remember_qp_attr(link_new); 800b1570a87SKarsten Graul 801b1570a87SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 802b1570a87SKarsten Graul if (rc) { 803b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 804b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 805b1570a87SKarsten Graul return -ENOLINK; 806b1570a87SKarsten Graul } 807b1570a87SKarsten Graul 808b1570a87SKarsten Graul /* send CONFIRM LINK response over RoCE fabric */ 809b1570a87SKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 810b1570a87SKarsten Graul if (rc) { 811b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 812b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 813b1570a87SKarsten Graul return -ENOLINK; 814b1570a87SKarsten Graul } 815b1570a87SKarsten Graul smc_llc_link_active(link_new); 816ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 817ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 818ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 819ad6c111bSKarsten Graul else 820ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 821b1570a87SKarsten Graul return 0; 822b1570a87SKarsten Graul } 823b1570a87SKarsten Graul 824336ba09fSKarsten Graul static void smc_llc_save_add_link_info(struct smc_link *link, 825336ba09fSKarsten Graul struct smc_llc_msg_add_link *add_llc) 826336ba09fSKarsten Graul { 827336ba09fSKarsten Graul link->peer_qpn = ntoh24(add_llc->sender_qp_num); 828336ba09fSKarsten Graul memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 829336ba09fSKarsten Graul memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 830336ba09fSKarsten Graul link->peer_psn = ntoh24(add_llc->initial_psn); 831336ba09fSKarsten Graul link->peer_mtu = add_llc->qp_mtu; 832336ba09fSKarsten Graul } 833336ba09fSKarsten Graul 834336ba09fSKarsten Graul /* as an SMC client, process an add link request */ 835336ba09fSKarsten Graul int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 836336ba09fSKarsten Graul { 837336ba09fSKarsten Graul struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 838336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 839336ba09fSKarsten Graul struct smc_link_group *lgr = smc_get_lgr(link); 840336ba09fSKarsten Graul struct smc_link *lnk_new = NULL; 841336ba09fSKarsten Graul struct smc_init_info ini; 842336ba09fSKarsten Graul int lnk_idx, rc = 0; 843336ba09fSKarsten Graul 844336ba09fSKarsten Graul ini.vlan_id = lgr->vlan_id; 845336ba09fSKarsten Graul smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 846336ba09fSKarsten Graul if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 847336ba09fSKarsten Graul !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { 848336ba09fSKarsten Graul if (!ini.ib_dev) 849336ba09fSKarsten Graul goto out_reject; 850336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 851336ba09fSKarsten Graul } 852336ba09fSKarsten Graul if (!ini.ib_dev) { 853336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 854336ba09fSKarsten Graul ini.ib_dev = link->smcibdev; 855336ba09fSKarsten Graul ini.ib_port = link->ibport; 856336ba09fSKarsten Graul } 857336ba09fSKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 858336ba09fSKarsten Graul if (lnk_idx < 0) 859336ba09fSKarsten Graul goto out_reject; 860336ba09fSKarsten Graul lnk_new = &lgr->lnk[lnk_idx]; 861336ba09fSKarsten Graul rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); 862336ba09fSKarsten Graul if (rc) 863336ba09fSKarsten Graul goto out_reject; 864336ba09fSKarsten Graul smc_llc_save_add_link_info(lnk_new, llc); 86545fa8da0SKarsten Graul lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 86645fa8da0SKarsten Graul smc_llc_link_set_uid(lnk_new); 867336ba09fSKarsten Graul 868336ba09fSKarsten Graul rc = smc_ib_ready_link(lnk_new); 869336ba09fSKarsten Graul if (rc) 870336ba09fSKarsten Graul goto out_clear_lnk; 871336ba09fSKarsten Graul 872336ba09fSKarsten Graul rc = smcr_buf_map_lgr(lnk_new); 873336ba09fSKarsten Graul if (rc) 874336ba09fSKarsten Graul goto out_clear_lnk; 875336ba09fSKarsten Graul 876336ba09fSKarsten Graul rc = smc_llc_send_add_link(link, 877336ba09fSKarsten Graul lnk_new->smcibdev->mac[ini.ib_port - 1], 878336ba09fSKarsten Graul lnk_new->gid, lnk_new, SMC_LLC_RESP); 879336ba09fSKarsten Graul if (rc) 880336ba09fSKarsten Graul goto out_clear_lnk; 88187f88cdaSKarsten Graul rc = smc_llc_cli_rkey_exchange(link, lnk_new); 882336ba09fSKarsten Graul if (rc) { 883336ba09fSKarsten Graul rc = 0; 884336ba09fSKarsten Graul goto out_clear_lnk; 885336ba09fSKarsten Graul } 886b1570a87SKarsten Graul rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); 887336ba09fSKarsten Graul if (!rc) 888336ba09fSKarsten Graul goto out; 889336ba09fSKarsten Graul out_clear_lnk: 8900a99be43SKarsten Graul smcr_link_clear(lnk_new, false); 891336ba09fSKarsten Graul out_reject: 892336ba09fSKarsten Graul smc_llc_cli_add_link_reject(qentry); 893336ba09fSKarsten Graul out: 894336ba09fSKarsten Graul kfree(qentry); 895336ba09fSKarsten Graul return rc; 896336ba09fSKarsten Graul } 897336ba09fSKarsten Graul 898b1570a87SKarsten Graul static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 899b1570a87SKarsten Graul { 900b1570a87SKarsten Graul struct smc_llc_qentry *qentry; 901b1570a87SKarsten Graul 902b1570a87SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 903b1570a87SKarsten Graul 904b1570a87SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 905b1570a87SKarsten Graul smc_llc_cli_add_link(qentry->link, qentry); 906b1570a87SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 907b1570a87SKarsten Graul } 908b1570a87SKarsten Graul 9099c416878SKarsten Graul static int smc_llc_active_link_count(struct smc_link_group *lgr) 9109c416878SKarsten Graul { 9119c416878SKarsten Graul int i, link_count = 0; 9129c416878SKarsten Graul 9139c416878SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 9149c416878SKarsten Graul if (!smc_link_usable(&lgr->lnk[i])) 9159c416878SKarsten Graul continue; 9169c416878SKarsten Graul link_count++; 9179c416878SKarsten Graul } 9189c416878SKarsten Graul return link_count; 9199c416878SKarsten Graul } 9209c416878SKarsten Graul 921c9a5d243SKarsten Graul /* find the asymmetric link when 3 links are established */ 922c9a5d243SKarsten Graul static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 923c9a5d243SKarsten Graul { 924c9a5d243SKarsten Graul int asym_idx = -ENOENT; 925c9a5d243SKarsten Graul int i, j, k; 926c9a5d243SKarsten Graul bool found; 927c9a5d243SKarsten Graul 928c9a5d243SKarsten Graul /* determine asymmetric link */ 929c9a5d243SKarsten Graul found = false; 930c9a5d243SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 931c9a5d243SKarsten Graul for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 932c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[i]) || 933c9a5d243SKarsten Graul !smc_link_usable(&lgr->lnk[j])) 934c9a5d243SKarsten Graul continue; 935c9a5d243SKarsten Graul if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 936c9a5d243SKarsten Graul SMC_GID_SIZE)) { 937c9a5d243SKarsten Graul found = true; /* asym_lnk is i or j */ 938c9a5d243SKarsten Graul break; 939c9a5d243SKarsten Graul } 940c9a5d243SKarsten Graul } 941c9a5d243SKarsten Graul if (found) 942c9a5d243SKarsten Graul break; 943c9a5d243SKarsten Graul } 944c9a5d243SKarsten Graul if (!found) 945c9a5d243SKarsten Graul goto out; /* no asymmetric link */ 946c9a5d243SKarsten Graul for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 947c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[k])) 948c9a5d243SKarsten Graul continue; 949c9a5d243SKarsten Graul if (k != i && 950c9a5d243SKarsten Graul !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 951c9a5d243SKarsten Graul SMC_GID_SIZE)) { 952c9a5d243SKarsten Graul asym_idx = i; 953c9a5d243SKarsten Graul break; 954c9a5d243SKarsten Graul } 955c9a5d243SKarsten Graul if (k != j && 956c9a5d243SKarsten Graul !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 957c9a5d243SKarsten Graul SMC_GID_SIZE)) { 958c9a5d243SKarsten Graul asym_idx = j; 959c9a5d243SKarsten Graul break; 960c9a5d243SKarsten Graul } 961c9a5d243SKarsten Graul } 962c9a5d243SKarsten Graul out: 963c9a5d243SKarsten Graul return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 964c9a5d243SKarsten Graul } 965c9a5d243SKarsten Graul 966c9a5d243SKarsten Graul static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 967c9a5d243SKarsten Graul { 968c9a5d243SKarsten Graul struct smc_link *lnk_new = NULL, *lnk_asym; 969c9a5d243SKarsten Graul struct smc_llc_qentry *qentry; 970c9a5d243SKarsten Graul int rc; 971c9a5d243SKarsten Graul 972c9a5d243SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 973c9a5d243SKarsten Graul if (!lnk_asym) 974c9a5d243SKarsten Graul return; /* no asymmetric link */ 975c9a5d243SKarsten Graul if (!smc_link_downing(&lnk_asym->state)) 976c9a5d243SKarsten Graul return; 977c6f02ebeSKarsten Graul lnk_new = smc_switch_conns(lgr, lnk_asym, false); 978c9a5d243SKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_asym); 979c9a5d243SKarsten Graul if (!lnk_new) 980c9a5d243SKarsten Graul goto out_free; 981c9a5d243SKarsten Graul /* change flow type from ADD_LINK into DEL_LINK */ 982c9a5d243SKarsten Graul lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 983c9a5d243SKarsten Graul rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 984c9a5d243SKarsten Graul true, SMC_LLC_DEL_NO_ASYM_NEEDED); 985c9a5d243SKarsten Graul if (rc) { 986c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 987c9a5d243SKarsten Graul goto out_free; 988c9a5d243SKarsten Graul } 989c9a5d243SKarsten Graul qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 990c9a5d243SKarsten Graul SMC_LLC_DELETE_LINK); 991c9a5d243SKarsten Graul if (!qentry) { 992c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 993c9a5d243SKarsten Graul goto out_free; 994c9a5d243SKarsten Graul } 995c9a5d243SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 996c9a5d243SKarsten Graul out_free: 9970a99be43SKarsten Graul smcr_link_clear(lnk_asym, true); 998c9a5d243SKarsten Graul } 999c9a5d243SKarsten Graul 100057b49924SKarsten Graul static int smc_llc_srv_rkey_exchange(struct smc_link *link, 100157b49924SKarsten Graul struct smc_link *link_new) 100257b49924SKarsten Graul { 100357b49924SKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 100457b49924SKarsten Graul struct smc_link_group *lgr = link->lgr; 100557b49924SKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 100657b49924SKarsten Graul struct smc_llc_qentry *qentry = NULL; 100757b49924SKarsten Graul struct smc_buf_desc *buf_pos; 100857b49924SKarsten Graul int buf_lst; 100957b49924SKarsten Graul int rc = 0; 101057b49924SKarsten Graul int i; 101157b49924SKarsten Graul 101257b49924SKarsten Graul mutex_lock(&lgr->rmbs_lock); 101357b49924SKarsten Graul num_rkeys_send = lgr->conns_num; 101457b49924SKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 101557b49924SKarsten Graul do { 101657b49924SKarsten Graul smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 101757b49924SKarsten Graul &buf_lst, &buf_pos); 101857b49924SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 101957b49924SKarsten Graul SMC_LLC_ADD_LINK_CONT); 102057b49924SKarsten Graul if (!qentry) { 102157b49924SKarsten Graul rc = -ETIMEDOUT; 102257b49924SKarsten Graul goto out; 102357b49924SKarsten Graul } 102457b49924SKarsten Graul addc_llc = &qentry->msg.add_link_cont; 102557b49924SKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 102657b49924SKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 102757b49924SKarsten Graul for (i = 0; i < max; i++) { 102857b49924SKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 102957b49924SKarsten Graul addc_llc->rt[i].rmb_key, 103057b49924SKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 103157b49924SKarsten Graul addc_llc->rt[i].rmb_key_new); 103257b49924SKarsten Graul num_rkeys_recv--; 103357b49924SKarsten Graul } 103457b49924SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 103557b49924SKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 103657b49924SKarsten Graul out: 103757b49924SKarsten Graul mutex_unlock(&lgr->rmbs_lock); 103857b49924SKarsten Graul return rc; 103957b49924SKarsten Graul } 104057b49924SKarsten Graul 10411551c95bSKarsten Graul static int smc_llc_srv_conf_link(struct smc_link *link, 10421551c95bSKarsten Graul struct smc_link *link_new, 10431551c95bSKarsten Graul enum smc_lgr_type lgr_new_t) 10441551c95bSKarsten Graul { 10451551c95bSKarsten Graul struct smc_link_group *lgr = link->lgr; 10461551c95bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 10471551c95bSKarsten Graul int rc; 10481551c95bSKarsten Graul 10491551c95bSKarsten Graul /* send CONFIRM LINK request over the RoCE fabric */ 10501551c95bSKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 10511551c95bSKarsten Graul if (rc) 10521551c95bSKarsten Graul return -ENOLINK; 10531551c95bSKarsten Graul /* receive CONFIRM LINK response over the RoCE fabric */ 10541551c95bSKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 10551551c95bSKarsten Graul SMC_LLC_CONFIRM_LINK); 10561551c95bSKarsten Graul if (!qentry) { 10571551c95bSKarsten Graul /* send DELETE LINK */ 10581551c95bSKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 10591551c95bSKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 10601551c95bSKarsten Graul return -ENOLINK; 10611551c95bSKarsten Graul } 1062649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 10631551c95bSKarsten Graul smc_llc_link_active(link_new); 1064ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1065ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1066ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1067ad6c111bSKarsten Graul else 1068ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 10691551c95bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 10701551c95bSKarsten Graul return 0; 10711551c95bSKarsten Graul } 10721551c95bSKarsten Graul 10732d2209f2SKarsten Graul int smc_llc_srv_add_link(struct smc_link *link) 10742d2209f2SKarsten Graul { 10752d2209f2SKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 10762d2209f2SKarsten Graul struct smc_link_group *lgr = link->lgr; 10772d2209f2SKarsten Graul struct smc_llc_msg_add_link *add_llc; 10782d2209f2SKarsten Graul struct smc_llc_qentry *qentry = NULL; 10792d2209f2SKarsten Graul struct smc_link *link_new; 10802d2209f2SKarsten Graul struct smc_init_info ini; 10812d2209f2SKarsten Graul int lnk_idx, rc = 0; 10822d2209f2SKarsten Graul 10832d2209f2SKarsten Graul /* ignore client add link recommendation, start new flow */ 10842d2209f2SKarsten Graul ini.vlan_id = lgr->vlan_id; 10852d2209f2SKarsten Graul smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 10862d2209f2SKarsten Graul if (!ini.ib_dev) { 10872d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 10882d2209f2SKarsten Graul ini.ib_dev = link->smcibdev; 10892d2209f2SKarsten Graul ini.ib_port = link->ibport; 10902d2209f2SKarsten Graul } 10912d2209f2SKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 10922d2209f2SKarsten Graul if (lnk_idx < 0) 10932d2209f2SKarsten Graul return 0; 10942d2209f2SKarsten Graul 10952d2209f2SKarsten Graul rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); 10962d2209f2SKarsten Graul if (rc) 10972d2209f2SKarsten Graul return rc; 10982d2209f2SKarsten Graul link_new = &lgr->lnk[lnk_idx]; 10992d2209f2SKarsten Graul rc = smc_llc_send_add_link(link, 11002d2209f2SKarsten Graul link_new->smcibdev->mac[ini.ib_port - 1], 11012d2209f2SKarsten Graul link_new->gid, link_new, SMC_LLC_REQ); 11022d2209f2SKarsten Graul if (rc) 11032d2209f2SKarsten Graul goto out_err; 11042d2209f2SKarsten Graul /* receive ADD LINK response over the RoCE fabric */ 11052d2209f2SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 11062d2209f2SKarsten Graul if (!qentry) { 11072d2209f2SKarsten Graul rc = -ETIMEDOUT; 11082d2209f2SKarsten Graul goto out_err; 11092d2209f2SKarsten Graul } 11102d2209f2SKarsten Graul add_llc = &qentry->msg.add_link; 11112d2209f2SKarsten Graul if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 11122d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11132d2209f2SKarsten Graul rc = -ENOLINK; 11142d2209f2SKarsten Graul goto out_err; 11152d2209f2SKarsten Graul } 11162d2209f2SKarsten Graul if (lgr->type == SMC_LGR_SINGLE && 11172d2209f2SKarsten Graul (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 11182d2209f2SKarsten Graul !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { 11192d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 11202d2209f2SKarsten Graul } 11212d2209f2SKarsten Graul smc_llc_save_add_link_info(link_new, add_llc); 11222d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11232d2209f2SKarsten Graul 11242d2209f2SKarsten Graul rc = smc_ib_ready_link(link_new); 11252d2209f2SKarsten Graul if (rc) 11262d2209f2SKarsten Graul goto out_err; 11272d2209f2SKarsten Graul rc = smcr_buf_map_lgr(link_new); 11282d2209f2SKarsten Graul if (rc) 11292d2209f2SKarsten Graul goto out_err; 11302d2209f2SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 11312d2209f2SKarsten Graul if (rc) 11322d2209f2SKarsten Graul goto out_err; 113357b49924SKarsten Graul rc = smc_llc_srv_rkey_exchange(link, link_new); 11342d2209f2SKarsten Graul if (rc) 11352d2209f2SKarsten Graul goto out_err; 11361551c95bSKarsten Graul rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 11372d2209f2SKarsten Graul if (rc) 11382d2209f2SKarsten Graul goto out_err; 11392d2209f2SKarsten Graul return 0; 11402d2209f2SKarsten Graul out_err: 11410a99be43SKarsten Graul smcr_link_clear(link_new, false); 11422d2209f2SKarsten Graul return rc; 11432d2209f2SKarsten Graul } 11442d2209f2SKarsten Graul 11452d2209f2SKarsten Graul static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 11462d2209f2SKarsten Graul { 11472d2209f2SKarsten Graul struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 11482d2209f2SKarsten Graul int rc; 11492d2209f2SKarsten Graul 11502d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11512d2209f2SKarsten Graul 11522d2209f2SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 11532d2209f2SKarsten Graul rc = smc_llc_srv_add_link(link); 11542d2209f2SKarsten Graul if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 11552d2209f2SKarsten Graul /* delete any asymmetric link */ 1156c9a5d243SKarsten Graul smc_llc_delete_asym_link(lgr); 11572d2209f2SKarsten Graul } 11582d2209f2SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 11592d2209f2SKarsten Graul } 11602d2209f2SKarsten Graul 11614dadd151SKarsten Graul /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */ 11624dadd151SKarsten Graul void smc_llc_srv_add_link_local(struct smc_link *link) 11634dadd151SKarsten Graul { 11644dadd151SKarsten Graul struct smc_llc_msg_add_link add_llc = {0}; 11654dadd151SKarsten Graul 11664dadd151SKarsten Graul add_llc.hd.length = sizeof(add_llc); 11674dadd151SKarsten Graul add_llc.hd.common.type = SMC_LLC_ADD_LINK; 11684dadd151SKarsten Graul /* no dev and port needed, we as server ignore client data anyway */ 11694dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 11704dadd151SKarsten Graul } 11714dadd151SKarsten Graul 1172b45e7f98SKarsten Graul /* worker to process an add link message */ 1173b45e7f98SKarsten Graul static void smc_llc_add_link_work(struct work_struct *work) 1174b45e7f98SKarsten Graul { 1175b45e7f98SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1176b45e7f98SKarsten Graul llc_add_link_work); 1177b45e7f98SKarsten Graul 1178b45e7f98SKarsten Graul if (list_empty(&lgr->list)) { 1179b45e7f98SKarsten Graul /* link group is terminating */ 1180b45e7f98SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1181b45e7f98SKarsten Graul goto out; 1182b45e7f98SKarsten Graul } 1183b45e7f98SKarsten Graul 1184b1570a87SKarsten Graul if (lgr->role == SMC_CLNT) 1185b1570a87SKarsten Graul smc_llc_process_cli_add_link(lgr); 11862d2209f2SKarsten Graul else 11872d2209f2SKarsten Graul smc_llc_process_srv_add_link(lgr); 1188b45e7f98SKarsten Graul out: 1189b45e7f98SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1190b45e7f98SKarsten Graul } 1191b45e7f98SKarsten Graul 11924dadd151SKarsten Graul /* enqueue a local del_link msg to trigger a new del_link flow, 11934dadd151SKarsten Graul * called only for role SMC_SERV 11944dadd151SKarsten Graul */ 11954dadd151SKarsten Graul void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 11964dadd151SKarsten Graul { 11974dadd151SKarsten Graul struct smc_llc_msg_del_link del_llc = {0}; 11984dadd151SKarsten Graul 11994dadd151SKarsten Graul del_llc.hd.length = sizeof(del_llc); 12004dadd151SKarsten Graul del_llc.hd.common.type = SMC_LLC_DELETE_LINK; 12014dadd151SKarsten Graul del_llc.link_num = del_link_id; 12024dadd151SKarsten Graul del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 12034dadd151SKarsten Graul del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 12044dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 12054dadd151SKarsten Graul } 12064dadd151SKarsten Graul 12079c416878SKarsten Graul static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 12089c416878SKarsten Graul { 12099c416878SKarsten Graul struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 12109c416878SKarsten Graul struct smc_llc_msg_del_link *del_llc; 12119c416878SKarsten Graul struct smc_llc_qentry *qentry; 12129c416878SKarsten Graul int active_links; 12139c416878SKarsten Graul int lnk_idx; 12149c416878SKarsten Graul 12159c416878SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 12169c416878SKarsten Graul lnk = qentry->link; 12179c416878SKarsten Graul del_llc = &qentry->msg.delete_link; 12189c416878SKarsten Graul 12199c416878SKarsten Graul if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 12209c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 12219c416878SKarsten Graul goto out; 12229c416878SKarsten Graul } 12239c416878SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 12249c416878SKarsten Graul /* delete single link */ 12259c416878SKarsten Graul for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 12269c416878SKarsten Graul if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 12279c416878SKarsten Graul continue; 12289c416878SKarsten Graul lnk_del = &lgr->lnk[lnk_idx]; 12299c416878SKarsten Graul break; 12309c416878SKarsten Graul } 12319c416878SKarsten Graul del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 12329c416878SKarsten Graul if (!lnk_del) { 12339c416878SKarsten Graul /* link was not found */ 12349c416878SKarsten Graul del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 12359c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); 12369c416878SKarsten Graul goto out_unlock; 12379c416878SKarsten Graul } 12389c416878SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 12399c416878SKarsten Graul 12409c416878SKarsten Graul del_llc->reason = 0; 12419c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); /* response */ 12429c416878SKarsten Graul 12439c416878SKarsten Graul if (smc_link_downing(&lnk_del->state)) { 1244c6f02ebeSKarsten Graul smc_switch_conns(lgr, lnk_del, false); 12459c416878SKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_del); 12469c416878SKarsten Graul } 12470a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 12489c416878SKarsten Graul 12499c416878SKarsten Graul active_links = smc_llc_active_link_count(lgr); 12509c416878SKarsten Graul if (lnk_del == lnk_asym) { 12519c416878SKarsten Graul /* expected deletion of asym link, don't change lgr state */ 12529c416878SKarsten Graul } else if (active_links == 1) { 1253ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 12549c416878SKarsten Graul } else if (!active_links) { 1255ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 12569c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 12579c416878SKarsten Graul } 12589c416878SKarsten Graul out_unlock: 12599c416878SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 12609c416878SKarsten Graul out: 12619c416878SKarsten Graul kfree(qentry); 12629c416878SKarsten Graul } 12639c416878SKarsten Graul 1264f3811fd7SKarsten Graul /* try to send a DELETE LINK ALL request on any active link, 1265f3811fd7SKarsten Graul * waiting for send completion 1266f3811fd7SKarsten Graul */ 1267f3811fd7SKarsten Graul void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1268f3811fd7SKarsten Graul { 1269f3811fd7SKarsten Graul struct smc_llc_msg_del_link delllc = {0}; 1270f3811fd7SKarsten Graul int i; 1271f3811fd7SKarsten Graul 1272f3811fd7SKarsten Graul delllc.hd.common.type = SMC_LLC_DELETE_LINK; 1273f3811fd7SKarsten Graul delllc.hd.length = sizeof(delllc); 1274f3811fd7SKarsten Graul if (ord) 1275f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1276f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1277f3811fd7SKarsten Graul delllc.reason = htonl(rsn); 1278f3811fd7SKarsten Graul 1279f3811fd7SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1280f3811fd7SKarsten Graul if (!smc_link_usable(&lgr->lnk[i])) 1281f3811fd7SKarsten Graul continue; 1282f3811fd7SKarsten Graul if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1283f3811fd7SKarsten Graul break; 1284f3811fd7SKarsten Graul } 1285f3811fd7SKarsten Graul } 1286f3811fd7SKarsten Graul 128708ae27ddSKarsten Graul static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 128808ae27ddSKarsten Graul { 128908ae27ddSKarsten Graul struct smc_llc_msg_del_link *del_llc; 129008ae27ddSKarsten Graul struct smc_link *lnk, *lnk_del; 129108ae27ddSKarsten Graul struct smc_llc_qentry *qentry; 129208ae27ddSKarsten Graul int active_links; 129308ae27ddSKarsten Graul int i; 129408ae27ddSKarsten Graul 129508ae27ddSKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 129608ae27ddSKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 129708ae27ddSKarsten Graul lnk = qentry->link; 129808ae27ddSKarsten Graul del_llc = &qentry->msg.delete_link; 129908ae27ddSKarsten Graul 130008ae27ddSKarsten Graul if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 130108ae27ddSKarsten Graul /* delete entire lgr */ 1302f3811fd7SKarsten Graul smc_llc_send_link_delete_all(lgr, true, ntohl( 1303f3811fd7SKarsten Graul qentry->msg.delete_link.reason)); 130408ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 130508ae27ddSKarsten Graul goto out; 130608ae27ddSKarsten Graul } 130708ae27ddSKarsten Graul /* delete single link */ 130808ae27ddSKarsten Graul lnk_del = NULL; 130908ae27ddSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 131008ae27ddSKarsten Graul if (lgr->lnk[i].link_id == del_llc->link_num) { 131108ae27ddSKarsten Graul lnk_del = &lgr->lnk[i]; 131208ae27ddSKarsten Graul break; 131308ae27ddSKarsten Graul } 131408ae27ddSKarsten Graul } 131508ae27ddSKarsten Graul if (!lnk_del) 131608ae27ddSKarsten Graul goto out; /* asymmetric link already deleted */ 131708ae27ddSKarsten Graul 131808ae27ddSKarsten Graul if (smc_link_downing(&lnk_del->state)) { 1319c6f02ebeSKarsten Graul smc_switch_conns(lgr, lnk_del, false); 132008ae27ddSKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_del); 132108ae27ddSKarsten Graul } 132208ae27ddSKarsten Graul if (!list_empty(&lgr->list)) { 132308ae27ddSKarsten Graul /* qentry is either a request from peer (send it back to 132408ae27ddSKarsten Graul * initiate the DELETE_LINK processing), or a locally 132508ae27ddSKarsten Graul * enqueued DELETE_LINK request (forward it) 132608ae27ddSKarsten Graul */ 132708ae27ddSKarsten Graul if (!smc_llc_send_message(lnk, &qentry->msg)) { 132808ae27ddSKarsten Graul struct smc_llc_qentry *qentry2; 132908ae27ddSKarsten Graul 133008ae27ddSKarsten Graul qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 133108ae27ddSKarsten Graul SMC_LLC_DELETE_LINK); 1332ca7e3edcSYueHaibing if (qentry2) 133308ae27ddSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 133408ae27ddSKarsten Graul } 133508ae27ddSKarsten Graul } 13360a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 133708ae27ddSKarsten Graul 133808ae27ddSKarsten Graul active_links = smc_llc_active_link_count(lgr); 133908ae27ddSKarsten Graul if (active_links == 1) { 1340ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 134108ae27ddSKarsten Graul } else if (!active_links) { 1342ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 134308ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 134408ae27ddSKarsten Graul } 134508ae27ddSKarsten Graul 134608ae27ddSKarsten Graul if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 134708ae27ddSKarsten Graul /* trigger setup of asymm alt link */ 13484dadd151SKarsten Graul smc_llc_srv_add_link_local(lnk); 134908ae27ddSKarsten Graul } 135008ae27ddSKarsten Graul out: 135108ae27ddSKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 135208ae27ddSKarsten Graul kfree(qentry); 135308ae27ddSKarsten Graul } 135408ae27ddSKarsten Graul 13559ec6bf19SKarsten Graul static void smc_llc_delete_link_work(struct work_struct *work) 135652bedf37SKarsten Graul { 13579ec6bf19SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 13589ec6bf19SKarsten Graul llc_del_link_work); 135952bedf37SKarsten Graul 13609ec6bf19SKarsten Graul if (list_empty(&lgr->list)) { 13619ec6bf19SKarsten Graul /* link group is terminating */ 13629ec6bf19SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 13639ec6bf19SKarsten Graul goto out; 136452bedf37SKarsten Graul } 13659c416878SKarsten Graul 13669c416878SKarsten Graul if (lgr->role == SMC_CLNT) 13679c416878SKarsten Graul smc_llc_process_cli_delete_link(lgr); 136808ae27ddSKarsten Graul else 136908ae27ddSKarsten Graul smc_llc_process_srv_delete_link(lgr); 13709ec6bf19SKarsten Graul out: 13719ec6bf19SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 137252bedf37SKarsten Graul } 137352bedf37SKarsten Graul 13743bc67e09SKarsten Graul /* process a confirm_rkey request from peer, remote flow */ 13753bc67e09SKarsten Graul static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 13764ed75de5SKarsten Graul { 13773bc67e09SKarsten Graul struct smc_llc_msg_confirm_rkey *llc; 13783bc67e09SKarsten Graul struct smc_llc_qentry *qentry; 13793bc67e09SKarsten Graul struct smc_link *link; 13803bc67e09SKarsten Graul int num_entries; 13813bc67e09SKarsten Graul int rk_idx; 13823bc67e09SKarsten Graul int i; 13834ed75de5SKarsten Graul 13843bc67e09SKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 13853bc67e09SKarsten Graul llc = &qentry->msg.confirm_rkey; 13863bc67e09SKarsten Graul link = qentry->link; 13873bc67e09SKarsten Graul 13883bc67e09SKarsten Graul num_entries = llc->rtoken[0].num_rkeys; 13893bc67e09SKarsten Graul /* first rkey entry is for receiving link */ 13903bc67e09SKarsten Graul rk_idx = smc_rtoken_add(link, 13914ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 13924ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 13933bc67e09SKarsten Graul if (rk_idx < 0) 13943bc67e09SKarsten Graul goto out_err; 13954ed75de5SKarsten Graul 13963bc67e09SKarsten Graul for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 13973bc67e09SKarsten Graul smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 13983bc67e09SKarsten Graul llc->rtoken[i].rmb_vaddr, 13993bc67e09SKarsten Graul llc->rtoken[i].rmb_key); 14003bc67e09SKarsten Graul /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 14013bc67e09SKarsten Graul goto out; 14023bc67e09SKarsten Graul out_err: 14034ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 14043bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 14053bc67e09SKarsten Graul out: 14063bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 14073bc67e09SKarsten Graul smc_llc_send_message(link, &qentry->msg); 14083bc67e09SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 14094ed75de5SKarsten Graul } 14104ed75de5SKarsten Graul 1411218b24feSKarsten Graul /* process a delete_rkey request from peer, remote flow */ 1412218b24feSKarsten Graul static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 14134ed75de5SKarsten Graul { 1414218b24feSKarsten Graul struct smc_llc_msg_delete_rkey *llc; 1415218b24feSKarsten Graul struct smc_llc_qentry *qentry; 1416218b24feSKarsten Graul struct smc_link *link; 14174ed75de5SKarsten Graul u8 err_mask = 0; 14184ed75de5SKarsten Graul int i, max; 14194ed75de5SKarsten Graul 1420218b24feSKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 1421218b24feSKarsten Graul llc = &qentry->msg.delete_rkey; 1422218b24feSKarsten Graul link = qentry->link; 1423218b24feSKarsten Graul 14244ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 14254ed75de5SKarsten Graul for (i = 0; i < max; i++) { 1426387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 14274ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 14284ed75de5SKarsten Graul } 14294ed75de5SKarsten Graul if (err_mask) { 14304ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 14314ed75de5SKarsten Graul llc->err_mask = err_mask; 14324ed75de5SKarsten Graul } 1433218b24feSKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 1434218b24feSKarsten Graul smc_llc_send_message(link, &qentry->msg); 1435218b24feSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1436218b24feSKarsten Graul } 14374ed75de5SKarsten Graul 14383e0c40afSKarsten Graul static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 14393e0c40afSKarsten Graul { 14403e0c40afSKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " 14413e0c40afSKarsten Graul "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); 14423e0c40afSKarsten Graul smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 14433e0c40afSKarsten Graul smc_lgr_terminate_sched(lgr); 14443e0c40afSKarsten Graul } 14453e0c40afSKarsten Graul 14466c8968c4SKarsten Graul /* flush the llc event queue */ 144700a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 14489bf9abeaSUrsula Braun { 14496c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 14509bf9abeaSUrsula Braun 14516c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 14526c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 14536c8968c4SKarsten Graul list_del_init(&qentry->list); 14546c8968c4SKarsten Graul kfree(qentry); 14556c8968c4SKarsten Graul } 14566c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 14576c8968c4SKarsten Graul } 14586c8968c4SKarsten Graul 14596c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 14606c8968c4SKarsten Graul { 14616c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 14626c8968c4SKarsten Graul struct smc_link *link = qentry->link; 14630fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 14646c8968c4SKarsten Graul 1465d854fcbfSKarsten Graul if (!smc_link_usable(link)) 14666c8968c4SKarsten Graul goto out; 1467313164daSKarsten Graul 1468313164daSKarsten Graul switch (llc->raw.hdr.common.type) { 1469313164daSKarsten Graul case SMC_LLC_TEST_LINK: 147056e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 147156e8091cSKarsten Graul smc_llc_send_message(link, llc); 1472313164daSKarsten Graul break; 147352bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 14740fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 14750fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 14760fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 14770fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 14780fb0b02bSKarsten Graul /* a flow is waiting for this message */ 14790fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 14800fb0b02bSKarsten Graul qentry); 1481*6778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 14820fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 14830fb0b02bSKarsten Graul qentry)) { 1484b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 14850fb0b02bSKarsten Graul } 14860fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 14870fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 1488b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 14890fb0b02bSKarsten Graul } 14900fb0b02bSKarsten Graul return; 14910fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 149287f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 14930fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 14940fb0b02bSKarsten Graul /* a flow is waiting for this message */ 14950fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 1496*6778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 14970fb0b02bSKarsten Graul return; 14980fb0b02bSKarsten Graul } 149952bedf37SKarsten Graul break; 150052bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 15019ec6bf19SKarsten Graul if (lgr->role == SMC_CLNT) { 15029ec6bf19SKarsten Graul /* server requests to delete this link, send response */ 15039ec6bf19SKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 15049ec6bf19SKarsten Graul /* DEL LINK REQ during ADD LINK SEQ */ 15059ec6bf19SKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 15069ec6bf19SKarsten Graul qentry); 1507*6778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 15089ec6bf19SKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 15099ec6bf19SKarsten Graul qentry)) { 15109ec6bf19SKarsten Graul schedule_work(&lgr->llc_del_link_work); 15119ec6bf19SKarsten Graul } 15129ec6bf19SKarsten Graul } else { 15139ec6bf19SKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 15149ec6bf19SKarsten Graul !lgr->llc_flow_lcl.qentry) { 15159ec6bf19SKarsten Graul /* DEL LINK REQ during ADD LINK SEQ */ 15169ec6bf19SKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 15179ec6bf19SKarsten Graul qentry); 1518*6778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 15199ec6bf19SKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 15209ec6bf19SKarsten Graul qentry)) { 15219ec6bf19SKarsten Graul schedule_work(&lgr->llc_del_link_work); 15229ec6bf19SKarsten Graul } 15239ec6bf19SKarsten Graul } 15249ec6bf19SKarsten Graul return; 15254ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 15263bc67e09SKarsten Graul /* new request from remote, assign to remote flow */ 15273bc67e09SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 15283bc67e09SKarsten Graul /* process here, does not wait for more llc msgs */ 15293bc67e09SKarsten Graul smc_llc_rmt_conf_rkey(lgr); 15303bc67e09SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 15313bc67e09SKarsten Graul } 15323bc67e09SKarsten Graul return; 15334ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 153442d18accSKarsten Graul /* not used because max links is 3, and 3 rkeys fit into 153542d18accSKarsten Graul * one CONFIRM_RKEY message 153642d18accSKarsten Graul */ 15374ed75de5SKarsten Graul break; 15384ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 1539218b24feSKarsten Graul /* new request from remote, assign to remote flow */ 1540218b24feSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1541218b24feSKarsten Graul /* process here, does not wait for more llc msgs */ 1542218b24feSKarsten Graul smc_llc_rmt_delete_rkey(lgr); 1543218b24feSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1544218b24feSKarsten Graul } 1545218b24feSKarsten Graul return; 15463e0c40afSKarsten Graul default: 15473e0c40afSKarsten Graul smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 15483e0c40afSKarsten Graul break; 1549313164daSKarsten Graul } 15506c8968c4SKarsten Graul out: 15516c8968c4SKarsten Graul kfree(qentry); 15526c8968c4SKarsten Graul } 15536c8968c4SKarsten Graul 15546c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 15556c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 15566c8968c4SKarsten Graul { 15576c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 15586c8968c4SKarsten Graul llc_event_work); 15596c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 15606c8968c4SKarsten Graul 1561555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1562555da9afSKarsten Graul if (smc_link_usable(lgr->delayed_event->link)) { 1563555da9afSKarsten Graul smc_llc_event_handler(lgr->delayed_event); 1564555da9afSKarsten Graul } else { 1565555da9afSKarsten Graul qentry = lgr->delayed_event; 1566555da9afSKarsten Graul lgr->delayed_event = NULL; 1567555da9afSKarsten Graul kfree(qentry); 1568555da9afSKarsten Graul } 1569555da9afSKarsten Graul } 1570555da9afSKarsten Graul 15716c8968c4SKarsten Graul again: 15726c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 15736c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 15746c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 15756c8968c4SKarsten Graul struct smc_llc_qentry, list); 15766c8968c4SKarsten Graul list_del_init(&qentry->list); 15776c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 15786c8968c4SKarsten Graul smc_llc_event_handler(qentry); 15796c8968c4SKarsten Graul goto again; 15806c8968c4SKarsten Graul } 15816c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 15826c8968c4SKarsten Graul } 15836c8968c4SKarsten Graul 1584ef79d439SKarsten Graul /* process llc responses in tasklet context */ 1585a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 1586a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 1587ef79d439SKarsten Graul { 1588a6688d91SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.type; 1589ef79d439SKarsten Graul 1590a6688d91SKarsten Graul switch (llc_type) { 1591ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 1592ef79d439SKarsten Graul if (link->state == SMC_LNK_ACTIVE) 1593ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 1594ef79d439SKarsten Graul break; 1595ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 15969ec6bf19SKarsten Graul case SMC_LLC_DELETE_LINK: 15974667bb4aSKarsten Graul case SMC_LLC_CONFIRM_LINK: 159887f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 15993d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 16006d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 16014667bb4aSKarsten Graul /* assign responses to the local flow, we requested them */ 16024667bb4aSKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 1603*6778a6beSKarsten Graul wake_up(&link->lgr->llc_msg_waiter); 16044667bb4aSKarsten Graul return; 1605ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 160642d18accSKarsten Graul /* not used because max links is 3 */ 1607ef79d439SKarsten Graul break; 16083e0c40afSKarsten Graul default: 16093e0c40afSKarsten Graul smc_llc_protocol_violation(link->lgr, llc_type); 16103e0c40afSKarsten Graul break; 1611ef79d439SKarsten Graul } 1612a6688d91SKarsten Graul kfree(qentry); 1613ef79d439SKarsten Graul } 1614ef79d439SKarsten Graul 1615a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 16166c8968c4SKarsten Graul { 16176c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 16186c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 16196c8968c4SKarsten Graul unsigned long flags; 16206c8968c4SKarsten Graul 16216c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 16226c8968c4SKarsten Graul if (!qentry) 16236c8968c4SKarsten Graul return; 16246c8968c4SKarsten Graul qentry->link = link; 16256c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 16266c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 1627a6688d91SKarsten Graul 1628a6688d91SKarsten Graul /* process responses immediately */ 1629a6688d91SKarsten Graul if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 1630a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 1631a6688d91SKarsten Graul return; 1632a6688d91SKarsten Graul } 1633a6688d91SKarsten Graul 1634a6688d91SKarsten Graul /* add requests to event queue */ 16356c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 16366c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 16376c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 1638*6778a6beSKarsten Graul schedule_work(&lgr->llc_event_work); 16399bf9abeaSUrsula Braun } 16409bf9abeaSUrsula Braun 1641a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 1642a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 1643a6688d91SKarsten Graul { 1644a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 1645a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 1646a6688d91SKarsten Graul 1647a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 1648a6688d91SKarsten Graul return; /* short message */ 1649a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 1650a6688d91SKarsten Graul return; /* invalid message */ 1651a6688d91SKarsten Graul 1652a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 1653a6688d91SKarsten Graul } 1654a6688d91SKarsten Graul 165544aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 1656877ae5beSKarsten Graul 1657877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 1658877ae5beSKarsten Graul { 1659877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 1660877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 1661877ae5beSKarsten Graul unsigned long next_interval; 1662877ae5beSKarsten Graul unsigned long expire_time; 1663877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 1664877ae5beSKarsten Graul int rc; 1665877ae5beSKarsten Graul 1666877ae5beSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 1667877ae5beSKarsten Graul return; /* don't reschedule worker */ 1668877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 1669877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 1670877ae5beSKarsten Graul next_interval = expire_time - jiffies; 1671877ae5beSKarsten Graul goto out; 1672877ae5beSKarsten Graul } 1673877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 1674d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 1675877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 1676877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 1677877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 16781020e1efSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 16791020e1efSKarsten Graul return; /* link state changed */ 1680877ae5beSKarsten Graul if (rc <= 0) { 168187523930SKarsten Graul smcr_link_down_cond_sched(link); 1682877ae5beSKarsten Graul return; 1683877ae5beSKarsten Graul } 1684877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 1685877ae5beSKarsten Graul out: 16861020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 1687877ae5beSKarsten Graul } 1688877ae5beSKarsten Graul 168900a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 169000a049cfSKarsten Graul { 169100a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 169200a049cfSKarsten Graul 169300a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 1694b45e7f98SKarsten Graul INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 16959ec6bf19SKarsten Graul INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 169600a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 169700a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 1698555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 1699*6778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_flow_waiter); 1700*6778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_msg_waiter); 1701d5500667SKarsten Graul mutex_init(&lgr->llc_conf_mutex); 170200a049cfSKarsten Graul lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 170300a049cfSKarsten Graul } 170400a049cfSKarsten Graul 170500a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 170600a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 170700a049cfSKarsten Graul { 170800a049cfSKarsten Graul smc_llc_event_flush(lgr); 1709*6778a6beSKarsten Graul wake_up_all(&lgr->llc_flow_waiter); 1710*6778a6beSKarsten Graul wake_up_all(&lgr->llc_msg_waiter); 171100a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 1712b45e7f98SKarsten Graul cancel_work_sync(&lgr->llc_add_link_work); 17139ec6bf19SKarsten Graul cancel_work_sync(&lgr->llc_del_link_work); 1714555da9afSKarsten Graul if (lgr->delayed_event) { 1715555da9afSKarsten Graul kfree(lgr->delayed_event); 1716555da9afSKarsten Graul lgr->delayed_event = NULL; 1717555da9afSKarsten Graul } 171800a049cfSKarsten Graul } 171900a049cfSKarsten Graul 17202a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 1721877ae5beSKarsten Graul { 1722877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 1723877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 17242a4c57a9SKarsten Graul return 0; 1725b32cf4abSKarsten Graul } 1726b32cf4abSKarsten Graul 172700a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 1728b32cf4abSKarsten Graul { 17290a99be43SKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " 17300a99be43SKarsten Graul "peerid %*phN, ibdev %s, ibport %d\n", 17310a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 17320a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 17330a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 17340a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 1735877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 173600a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 173700a049cfSKarsten Graul link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 17381020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 1739877ae5beSKarsten Graul link->llc_testlink_time); 1740877ae5beSKarsten Graul } 1741877ae5beSKarsten Graul } 1742877ae5beSKarsten Graul 1743877ae5beSKarsten Graul /* called in worker context */ 17440a99be43SKarsten Graul void smc_llc_link_clear(struct smc_link *link, bool log) 1745877ae5beSKarsten Graul { 17460a99be43SKarsten Graul if (log) 17470a99be43SKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" 17480a99be43SKarsten Graul ", peerid %*phN, ibdev %s, ibport %d\n", 17490a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 17500a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 17510a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 17520a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 17532140ac26SKarsten Graul complete(&link->llc_testlink_resp); 17542140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 17552140ac26SKarsten Graul smc_wr_wakeup_reg_wait(link); 17562140ac26SKarsten Graul smc_wr_wakeup_tx_wait(link); 1757877ae5beSKarsten Graul } 1758877ae5beSKarsten Graul 17593d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 17603d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 176144aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 176244aa81ceSKarsten Graul { 17633d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 17643d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 17653d88a21bSKarsten Graul int rc = 0; 176644aa81ceSKarsten Graul 17673d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 17683d88a21bSKarsten Graul if (rc) 17693d88a21bSKarsten Graul goto out; 177044aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 17713d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 17723d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 17733d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 17743d88a21bSKarsten Graul rc = -EFAULT; 17753d88a21bSKarsten Graul out: 17763d88a21bSKarsten Graul if (qentry) 17773d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 17783d88a21bSKarsten Graul return rc; 177944aa81ceSKarsten Graul } 178044aa81ceSKarsten Graul 178160e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 17826d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 178360e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 178460e03c62SKarsten Graul { 17856d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 17866d74c3a8SKarsten Graul struct smc_link *send_link; 17870b29ec64SUrsula Braun int rc = 0; 178860e03c62SKarsten Graul 17896d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 17906d74c3a8SKarsten Graul if (!send_link) 17916d74c3a8SKarsten Graul return -ENOLINK; 17926d74c3a8SKarsten Graul 17936d74c3a8SKarsten Graul /* protected by llc_flow control */ 17946d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 179560e03c62SKarsten Graul if (rc) 179660e03c62SKarsten Graul goto out; 179760e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 17986d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 17996d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 18006d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 180160e03c62SKarsten Graul rc = -EFAULT; 180260e03c62SKarsten Graul out: 18036d74c3a8SKarsten Graul if (qentry) 18046d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 180560e03c62SKarsten Graul return rc; 180660e03c62SKarsten Graul } 180760e03c62SKarsten Graul 180845fa8da0SKarsten Graul void smc_llc_link_set_uid(struct smc_link *link) 180945fa8da0SKarsten Graul { 181045fa8da0SKarsten Graul __be32 link_uid; 181145fa8da0SKarsten Graul 181245fa8da0SKarsten Graul link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 181345fa8da0SKarsten Graul memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 181445fa8da0SKarsten Graul } 181545fa8da0SKarsten Graul 1816649758ffSKarsten Graul /* save peers link user id, used for debug purposes */ 1817649758ffSKarsten Graul void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 1818649758ffSKarsten Graul { 1819649758ffSKarsten Graul memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 1820649758ffSKarsten Graul SMC_LGR_ID_SIZE); 1821649758ffSKarsten Graul } 1822649758ffSKarsten Graul 182392334cfcSKarsten Graul /* evaluate confirm link request or response */ 182492334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 182592334cfcSKarsten Graul enum smc_llc_reqresp type) 182692334cfcSKarsten Graul { 182745fa8da0SKarsten Graul if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 182892334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 182945fa8da0SKarsten Graul smc_llc_link_set_uid(qentry->link); 183045fa8da0SKarsten Graul } 183192334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 183292334cfcSKarsten Graul return -ENOTSUPP; 183392334cfcSKarsten Graul return 0; 183492334cfcSKarsten Graul } 183592334cfcSKarsten Graul 18369bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 18379bf9abeaSUrsula Braun 18389bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 18399bf9abeaSUrsula Braun { 18409bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 18419bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 18429bf9abeaSUrsula Braun }, 18439bf9abeaSUrsula Braun { 1844313164daSKarsten Graul .handler = smc_llc_rx_handler, 1845313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 1846313164daSKarsten Graul }, 1847313164daSKarsten Graul { 18484ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 184952bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 185052bedf37SKarsten Graul }, 185152bedf37SKarsten Graul { 185252bedf37SKarsten Graul .handler = smc_llc_rx_handler, 185387f88cdaSKarsten Graul .type = SMC_LLC_ADD_LINK_CONT 185487f88cdaSKarsten Graul }, 185587f88cdaSKarsten Graul { 185687f88cdaSKarsten Graul .handler = smc_llc_rx_handler, 185752bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 185852bedf37SKarsten Graul }, 185952bedf37SKarsten Graul { 186052bedf37SKarsten Graul .handler = smc_llc_rx_handler, 18614ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 18624ed75de5SKarsten Graul }, 18634ed75de5SKarsten Graul { 18644ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 18654ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 18664ed75de5SKarsten Graul }, 18674ed75de5SKarsten Graul { 18684ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 18694ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 18704ed75de5SKarsten Graul }, 18714ed75de5SKarsten Graul { 18729bf9abeaSUrsula Braun .handler = NULL, 18739bf9abeaSUrsula Braun } 18749bf9abeaSUrsula Braun }; 18759bf9abeaSUrsula Braun 18769bf9abeaSUrsula Braun int __init smc_llc_init(void) 18779bf9abeaSUrsula Braun { 18789bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 18799bf9abeaSUrsula Braun int rc = 0; 18809bf9abeaSUrsula Braun 18819bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 18829bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 18839bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 18849bf9abeaSUrsula Braun if (rc) 18859bf9abeaSUrsula Braun break; 18869bf9abeaSUrsula Braun } 18879bf9abeaSUrsula Braun return rc; 18889bf9abeaSUrsula Braun } 1889