1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 20336ba09fSKarsten Graul #include "smc_pnet.h" 219bf9abeaSUrsula Braun 220f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 230f627126SStefan Raspl 240f627126SStefan Raspl struct smc_llc_hdr { 250f627126SStefan Raspl struct smc_wr_rx_hdr common; 260f627126SStefan Raspl u8 length; /* 44 */ 2752bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 2852bedf37SKarsten Graul u8 reserved:4, 2952bedf37SKarsten Graul add_link_rej_rsn:4; 3052bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3152bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3252bedf37SKarsten Graul reserved:4; 3352bedf37SKarsten Graul #endif 340f627126SStefan Raspl u8 flags; 350f627126SStefan Raspl }; 360f627126SStefan Raspl 3775d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 3875d320d6SKarsten Graul 390f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 400f627126SStefan Raspl struct smc_llc_hdr hd; 410f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 420f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 430f627126SStefan Raspl u8 sender_qp_num[3]; 440f627126SStefan Raspl u8 link_num; 450f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 460f627126SStefan Raspl u8 max_links; 470f627126SStefan Raspl u8 reserved[9]; 480f627126SStefan Raspl }; 490f627126SStefan Raspl 5052bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5152bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 5252bedf37SKarsten Graul 5352bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 5452bedf37SKarsten Graul 5552bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 5652bedf37SKarsten Graul struct smc_llc_hdr hd; 5752bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 5852bedf37SKarsten Graul u8 reserved2[2]; 5952bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 6052bedf37SKarsten Graul u8 sender_qp_num[3]; 6152bedf37SKarsten Graul u8 link_num; 62fbed3b37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 63fbed3b37SKarsten Graul u8 reserved3 : 4, 64fbed3b37SKarsten Graul qp_mtu : 4; 65fbed3b37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 66fbed3b37SKarsten Graul u8 qp_mtu : 4, 67fbed3b37SKarsten Graul reserved3 : 4; 68fbed3b37SKarsten Graul #endif 6952bedf37SKarsten Graul u8 initial_psn[3]; 7052bedf37SKarsten Graul u8 reserved[8]; 7152bedf37SKarsten Graul }; 7252bedf37SKarsten Graul 7387f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt { 7487f88cdaSKarsten Graul __be32 rmb_key; 7587f88cdaSKarsten Graul __be32 rmb_key_new; 7687f88cdaSKarsten Graul __be64 rmb_vaddr_new; 7787f88cdaSKarsten Graul }; 7887f88cdaSKarsten Graul 7987f88cdaSKarsten Graul #define SMC_LLC_RKEYS_PER_CONT_MSG 2 8087f88cdaSKarsten Graul 8187f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont { /* type 0x03 */ 8287f88cdaSKarsten Graul struct smc_llc_hdr hd; 8387f88cdaSKarsten Graul u8 link_num; 8487f88cdaSKarsten Graul u8 num_rkeys; 8587f88cdaSKarsten Graul u8 reserved2[2]; 8687f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 8787f88cdaSKarsten Graul u8 reserved[4]; 8887f88cdaSKarsten Graul } __packed; /* format defined in RFC7609 */ 8987f88cdaSKarsten Graul 9052bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 9152bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 9252bedf37SKarsten Graul 9352bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 9452bedf37SKarsten Graul struct smc_llc_hdr hd; 9552bedf37SKarsten Graul u8 link_num; 9652bedf37SKarsten Graul __be32 reason; 9752bedf37SKarsten Graul u8 reserved[35]; 9852bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 9952bedf37SKarsten Graul 100313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 101313164daSKarsten Graul struct smc_llc_hdr hd; 102313164daSKarsten Graul u8 user_data[16]; 103313164daSKarsten Graul u8 reserved[24]; 104313164daSKarsten Graul }; 105313164daSKarsten Graul 1064ed75de5SKarsten Graul struct smc_rmb_rtoken { 1074ed75de5SKarsten Graul union { 1084ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 1094ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 1104ed75de5SKarsten Graul /* rtoken is always for the current link */ 1114ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 1124ed75de5SKarsten Graul }; 1134ed75de5SKarsten Graul __be32 rmb_key; 1144ed75de5SKarsten Graul __be64 rmb_vaddr; 1154ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 1164ed75de5SKarsten Graul 1174ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 1184ed75de5SKarsten Graul 1194ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 1204ed75de5SKarsten Graul struct smc_llc_hdr hd; 1214ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1224ed75de5SKarsten Graul u8 reserved; 1234ed75de5SKarsten Graul }; 1244ed75de5SKarsten Graul 1254ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 1263bc67e09SKarsten Graul #define SMC_LLC_FLAG_RKEY_RETRY 0x10 1274ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1284ed75de5SKarsten Graul 1294ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1304ed75de5SKarsten Graul struct smc_llc_hdr hd; 1314ed75de5SKarsten Graul u8 num_rkeys; 1324ed75de5SKarsten Graul u8 err_mask; 1334ed75de5SKarsten Graul u8 reserved[2]; 1344ed75de5SKarsten Graul __be32 rkey[8]; 1354ed75de5SKarsten Graul u8 reserved2[4]; 1364ed75de5SKarsten Graul }; 1374ed75de5SKarsten Graul 1380f627126SStefan Raspl union smc_llc_msg { 1390f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 14052bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 14187f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont add_link_cont; 14252bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1434ed75de5SKarsten Graul 1444ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1454ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1464ed75de5SKarsten Graul 147313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1480f627126SStefan Raspl struct { 1490f627126SStefan Raspl struct smc_llc_hdr hdr; 1500f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1510f627126SStefan Raspl } raw; 1520f627126SStefan Raspl }; 1530f627126SStefan Raspl 1540f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1550f627126SStefan Raspl 1566c8968c4SKarsten Graul struct smc_llc_qentry { 1576c8968c4SKarsten Graul struct list_head list; 1586c8968c4SKarsten Graul struct smc_link *link; 1596c8968c4SKarsten Graul union smc_llc_msg msg; 1606c8968c4SKarsten Graul }; 1616c8968c4SKarsten Graul 1624dadd151SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 1634dadd151SKarsten Graul 164555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 165555da9afSKarsten Graul { 166555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 167555da9afSKarsten Graul 168555da9afSKarsten Graul flow->qentry = NULL; 169555da9afSKarsten Graul return qentry; 170555da9afSKarsten Graul } 171555da9afSKarsten Graul 172555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 173555da9afSKarsten Graul { 174555da9afSKarsten Graul struct smc_llc_qentry *qentry; 175555da9afSKarsten Graul 176555da9afSKarsten Graul if (flow->qentry) { 177555da9afSKarsten Graul qentry = flow->qentry; 178555da9afSKarsten Graul flow->qentry = NULL; 179555da9afSKarsten Graul kfree(qentry); 180555da9afSKarsten Graul } 181555da9afSKarsten Graul } 182555da9afSKarsten Graul 183555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 184555da9afSKarsten Graul struct smc_llc_qentry *qentry) 185555da9afSKarsten Graul { 186555da9afSKarsten Graul flow->qentry = qentry; 187555da9afSKarsten Graul } 188555da9afSKarsten Graul 1896778a6beSKarsten Graul static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 1906778a6beSKarsten Graul struct smc_llc_qentry *qentry) 1916778a6beSKarsten Graul { 1926778a6beSKarsten Graul u8 msg_type = qentry->msg.raw.hdr.common.type; 1936778a6beSKarsten Graul 1946778a6beSKarsten Graul if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 1956778a6beSKarsten Graul flow_type != msg_type && !lgr->delayed_event) { 1966778a6beSKarsten Graul lgr->delayed_event = qentry; 1976778a6beSKarsten Graul return; 1986778a6beSKarsten Graul } 1996778a6beSKarsten Graul /* drop parallel or already-in-progress llc requests */ 2006778a6beSKarsten Graul if (flow_type != msg_type) 2016778a6beSKarsten Graul pr_warn_once("smc: SMC-R lg %*phN dropped parallel " 2026778a6beSKarsten Graul "LLC msg: msg %d flow %d role %d\n", 2036778a6beSKarsten Graul SMC_LGR_ID_SIZE, &lgr->id, 2046778a6beSKarsten Graul qentry->msg.raw.hdr.common.type, 2056778a6beSKarsten Graul flow_type, lgr->role); 2066778a6beSKarsten Graul kfree(qentry); 2076778a6beSKarsten Graul } 2086778a6beSKarsten Graul 209555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 210555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 211555da9afSKarsten Graul struct smc_llc_qentry *qentry) 212555da9afSKarsten Graul { 213555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 214555da9afSKarsten Graul 215555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 216555da9afSKarsten Graul if (flow->type) { 217555da9afSKarsten Graul /* a flow is already active */ 2186778a6beSKarsten Graul smc_llc_flow_parallel(lgr, flow->type, qentry); 219555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 220555da9afSKarsten Graul return false; 221555da9afSKarsten Graul } 222555da9afSKarsten Graul switch (qentry->msg.raw.hdr.common.type) { 223555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 224555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 225555da9afSKarsten Graul break; 226555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 227555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 228555da9afSKarsten Graul break; 229555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 230555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 231555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 232555da9afSKarsten Graul break; 233555da9afSKarsten Graul default: 234555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 235555da9afSKarsten Graul } 236555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 2376778a6beSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 238555da9afSKarsten Graul return true; 239555da9afSKarsten Graul } 240555da9afSKarsten Graul 241555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 242555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 243555da9afSKarsten Graul enum smc_llc_flowtype type) 244555da9afSKarsten Graul { 245555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 246555da9afSKarsten Graul int rc; 247555da9afSKarsten Graul 248555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 249555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 250555da9afSKarsten Graul */ 251555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 252555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 253555da9afSKarsten Graul again: 254555da9afSKarsten Graul if (list_empty(&lgr->list)) 255555da9afSKarsten Graul return -ENODEV; 256555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 257555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 258555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 259555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 260555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 261555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 262555da9afSKarsten Graul return 0; 263555da9afSKarsten Graul } 264555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 2656778a6beSKarsten Graul rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 266555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 267555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 2686778a6beSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote))), 2696778a6beSKarsten Graul SMC_LLC_WAIT_TIME * 10); 270555da9afSKarsten Graul if (!rc) 271555da9afSKarsten Graul return -ETIMEDOUT; 272555da9afSKarsten Graul goto again; 273555da9afSKarsten Graul } 274555da9afSKarsten Graul 275555da9afSKarsten Graul /* finish the current llc flow */ 276555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 277555da9afSKarsten Graul { 278555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 279555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 280555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 281555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 282555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 283555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 284555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 285555da9afSKarsten Graul else 2866778a6beSKarsten Graul wake_up(&lgr->llc_flow_waiter); 287555da9afSKarsten Graul } 288555da9afSKarsten Graul 289555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 290555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 291555da9afSKarsten Graul */ 292555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 293555da9afSKarsten Graul struct smc_link *lnk, 294555da9afSKarsten Graul int time_out, u8 exp_msg) 295555da9afSKarsten Graul { 296555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 2976778a6beSKarsten Graul u8 rcv_msg; 298555da9afSKarsten Graul 2996778a6beSKarsten Graul wait_event_timeout(lgr->llc_msg_waiter, 300555da9afSKarsten Graul (flow->qentry || 301555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 302555da9afSKarsten Graul list_empty(&lgr->list)), 303555da9afSKarsten Graul time_out); 304555da9afSKarsten Graul if (!flow->qentry || 305555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 306555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 307555da9afSKarsten Graul goto out; 308555da9afSKarsten Graul } 3096778a6beSKarsten Graul rcv_msg = flow->qentry->msg.raw.hdr.common.type; 3106778a6beSKarsten Graul if (exp_msg && rcv_msg != exp_msg) { 311555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 3126778a6beSKarsten Graul rcv_msg == SMC_LLC_DELETE_LINK) { 313555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 314555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 315555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 316555da9afSKarsten Graul return NULL; 317555da9afSKarsten Graul } 3186778a6beSKarsten Graul pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: " 3196778a6beSKarsten Graul "msg %d exp %d flow %d role %d flags %x\n", 3206778a6beSKarsten Graul SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg, 3216778a6beSKarsten Graul flow->type, lgr->role, 3226778a6beSKarsten Graul flow->qentry->msg.raw.hdr.flags); 323555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 324555da9afSKarsten Graul } 325555da9afSKarsten Graul out: 326555da9afSKarsten Graul return flow->qentry; 327555da9afSKarsten Graul } 328555da9afSKarsten Graul 3299bf9abeaSUrsula Braun /********************************** send *************************************/ 3309bf9abeaSUrsula Braun 3319bf9abeaSUrsula Braun struct smc_llc_tx_pend { 3329bf9abeaSUrsula Braun }; 3339bf9abeaSUrsula Braun 3349bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 3359bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 3369bf9abeaSUrsula Braun struct smc_link *link, 3379bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3389bf9abeaSUrsula Braun { 3399bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3409bf9abeaSUrsula Braun } 3419bf9abeaSUrsula Braun 3429bf9abeaSUrsula Braun /** 3439bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3449bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3459bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3469bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3479bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3489bf9abeaSUrsula Braun * 3499bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3509bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3519bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3529bf9abeaSUrsula Braun * 3539bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 3549bf9abeaSUrsula Braun */ 3559bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 3569bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 3579bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 3589bf9abeaSUrsula Braun { 3599bf9abeaSUrsula Braun int rc; 3609bf9abeaSUrsula Braun 361ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 362ad6f317fSUrsula Braun pend); 3639bf9abeaSUrsula Braun if (rc < 0) 3649bf9abeaSUrsula Braun return rc; 3659bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3669bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 3679bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 3689bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3699bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 3709bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 3719bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3729bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 3739bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 3749bf9abeaSUrsula Braun return 0; 3759bf9abeaSUrsula Braun } 3769bf9abeaSUrsula Braun 3779bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 378947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 3799bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 3809bf9abeaSUrsula Braun { 3819bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 3829bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 3839bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 3849bf9abeaSUrsula Braun int rc; 3859bf9abeaSUrsula Braun 3869bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 3879bf9abeaSUrsula Braun if (rc) 3889bf9abeaSUrsula Braun return rc; 3899bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 3909bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 3919bf9abeaSUrsula Braun confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 3929bf9abeaSUrsula Braun confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 39375d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 3949bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 3959bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 396947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 397947541f3SUrsula Braun ETH_ALEN); 3987005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 3999bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 4002be922f3SKarsten Graul confllc->link_num = link->link_id; 40145fa8da0SKarsten Graul memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 402b1570a87SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 40352bedf37SKarsten Graul /* send llc message */ 40452bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 40552bedf37SKarsten Graul return rc; 40652bedf37SKarsten Graul } 40752bedf37SKarsten Graul 40844aa81ceSKarsten Graul /* send LLC confirm rkey request */ 4093d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 41044aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 41144aa81ceSKarsten Graul { 41244aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 41344aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 41444aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 4153d88a21bSKarsten Graul struct smc_link *link; 4163d88a21bSKarsten Graul int i, rc, rtok_ix; 41744aa81ceSKarsten Graul 4183d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 41944aa81ceSKarsten Graul if (rc) 42044aa81ceSKarsten Graul return rc; 42144aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 42244aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 42344aa81ceSKarsten Graul rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 42444aa81ceSKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 4253d88a21bSKarsten Graul 4263d88a21bSKarsten Graul rtok_ix = 1; 4273d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 4283d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 429741a49a4SKarsten Graul if (smc_link_active(link) && link != send_link) { 4303d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 4313d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 432387707fdSKarsten Graul htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 4333d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 4343d88a21bSKarsten Graul (u64)sg_dma_address( 4353d88a21bSKarsten Graul rmb_desc->sgt[link->link_idx].sgl)); 4363d88a21bSKarsten Graul rtok_ix++; 4373d88a21bSKarsten Graul } 4383d88a21bSKarsten Graul } 4393d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 4403d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 4413d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 4423d88a21bSKarsten Graul htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 44344aa81ceSKarsten Graul rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 4443d88a21bSKarsten Graul (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 44544aa81ceSKarsten Graul /* send llc message */ 4463d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 44744aa81ceSKarsten Graul return rc; 44844aa81ceSKarsten Graul } 44944aa81ceSKarsten Graul 45060e03c62SKarsten Graul /* send LLC delete rkey request */ 45160e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 45260e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 45360e03c62SKarsten Graul { 45460e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 45560e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 45660e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 45760e03c62SKarsten Graul int rc; 45860e03c62SKarsten Graul 45960e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 46060e03c62SKarsten Graul if (rc) 46160e03c62SKarsten Graul return rc; 46260e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 46360e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 46460e03c62SKarsten Graul rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 46560e03c62SKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 46660e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 467387707fdSKarsten Graul rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 46860e03c62SKarsten Graul /* send llc message */ 46960e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 47060e03c62SKarsten Graul return rc; 47160e03c62SKarsten Graul } 47260e03c62SKarsten Graul 47352bedf37SKarsten Graul /* send ADD LINK request or response */ 4747005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 475fbed3b37SKarsten Graul struct smc_link *link_new, 47652bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 47752bedf37SKarsten Graul { 47852bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 47952bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 48052bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 48152bedf37SKarsten Graul int rc; 48252bedf37SKarsten Graul 48352bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 48452bedf37SKarsten Graul if (rc) 48552bedf37SKarsten Graul return rc; 48652bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 487fbed3b37SKarsten Graul 488fbed3b37SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 489fbed3b37SKarsten Graul addllc->hd.common.type = SMC_LLC_ADD_LINK; 490fbed3b37SKarsten Graul addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 491fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 492fbed3b37SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 493fbed3b37SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 494fbed3b37SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 495fbed3b37SKarsten Graul if (link_new) { 496fbed3b37SKarsten Graul addllc->link_num = link_new->link_id; 497fbed3b37SKarsten Graul hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 498fbed3b37SKarsten Graul hton24(addllc->initial_psn, link_new->psn_initial); 499fbed3b37SKarsten Graul if (reqresp == SMC_LLC_REQ) 500fbed3b37SKarsten Graul addllc->qp_mtu = link_new->path_mtu; 501fbed3b37SKarsten Graul else 502fbed3b37SKarsten Graul addllc->qp_mtu = min(link_new->path_mtu, 503fbed3b37SKarsten Graul link_new->peer_mtu); 504fbed3b37SKarsten Graul } 50552bedf37SKarsten Graul /* send llc message */ 50652bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 50752bedf37SKarsten Graul return rc; 50852bedf37SKarsten Graul } 50952bedf37SKarsten Graul 51052bedf37SKarsten Graul /* send DELETE LINK request or response */ 511fbed3b37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 512fbed3b37SKarsten Graul enum smc_llc_reqresp reqresp, bool orderly, 513fbed3b37SKarsten Graul u32 reason) 51452bedf37SKarsten Graul { 51552bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 51652bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 51752bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 51852bedf37SKarsten Graul int rc; 51952bedf37SKarsten Graul 52052bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 52152bedf37SKarsten Graul if (rc) 52252bedf37SKarsten Graul return rc; 52352bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 524fbed3b37SKarsten Graul 525fbed3b37SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 526fbed3b37SKarsten Graul delllc->hd.common.type = SMC_LLC_DELETE_LINK; 527fbed3b37SKarsten Graul delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 528fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 529fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 530fbed3b37SKarsten Graul if (orderly) 531fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 532fbed3b37SKarsten Graul if (link_del_id) 533fbed3b37SKarsten Graul delllc->link_num = link_del_id; 534fbed3b37SKarsten Graul else 535fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 536fbed3b37SKarsten Graul delllc->reason = htonl(reason); 5379bf9abeaSUrsula Braun /* send llc message */ 5389bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 5399bf9abeaSUrsula Braun return rc; 5409bf9abeaSUrsula Braun } 5419bf9abeaSUrsula Braun 542d97935faSKarsten Graul /* send LLC test link request */ 543d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 544313164daSKarsten Graul { 545313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 546313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 547313164daSKarsten Graul struct smc_wr_buf *wr_buf; 548313164daSKarsten Graul int rc; 549313164daSKarsten Graul 550313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 551313164daSKarsten Graul if (rc) 552313164daSKarsten Graul return rc; 553313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 554313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 555313164daSKarsten Graul testllc->hd.common.type = SMC_LLC_TEST_LINK; 556313164daSKarsten Graul testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 557313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 558313164daSKarsten Graul /* send llc message */ 559313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 560313164daSKarsten Graul return rc; 561313164daSKarsten Graul } 562313164daSKarsten Graul 5636c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 5646c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 5654ed75de5SKarsten Graul { 5664ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 5674ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 5684ed75de5SKarsten Graul int rc; 5694ed75de5SKarsten Graul 5706c8968c4SKarsten Graul if (!smc_link_usable(link)) 5716c8968c4SKarsten Graul return -ENOLINK; 5726c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 5734ed75de5SKarsten Graul if (rc) 5746c8968c4SKarsten Graul return rc; 5756c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 5766c8968c4SKarsten Graul return smc_wr_tx_send(link, pend); 5774ed75de5SKarsten Graul } 5784ed75de5SKarsten Graul 579f3811fd7SKarsten Graul /* schedule an llc send on link, may wait for buffers, 580f3811fd7SKarsten Graul * and wait for send completion notification. 581f3811fd7SKarsten Graul * @return 0 on success 582f3811fd7SKarsten Graul */ 583f3811fd7SKarsten Graul static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 584f3811fd7SKarsten Graul { 585f3811fd7SKarsten Graul struct smc_wr_tx_pend_priv *pend; 586f3811fd7SKarsten Graul struct smc_wr_buf *wr_buf; 587f3811fd7SKarsten Graul int rc; 588f3811fd7SKarsten Graul 589f3811fd7SKarsten Graul if (!smc_link_usable(link)) 590f3811fd7SKarsten Graul return -ENOLINK; 591f3811fd7SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 592f3811fd7SKarsten Graul if (rc) 593f3811fd7SKarsten Graul return rc; 594f3811fd7SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 595f3811fd7SKarsten Graul return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 596f3811fd7SKarsten Graul } 597f3811fd7SKarsten Graul 5989bf9abeaSUrsula Braun /********************************* receive ***********************************/ 5999bf9abeaSUrsula Braun 600336ba09fSKarsten Graul static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 601336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t) 602336ba09fSKarsten Graul { 603336ba09fSKarsten Graul int i; 604336ba09fSKarsten Graul 605336ba09fSKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 606336ba09fSKarsten Graul (lgr->type != SMC_LGR_SINGLE && 607336ba09fSKarsten Graul (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 608336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 609336ba09fSKarsten Graul return -EMLINK; 610336ba09fSKarsten Graul 611336ba09fSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 612336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 613336ba09fSKarsten Graul for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 614336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 615336ba09fSKarsten Graul return i; 616336ba09fSKarsten Graul } else { 617336ba09fSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 618336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 619336ba09fSKarsten Graul return i; 620336ba09fSKarsten Graul } 621336ba09fSKarsten Graul return -EMLINK; 622336ba09fSKarsten Graul } 623336ba09fSKarsten Graul 62487f88cdaSKarsten Graul /* return first buffer from any of the next buf lists */ 62587f88cdaSKarsten Graul static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 62687f88cdaSKarsten Graul int *buf_lst) 62787f88cdaSKarsten Graul { 62887f88cdaSKarsten Graul struct smc_buf_desc *buf_pos; 62987f88cdaSKarsten Graul 63087f88cdaSKarsten Graul while (*buf_lst < SMC_RMBE_SIZES) { 63187f88cdaSKarsten Graul buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 63287f88cdaSKarsten Graul struct smc_buf_desc, list); 63387f88cdaSKarsten Graul if (buf_pos) 63487f88cdaSKarsten Graul return buf_pos; 63587f88cdaSKarsten Graul (*buf_lst)++; 63687f88cdaSKarsten Graul } 63787f88cdaSKarsten Graul return NULL; 63887f88cdaSKarsten Graul } 63987f88cdaSKarsten Graul 64087f88cdaSKarsten Graul /* return next rmb from buffer lists */ 64187f88cdaSKarsten Graul static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 64287f88cdaSKarsten Graul int *buf_lst, 64387f88cdaSKarsten Graul struct smc_buf_desc *buf_pos) 64487f88cdaSKarsten Graul { 64587f88cdaSKarsten Graul struct smc_buf_desc *buf_next; 64687f88cdaSKarsten Graul 64787f88cdaSKarsten Graul if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 64887f88cdaSKarsten Graul (*buf_lst)++; 64987f88cdaSKarsten Graul return _smc_llc_get_next_rmb(lgr, buf_lst); 65087f88cdaSKarsten Graul } 65187f88cdaSKarsten Graul buf_next = list_next_entry(buf_pos, list); 65287f88cdaSKarsten Graul return buf_next; 65387f88cdaSKarsten Graul } 65487f88cdaSKarsten Graul 65587f88cdaSKarsten Graul static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 65687f88cdaSKarsten Graul int *buf_lst) 65787f88cdaSKarsten Graul { 65887f88cdaSKarsten Graul *buf_lst = 0; 65987f88cdaSKarsten Graul return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 66087f88cdaSKarsten Graul } 66187f88cdaSKarsten Graul 66287f88cdaSKarsten Graul /* send one add_link_continue msg */ 66387f88cdaSKarsten Graul static int smc_llc_add_link_cont(struct smc_link *link, 66487f88cdaSKarsten Graul struct smc_link *link_new, u8 *num_rkeys_todo, 66587f88cdaSKarsten Graul int *buf_lst, struct smc_buf_desc **buf_pos) 66687f88cdaSKarsten Graul { 66787f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 66887f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 66987f88cdaSKarsten Graul int prim_lnk_idx, lnk_idx, i, rc; 67087f88cdaSKarsten Graul struct smc_wr_tx_pend_priv *pend; 67187f88cdaSKarsten Graul struct smc_wr_buf *wr_buf; 67287f88cdaSKarsten Graul struct smc_buf_desc *rmb; 67387f88cdaSKarsten Graul u8 n; 67487f88cdaSKarsten Graul 67587f88cdaSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 67687f88cdaSKarsten Graul if (rc) 67787f88cdaSKarsten Graul return rc; 67887f88cdaSKarsten Graul addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 67987f88cdaSKarsten Graul memset(addc_llc, 0, sizeof(*addc_llc)); 68087f88cdaSKarsten Graul 68187f88cdaSKarsten Graul prim_lnk_idx = link->link_idx; 68287f88cdaSKarsten Graul lnk_idx = link_new->link_idx; 68387f88cdaSKarsten Graul addc_llc->link_num = link_new->link_id; 68487f88cdaSKarsten Graul addc_llc->num_rkeys = *num_rkeys_todo; 68587f88cdaSKarsten Graul n = *num_rkeys_todo; 68687f88cdaSKarsten Graul for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 68787f88cdaSKarsten Graul if (!*buf_pos) { 68887f88cdaSKarsten Graul addc_llc->num_rkeys = addc_llc->num_rkeys - 68987f88cdaSKarsten Graul *num_rkeys_todo; 69087f88cdaSKarsten Graul *num_rkeys_todo = 0; 69187f88cdaSKarsten Graul break; 69287f88cdaSKarsten Graul } 69387f88cdaSKarsten Graul rmb = *buf_pos; 69487f88cdaSKarsten Graul 69587f88cdaSKarsten Graul addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); 69687f88cdaSKarsten Graul addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); 69787f88cdaSKarsten Graul addc_llc->rt[i].rmb_vaddr_new = 69887f88cdaSKarsten Graul cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 69987f88cdaSKarsten Graul 70087f88cdaSKarsten Graul (*num_rkeys_todo)--; 70187f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 70287f88cdaSKarsten Graul while (*buf_pos && !(*buf_pos)->used) 70387f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 70487f88cdaSKarsten Graul } 70587f88cdaSKarsten Graul addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; 70687f88cdaSKarsten Graul addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 70787f88cdaSKarsten Graul if (lgr->role == SMC_CLNT) 70887f88cdaSKarsten Graul addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 70987f88cdaSKarsten Graul return smc_wr_tx_send(link, pend); 71087f88cdaSKarsten Graul } 71187f88cdaSKarsten Graul 71287f88cdaSKarsten Graul static int smc_llc_cli_rkey_exchange(struct smc_link *link, 71387f88cdaSKarsten Graul struct smc_link *link_new) 71487f88cdaSKarsten Graul { 71587f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 71687f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 71787f88cdaSKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 71887f88cdaSKarsten Graul struct smc_llc_qentry *qentry; 71987f88cdaSKarsten Graul struct smc_buf_desc *buf_pos; 72087f88cdaSKarsten Graul int buf_lst; 72187f88cdaSKarsten Graul int rc = 0; 72287f88cdaSKarsten Graul int i; 72387f88cdaSKarsten Graul 72487f88cdaSKarsten Graul mutex_lock(&lgr->rmbs_lock); 72587f88cdaSKarsten Graul num_rkeys_send = lgr->conns_num; 72687f88cdaSKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 72787f88cdaSKarsten Graul do { 72887f88cdaSKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 72987f88cdaSKarsten Graul SMC_LLC_ADD_LINK_CONT); 73087f88cdaSKarsten Graul if (!qentry) { 73187f88cdaSKarsten Graul rc = -ETIMEDOUT; 73287f88cdaSKarsten Graul break; 73387f88cdaSKarsten Graul } 73487f88cdaSKarsten Graul addc_llc = &qentry->msg.add_link_cont; 73587f88cdaSKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 73687f88cdaSKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 73787f88cdaSKarsten Graul for (i = 0; i < max; i++) { 73887f88cdaSKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 73987f88cdaSKarsten Graul addc_llc->rt[i].rmb_key, 74087f88cdaSKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 74187f88cdaSKarsten Graul addc_llc->rt[i].rmb_key_new); 74287f88cdaSKarsten Graul num_rkeys_recv--; 74387f88cdaSKarsten Graul } 74487f88cdaSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 74587f88cdaSKarsten Graul rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 74687f88cdaSKarsten Graul &buf_lst, &buf_pos); 74787f88cdaSKarsten Graul if (rc) 74887f88cdaSKarsten Graul break; 74987f88cdaSKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 75087f88cdaSKarsten Graul 75187f88cdaSKarsten Graul mutex_unlock(&lgr->rmbs_lock); 75287f88cdaSKarsten Graul return rc; 75387f88cdaSKarsten Graul } 75487f88cdaSKarsten Graul 755336ba09fSKarsten Graul /* prepare and send an add link reject response */ 756336ba09fSKarsten Graul static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 757336ba09fSKarsten Graul { 758336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 759336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 760336ba09fSKarsten Graul qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 761336ba09fSKarsten Graul return smc_llc_send_message(qentry->link, &qentry->msg); 762336ba09fSKarsten Graul } 763336ba09fSKarsten Graul 764b1570a87SKarsten Graul static int smc_llc_cli_conf_link(struct smc_link *link, 765b1570a87SKarsten Graul struct smc_init_info *ini, 766b1570a87SKarsten Graul struct smc_link *link_new, 767b1570a87SKarsten Graul enum smc_lgr_type lgr_new_t) 768b1570a87SKarsten Graul { 769b1570a87SKarsten Graul struct smc_link_group *lgr = link->lgr; 770b1570a87SKarsten Graul struct smc_llc_qentry *qentry = NULL; 771b1570a87SKarsten Graul int rc = 0; 772b1570a87SKarsten Graul 773b1570a87SKarsten Graul /* receive CONFIRM LINK request over RoCE fabric */ 774b1570a87SKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 775b1570a87SKarsten Graul if (!qentry) { 776b1570a87SKarsten Graul rc = smc_llc_send_delete_link(link, link_new->link_id, 777b1570a87SKarsten Graul SMC_LLC_REQ, false, 778b1570a87SKarsten Graul SMC_LLC_DEL_LOST_PATH); 779b1570a87SKarsten Graul return -ENOLINK; 780b1570a87SKarsten Graul } 781b1570a87SKarsten Graul if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 782b1570a87SKarsten Graul /* received DELETE_LINK instead */ 783b1570a87SKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 784b1570a87SKarsten Graul smc_llc_send_message(link, &qentry->msg); 785b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 786b1570a87SKarsten Graul return -ENOLINK; 787b1570a87SKarsten Graul } 788649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 789b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 790b1570a87SKarsten Graul 791b1570a87SKarsten Graul rc = smc_ib_modify_qp_rts(link_new); 792b1570a87SKarsten Graul if (rc) { 793b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 794b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 795b1570a87SKarsten Graul return -ENOLINK; 796b1570a87SKarsten Graul } 797b1570a87SKarsten Graul smc_wr_remember_qp_attr(link_new); 798b1570a87SKarsten Graul 799b1570a87SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 800b1570a87SKarsten Graul if (rc) { 801b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 802b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 803b1570a87SKarsten Graul return -ENOLINK; 804b1570a87SKarsten Graul } 805b1570a87SKarsten Graul 806b1570a87SKarsten Graul /* send CONFIRM LINK response over RoCE fabric */ 807b1570a87SKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 808b1570a87SKarsten Graul if (rc) { 809b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 810b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 811b1570a87SKarsten Graul return -ENOLINK; 812b1570a87SKarsten Graul } 813b1570a87SKarsten Graul smc_llc_link_active(link_new); 814ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 815ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 816ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 817ad6c111bSKarsten Graul else 818ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 819b1570a87SKarsten Graul return 0; 820b1570a87SKarsten Graul } 821b1570a87SKarsten Graul 822336ba09fSKarsten Graul static void smc_llc_save_add_link_info(struct smc_link *link, 823336ba09fSKarsten Graul struct smc_llc_msg_add_link *add_llc) 824336ba09fSKarsten Graul { 825336ba09fSKarsten Graul link->peer_qpn = ntoh24(add_llc->sender_qp_num); 826336ba09fSKarsten Graul memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 827336ba09fSKarsten Graul memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 828336ba09fSKarsten Graul link->peer_psn = ntoh24(add_llc->initial_psn); 829336ba09fSKarsten Graul link->peer_mtu = add_llc->qp_mtu; 830336ba09fSKarsten Graul } 831336ba09fSKarsten Graul 832336ba09fSKarsten Graul /* as an SMC client, process an add link request */ 833336ba09fSKarsten Graul int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 834336ba09fSKarsten Graul { 835336ba09fSKarsten Graul struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 836336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 837336ba09fSKarsten Graul struct smc_link_group *lgr = smc_get_lgr(link); 838336ba09fSKarsten Graul struct smc_link *lnk_new = NULL; 839336ba09fSKarsten Graul struct smc_init_info ini; 840336ba09fSKarsten Graul int lnk_idx, rc = 0; 841336ba09fSKarsten Graul 842fffe83c8SKarsten Graul if (!llc->qp_mtu) 843fffe83c8SKarsten Graul goto out_reject; 844fffe83c8SKarsten Graul 845336ba09fSKarsten Graul ini.vlan_id = lgr->vlan_id; 846336ba09fSKarsten Graul smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 847336ba09fSKarsten Graul if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 848336ba09fSKarsten Graul !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { 849336ba09fSKarsten Graul if (!ini.ib_dev) 850336ba09fSKarsten Graul goto out_reject; 851336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 852336ba09fSKarsten Graul } 853336ba09fSKarsten Graul if (!ini.ib_dev) { 854336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 855336ba09fSKarsten Graul ini.ib_dev = link->smcibdev; 856336ba09fSKarsten Graul ini.ib_port = link->ibport; 857336ba09fSKarsten Graul } 858336ba09fSKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 859336ba09fSKarsten Graul if (lnk_idx < 0) 860336ba09fSKarsten Graul goto out_reject; 861336ba09fSKarsten Graul lnk_new = &lgr->lnk[lnk_idx]; 862336ba09fSKarsten Graul rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); 863336ba09fSKarsten Graul if (rc) 864336ba09fSKarsten Graul goto out_reject; 865336ba09fSKarsten Graul smc_llc_save_add_link_info(lnk_new, llc); 86645fa8da0SKarsten Graul lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 86745fa8da0SKarsten Graul smc_llc_link_set_uid(lnk_new); 868336ba09fSKarsten Graul 869336ba09fSKarsten Graul rc = smc_ib_ready_link(lnk_new); 870336ba09fSKarsten Graul if (rc) 871336ba09fSKarsten Graul goto out_clear_lnk; 872336ba09fSKarsten Graul 873336ba09fSKarsten Graul rc = smcr_buf_map_lgr(lnk_new); 874336ba09fSKarsten Graul if (rc) 875336ba09fSKarsten Graul goto out_clear_lnk; 876336ba09fSKarsten Graul 877336ba09fSKarsten Graul rc = smc_llc_send_add_link(link, 878336ba09fSKarsten Graul lnk_new->smcibdev->mac[ini.ib_port - 1], 879336ba09fSKarsten Graul lnk_new->gid, lnk_new, SMC_LLC_RESP); 880336ba09fSKarsten Graul if (rc) 881336ba09fSKarsten Graul goto out_clear_lnk; 88287f88cdaSKarsten Graul rc = smc_llc_cli_rkey_exchange(link, lnk_new); 883336ba09fSKarsten Graul if (rc) { 884336ba09fSKarsten Graul rc = 0; 885336ba09fSKarsten Graul goto out_clear_lnk; 886336ba09fSKarsten Graul } 887b1570a87SKarsten Graul rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); 888336ba09fSKarsten Graul if (!rc) 889336ba09fSKarsten Graul goto out; 890336ba09fSKarsten Graul out_clear_lnk: 8910a99be43SKarsten Graul smcr_link_clear(lnk_new, false); 892336ba09fSKarsten Graul out_reject: 893336ba09fSKarsten Graul smc_llc_cli_add_link_reject(qentry); 894336ba09fSKarsten Graul out: 895336ba09fSKarsten Graul kfree(qentry); 896336ba09fSKarsten Graul return rc; 897336ba09fSKarsten Graul } 898336ba09fSKarsten Graul 899c48254faSKarsten Graul /* as an SMC client, invite server to start the add_link processing */ 900c48254faSKarsten Graul static void smc_llc_cli_add_link_invite(struct smc_link *link, 901c48254faSKarsten Graul struct smc_llc_qentry *qentry) 902c48254faSKarsten Graul { 903c48254faSKarsten Graul struct smc_link_group *lgr = smc_get_lgr(link); 904c48254faSKarsten Graul struct smc_init_info ini; 905c48254faSKarsten Graul 906c48254faSKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 907c48254faSKarsten Graul lgr->type == SMC_LGR_ASYMMETRIC_PEER) 908c48254faSKarsten Graul goto out; 909c48254faSKarsten Graul 910c48254faSKarsten Graul ini.vlan_id = lgr->vlan_id; 911c48254faSKarsten Graul smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 912c48254faSKarsten Graul if (!ini.ib_dev) 913c48254faSKarsten Graul goto out; 914c48254faSKarsten Graul 915c48254faSKarsten Graul smc_llc_send_add_link(link, ini.ib_dev->mac[ini.ib_port - 1], 916c48254faSKarsten Graul ini.ib_gid, NULL, SMC_LLC_REQ); 917c48254faSKarsten Graul out: 918c48254faSKarsten Graul kfree(qentry); 919c48254faSKarsten Graul } 920c48254faSKarsten Graul 921fffe83c8SKarsten Graul static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) 922fffe83c8SKarsten Graul { 923fffe83c8SKarsten Graul int i; 924fffe83c8SKarsten Graul 925fffe83c8SKarsten Graul for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++) 926fffe83c8SKarsten Graul if (llc->raw.data[i]) 927fffe83c8SKarsten Graul return false; 928fffe83c8SKarsten Graul return true; 929fffe83c8SKarsten Graul } 930fffe83c8SKarsten Graul 931c48254faSKarsten Graul static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) 932c48254faSKarsten Graul { 933c48254faSKarsten Graul if (llc->raw.hdr.common.type == SMC_LLC_ADD_LINK && 934fffe83c8SKarsten Graul smc_llc_is_empty_llc_message(llc)) 935c48254faSKarsten Graul return true; 936c48254faSKarsten Graul return false; 937c48254faSKarsten Graul } 938c48254faSKarsten Graul 939b1570a87SKarsten Graul static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 940b1570a87SKarsten Graul { 941b1570a87SKarsten Graul struct smc_llc_qentry *qentry; 942b1570a87SKarsten Graul 943b1570a87SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 944b1570a87SKarsten Graul 945b1570a87SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 946c48254faSKarsten Graul if (smc_llc_is_local_add_link(&qentry->msg)) 947c48254faSKarsten Graul smc_llc_cli_add_link_invite(qentry->link, qentry); 948c48254faSKarsten Graul else 949b1570a87SKarsten Graul smc_llc_cli_add_link(qentry->link, qentry); 950b1570a87SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 951b1570a87SKarsten Graul } 952b1570a87SKarsten Graul 9539c416878SKarsten Graul static int smc_llc_active_link_count(struct smc_link_group *lgr) 9549c416878SKarsten Graul { 9559c416878SKarsten Graul int i, link_count = 0; 9569c416878SKarsten Graul 9579c416878SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 958741a49a4SKarsten Graul if (!smc_link_active(&lgr->lnk[i])) 9599c416878SKarsten Graul continue; 9609c416878SKarsten Graul link_count++; 9619c416878SKarsten Graul } 9629c416878SKarsten Graul return link_count; 9639c416878SKarsten Graul } 9649c416878SKarsten Graul 965c9a5d243SKarsten Graul /* find the asymmetric link when 3 links are established */ 966c9a5d243SKarsten Graul static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 967c9a5d243SKarsten Graul { 968c9a5d243SKarsten Graul int asym_idx = -ENOENT; 969c9a5d243SKarsten Graul int i, j, k; 970c9a5d243SKarsten Graul bool found; 971c9a5d243SKarsten Graul 972c9a5d243SKarsten Graul /* determine asymmetric link */ 973c9a5d243SKarsten Graul found = false; 974c9a5d243SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 975c9a5d243SKarsten Graul for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 976c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[i]) || 977c9a5d243SKarsten Graul !smc_link_usable(&lgr->lnk[j])) 978c9a5d243SKarsten Graul continue; 979c9a5d243SKarsten Graul if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 980c9a5d243SKarsten Graul SMC_GID_SIZE)) { 981c9a5d243SKarsten Graul found = true; /* asym_lnk is i or j */ 982c9a5d243SKarsten Graul break; 983c9a5d243SKarsten Graul } 984c9a5d243SKarsten Graul } 985c9a5d243SKarsten Graul if (found) 986c9a5d243SKarsten Graul break; 987c9a5d243SKarsten Graul } 988c9a5d243SKarsten Graul if (!found) 989c9a5d243SKarsten Graul goto out; /* no asymmetric link */ 990c9a5d243SKarsten Graul for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 991c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[k])) 992c9a5d243SKarsten Graul continue; 993c9a5d243SKarsten Graul if (k != i && 994c9a5d243SKarsten Graul !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 995c9a5d243SKarsten Graul SMC_GID_SIZE)) { 996c9a5d243SKarsten Graul asym_idx = i; 997c9a5d243SKarsten Graul break; 998c9a5d243SKarsten Graul } 999c9a5d243SKarsten Graul if (k != j && 1000c9a5d243SKarsten Graul !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 1001c9a5d243SKarsten Graul SMC_GID_SIZE)) { 1002c9a5d243SKarsten Graul asym_idx = j; 1003c9a5d243SKarsten Graul break; 1004c9a5d243SKarsten Graul } 1005c9a5d243SKarsten Graul } 1006c9a5d243SKarsten Graul out: 1007c9a5d243SKarsten Graul return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 1008c9a5d243SKarsten Graul } 1009c9a5d243SKarsten Graul 1010c9a5d243SKarsten Graul static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 1011c9a5d243SKarsten Graul { 1012c9a5d243SKarsten Graul struct smc_link *lnk_new = NULL, *lnk_asym; 1013c9a5d243SKarsten Graul struct smc_llc_qentry *qentry; 1014c9a5d243SKarsten Graul int rc; 1015c9a5d243SKarsten Graul 1016c9a5d243SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 1017c9a5d243SKarsten Graul if (!lnk_asym) 1018c9a5d243SKarsten Graul return; /* no asymmetric link */ 1019c9a5d243SKarsten Graul if (!smc_link_downing(&lnk_asym->state)) 1020c9a5d243SKarsten Graul return; 1021c6f02ebeSKarsten Graul lnk_new = smc_switch_conns(lgr, lnk_asym, false); 1022c9a5d243SKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_asym); 1023c9a5d243SKarsten Graul if (!lnk_new) 1024c9a5d243SKarsten Graul goto out_free; 1025c9a5d243SKarsten Graul /* change flow type from ADD_LINK into DEL_LINK */ 1026c9a5d243SKarsten Graul lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 1027c9a5d243SKarsten Graul rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 1028c9a5d243SKarsten Graul true, SMC_LLC_DEL_NO_ASYM_NEEDED); 1029c9a5d243SKarsten Graul if (rc) { 1030c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 1031c9a5d243SKarsten Graul goto out_free; 1032c9a5d243SKarsten Graul } 1033c9a5d243SKarsten Graul qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 1034c9a5d243SKarsten Graul SMC_LLC_DELETE_LINK); 1035c9a5d243SKarsten Graul if (!qentry) { 1036c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 1037c9a5d243SKarsten Graul goto out_free; 1038c9a5d243SKarsten Graul } 1039c9a5d243SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1040c9a5d243SKarsten Graul out_free: 10410a99be43SKarsten Graul smcr_link_clear(lnk_asym, true); 1042c9a5d243SKarsten Graul } 1043c9a5d243SKarsten Graul 104457b49924SKarsten Graul static int smc_llc_srv_rkey_exchange(struct smc_link *link, 104557b49924SKarsten Graul struct smc_link *link_new) 104657b49924SKarsten Graul { 104757b49924SKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 104857b49924SKarsten Graul struct smc_link_group *lgr = link->lgr; 104957b49924SKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 105057b49924SKarsten Graul struct smc_llc_qentry *qentry = NULL; 105157b49924SKarsten Graul struct smc_buf_desc *buf_pos; 105257b49924SKarsten Graul int buf_lst; 105357b49924SKarsten Graul int rc = 0; 105457b49924SKarsten Graul int i; 105557b49924SKarsten Graul 105657b49924SKarsten Graul mutex_lock(&lgr->rmbs_lock); 105757b49924SKarsten Graul num_rkeys_send = lgr->conns_num; 105857b49924SKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 105957b49924SKarsten Graul do { 106057b49924SKarsten Graul smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 106157b49924SKarsten Graul &buf_lst, &buf_pos); 106257b49924SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 106357b49924SKarsten Graul SMC_LLC_ADD_LINK_CONT); 106457b49924SKarsten Graul if (!qentry) { 106557b49924SKarsten Graul rc = -ETIMEDOUT; 106657b49924SKarsten Graul goto out; 106757b49924SKarsten Graul } 106857b49924SKarsten Graul addc_llc = &qentry->msg.add_link_cont; 106957b49924SKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 107057b49924SKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 107157b49924SKarsten Graul for (i = 0; i < max; i++) { 107257b49924SKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 107357b49924SKarsten Graul addc_llc->rt[i].rmb_key, 107457b49924SKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 107557b49924SKarsten Graul addc_llc->rt[i].rmb_key_new); 107657b49924SKarsten Graul num_rkeys_recv--; 107757b49924SKarsten Graul } 107857b49924SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 107957b49924SKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 108057b49924SKarsten Graul out: 108157b49924SKarsten Graul mutex_unlock(&lgr->rmbs_lock); 108257b49924SKarsten Graul return rc; 108357b49924SKarsten Graul } 108457b49924SKarsten Graul 10851551c95bSKarsten Graul static int smc_llc_srv_conf_link(struct smc_link *link, 10861551c95bSKarsten Graul struct smc_link *link_new, 10871551c95bSKarsten Graul enum smc_lgr_type lgr_new_t) 10881551c95bSKarsten Graul { 10891551c95bSKarsten Graul struct smc_link_group *lgr = link->lgr; 10901551c95bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 10911551c95bSKarsten Graul int rc; 10921551c95bSKarsten Graul 10931551c95bSKarsten Graul /* send CONFIRM LINK request over the RoCE fabric */ 10941551c95bSKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 10951551c95bSKarsten Graul if (rc) 10961551c95bSKarsten Graul return -ENOLINK; 10971551c95bSKarsten Graul /* receive CONFIRM LINK response over the RoCE fabric */ 1098a35fffbfSKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0); 1099a35fffbfSKarsten Graul if (!qentry || 1100a35fffbfSKarsten Graul qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { 11011551c95bSKarsten Graul /* send DELETE LINK */ 11021551c95bSKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 11031551c95bSKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 1104a35fffbfSKarsten Graul if (qentry) 1105a35fffbfSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11061551c95bSKarsten Graul return -ENOLINK; 11071551c95bSKarsten Graul } 1108649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 11091551c95bSKarsten Graul smc_llc_link_active(link_new); 1110ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1111ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1112ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1113ad6c111bSKarsten Graul else 1114ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 11151551c95bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11161551c95bSKarsten Graul return 0; 11171551c95bSKarsten Graul } 11181551c95bSKarsten Graul 11192d2209f2SKarsten Graul int smc_llc_srv_add_link(struct smc_link *link) 11202d2209f2SKarsten Graul { 11212d2209f2SKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 11222d2209f2SKarsten Graul struct smc_link_group *lgr = link->lgr; 11232d2209f2SKarsten Graul struct smc_llc_msg_add_link *add_llc; 11242d2209f2SKarsten Graul struct smc_llc_qentry *qentry = NULL; 11252d2209f2SKarsten Graul struct smc_link *link_new; 11262d2209f2SKarsten Graul struct smc_init_info ini; 11272d2209f2SKarsten Graul int lnk_idx, rc = 0; 11282d2209f2SKarsten Graul 11292d2209f2SKarsten Graul /* ignore client add link recommendation, start new flow */ 11302d2209f2SKarsten Graul ini.vlan_id = lgr->vlan_id; 11312d2209f2SKarsten Graul smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); 11322d2209f2SKarsten Graul if (!ini.ib_dev) { 11332d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 11342d2209f2SKarsten Graul ini.ib_dev = link->smcibdev; 11352d2209f2SKarsten Graul ini.ib_port = link->ibport; 11362d2209f2SKarsten Graul } 11372d2209f2SKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 11382d2209f2SKarsten Graul if (lnk_idx < 0) 11392d2209f2SKarsten Graul return 0; 11402d2209f2SKarsten Graul 11412d2209f2SKarsten Graul rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); 11422d2209f2SKarsten Graul if (rc) 11432d2209f2SKarsten Graul return rc; 11442d2209f2SKarsten Graul link_new = &lgr->lnk[lnk_idx]; 11452d2209f2SKarsten Graul rc = smc_llc_send_add_link(link, 11462d2209f2SKarsten Graul link_new->smcibdev->mac[ini.ib_port - 1], 11472d2209f2SKarsten Graul link_new->gid, link_new, SMC_LLC_REQ); 11482d2209f2SKarsten Graul if (rc) 11492d2209f2SKarsten Graul goto out_err; 11502d2209f2SKarsten Graul /* receive ADD LINK response over the RoCE fabric */ 11512d2209f2SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 11522d2209f2SKarsten Graul if (!qentry) { 11532d2209f2SKarsten Graul rc = -ETIMEDOUT; 11542d2209f2SKarsten Graul goto out_err; 11552d2209f2SKarsten Graul } 11562d2209f2SKarsten Graul add_llc = &qentry->msg.add_link; 11572d2209f2SKarsten Graul if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 11582d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11592d2209f2SKarsten Graul rc = -ENOLINK; 11602d2209f2SKarsten Graul goto out_err; 11612d2209f2SKarsten Graul } 11622d2209f2SKarsten Graul if (lgr->type == SMC_LGR_SINGLE && 11632d2209f2SKarsten Graul (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 11642d2209f2SKarsten Graul !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { 11652d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 11662d2209f2SKarsten Graul } 11672d2209f2SKarsten Graul smc_llc_save_add_link_info(link_new, add_llc); 11682d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11692d2209f2SKarsten Graul 11702d2209f2SKarsten Graul rc = smc_ib_ready_link(link_new); 11712d2209f2SKarsten Graul if (rc) 11722d2209f2SKarsten Graul goto out_err; 11732d2209f2SKarsten Graul rc = smcr_buf_map_lgr(link_new); 11742d2209f2SKarsten Graul if (rc) 11752d2209f2SKarsten Graul goto out_err; 11762d2209f2SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 11772d2209f2SKarsten Graul if (rc) 11782d2209f2SKarsten Graul goto out_err; 117957b49924SKarsten Graul rc = smc_llc_srv_rkey_exchange(link, link_new); 11802d2209f2SKarsten Graul if (rc) 11812d2209f2SKarsten Graul goto out_err; 11821551c95bSKarsten Graul rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 11832d2209f2SKarsten Graul if (rc) 11842d2209f2SKarsten Graul goto out_err; 11852d2209f2SKarsten Graul return 0; 11862d2209f2SKarsten Graul out_err: 11870a99be43SKarsten Graul smcr_link_clear(link_new, false); 11882d2209f2SKarsten Graul return rc; 11892d2209f2SKarsten Graul } 11902d2209f2SKarsten Graul 11912d2209f2SKarsten Graul static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 11922d2209f2SKarsten Graul { 11932d2209f2SKarsten Graul struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 11942d2209f2SKarsten Graul int rc; 11952d2209f2SKarsten Graul 11962d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 11972d2209f2SKarsten Graul 11982d2209f2SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 11992d2209f2SKarsten Graul rc = smc_llc_srv_add_link(link); 12002d2209f2SKarsten Graul if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 12012d2209f2SKarsten Graul /* delete any asymmetric link */ 1202c9a5d243SKarsten Graul smc_llc_delete_asym_link(lgr); 12032d2209f2SKarsten Graul } 12042d2209f2SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 12052d2209f2SKarsten Graul } 12062d2209f2SKarsten Graul 1207c48254faSKarsten Graul /* enqueue a local add_link req to trigger a new add_link flow */ 1208c48254faSKarsten Graul void smc_llc_add_link_local(struct smc_link *link) 12094dadd151SKarsten Graul { 121016cb3653SPujin Shi struct smc_llc_msg_add_link add_llc = {}; 12114dadd151SKarsten Graul 12124dadd151SKarsten Graul add_llc.hd.length = sizeof(add_llc); 12134dadd151SKarsten Graul add_llc.hd.common.type = SMC_LLC_ADD_LINK; 1214c48254faSKarsten Graul /* no dev and port needed */ 12154dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 12164dadd151SKarsten Graul } 12174dadd151SKarsten Graul 1218b45e7f98SKarsten Graul /* worker to process an add link message */ 1219b45e7f98SKarsten Graul static void smc_llc_add_link_work(struct work_struct *work) 1220b45e7f98SKarsten Graul { 1221b45e7f98SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1222b45e7f98SKarsten Graul llc_add_link_work); 1223b45e7f98SKarsten Graul 1224b45e7f98SKarsten Graul if (list_empty(&lgr->list)) { 1225b45e7f98SKarsten Graul /* link group is terminating */ 1226b45e7f98SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1227b45e7f98SKarsten Graul goto out; 1228b45e7f98SKarsten Graul } 1229b45e7f98SKarsten Graul 1230b1570a87SKarsten Graul if (lgr->role == SMC_CLNT) 1231b1570a87SKarsten Graul smc_llc_process_cli_add_link(lgr); 12322d2209f2SKarsten Graul else 12332d2209f2SKarsten Graul smc_llc_process_srv_add_link(lgr); 1234b45e7f98SKarsten Graul out: 1235b45e7f98SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1236b45e7f98SKarsten Graul } 1237b45e7f98SKarsten Graul 12384dadd151SKarsten Graul /* enqueue a local del_link msg to trigger a new del_link flow, 12394dadd151SKarsten Graul * called only for role SMC_SERV 12404dadd151SKarsten Graul */ 12414dadd151SKarsten Graul void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 12424dadd151SKarsten Graul { 124316cb3653SPujin Shi struct smc_llc_msg_del_link del_llc = {}; 12444dadd151SKarsten Graul 12454dadd151SKarsten Graul del_llc.hd.length = sizeof(del_llc); 12464dadd151SKarsten Graul del_llc.hd.common.type = SMC_LLC_DELETE_LINK; 12474dadd151SKarsten Graul del_llc.link_num = del_link_id; 12484dadd151SKarsten Graul del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 12494dadd151SKarsten Graul del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 12504dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 12514dadd151SKarsten Graul } 12524dadd151SKarsten Graul 12539c416878SKarsten Graul static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 12549c416878SKarsten Graul { 12559c416878SKarsten Graul struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 12569c416878SKarsten Graul struct smc_llc_msg_del_link *del_llc; 12579c416878SKarsten Graul struct smc_llc_qentry *qentry; 12589c416878SKarsten Graul int active_links; 12599c416878SKarsten Graul int lnk_idx; 12609c416878SKarsten Graul 12619c416878SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 12629c416878SKarsten Graul lnk = qentry->link; 12639c416878SKarsten Graul del_llc = &qentry->msg.delete_link; 12649c416878SKarsten Graul 12659c416878SKarsten Graul if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 12669c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 12679c416878SKarsten Graul goto out; 12689c416878SKarsten Graul } 12699c416878SKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 12709c416878SKarsten Graul /* delete single link */ 12719c416878SKarsten Graul for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 12729c416878SKarsten Graul if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 12739c416878SKarsten Graul continue; 12749c416878SKarsten Graul lnk_del = &lgr->lnk[lnk_idx]; 12759c416878SKarsten Graul break; 12769c416878SKarsten Graul } 12779c416878SKarsten Graul del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 12789c416878SKarsten Graul if (!lnk_del) { 12799c416878SKarsten Graul /* link was not found */ 12809c416878SKarsten Graul del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 12819c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); 12829c416878SKarsten Graul goto out_unlock; 12839c416878SKarsten Graul } 12849c416878SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 12859c416878SKarsten Graul 12869c416878SKarsten Graul del_llc->reason = 0; 12879c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); /* response */ 12889c416878SKarsten Graul 12899c416878SKarsten Graul if (smc_link_downing(&lnk_del->state)) { 1290b7eede75SKarsten Graul if (smc_switch_conns(lgr, lnk_del, false)) 12919c416878SKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_del); 12929c416878SKarsten Graul } 12930a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 12949c416878SKarsten Graul 12959c416878SKarsten Graul active_links = smc_llc_active_link_count(lgr); 12969c416878SKarsten Graul if (lnk_del == lnk_asym) { 12979c416878SKarsten Graul /* expected deletion of asym link, don't change lgr state */ 12989c416878SKarsten Graul } else if (active_links == 1) { 1299ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 13009c416878SKarsten Graul } else if (!active_links) { 1301ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 13029c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 13039c416878SKarsten Graul } 13049c416878SKarsten Graul out_unlock: 13059c416878SKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 13069c416878SKarsten Graul out: 13079c416878SKarsten Graul kfree(qentry); 13089c416878SKarsten Graul } 13099c416878SKarsten Graul 1310f3811fd7SKarsten Graul /* try to send a DELETE LINK ALL request on any active link, 1311f3811fd7SKarsten Graul * waiting for send completion 1312f3811fd7SKarsten Graul */ 1313f3811fd7SKarsten Graul void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1314f3811fd7SKarsten Graul { 13157e94e46cSPujin Shi struct smc_llc_msg_del_link delllc = {}; 1316f3811fd7SKarsten Graul int i; 1317f3811fd7SKarsten Graul 1318f3811fd7SKarsten Graul delllc.hd.common.type = SMC_LLC_DELETE_LINK; 1319f3811fd7SKarsten Graul delllc.hd.length = sizeof(delllc); 1320f3811fd7SKarsten Graul if (ord) 1321f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1322f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1323f3811fd7SKarsten Graul delllc.reason = htonl(rsn); 1324f3811fd7SKarsten Graul 1325f3811fd7SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1326f3811fd7SKarsten Graul if (!smc_link_usable(&lgr->lnk[i])) 1327f3811fd7SKarsten Graul continue; 1328f3811fd7SKarsten Graul if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1329f3811fd7SKarsten Graul break; 1330f3811fd7SKarsten Graul } 1331f3811fd7SKarsten Graul } 1332f3811fd7SKarsten Graul 133308ae27ddSKarsten Graul static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 133408ae27ddSKarsten Graul { 133508ae27ddSKarsten Graul struct smc_llc_msg_del_link *del_llc; 133608ae27ddSKarsten Graul struct smc_link *lnk, *lnk_del; 133708ae27ddSKarsten Graul struct smc_llc_qentry *qentry; 133808ae27ddSKarsten Graul int active_links; 133908ae27ddSKarsten Graul int i; 134008ae27ddSKarsten Graul 134108ae27ddSKarsten Graul mutex_lock(&lgr->llc_conf_mutex); 134208ae27ddSKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 134308ae27ddSKarsten Graul lnk = qentry->link; 134408ae27ddSKarsten Graul del_llc = &qentry->msg.delete_link; 134508ae27ddSKarsten Graul 134608ae27ddSKarsten Graul if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 134708ae27ddSKarsten Graul /* delete entire lgr */ 1348f3811fd7SKarsten Graul smc_llc_send_link_delete_all(lgr, true, ntohl( 1349f3811fd7SKarsten Graul qentry->msg.delete_link.reason)); 135008ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 135108ae27ddSKarsten Graul goto out; 135208ae27ddSKarsten Graul } 135308ae27ddSKarsten Graul /* delete single link */ 135408ae27ddSKarsten Graul lnk_del = NULL; 135508ae27ddSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 135608ae27ddSKarsten Graul if (lgr->lnk[i].link_id == del_llc->link_num) { 135708ae27ddSKarsten Graul lnk_del = &lgr->lnk[i]; 135808ae27ddSKarsten Graul break; 135908ae27ddSKarsten Graul } 136008ae27ddSKarsten Graul } 136108ae27ddSKarsten Graul if (!lnk_del) 136208ae27ddSKarsten Graul goto out; /* asymmetric link already deleted */ 136308ae27ddSKarsten Graul 136408ae27ddSKarsten Graul if (smc_link_downing(&lnk_del->state)) { 1365b7eede75SKarsten Graul if (smc_switch_conns(lgr, lnk_del, false)) 136608ae27ddSKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_del); 136708ae27ddSKarsten Graul } 136808ae27ddSKarsten Graul if (!list_empty(&lgr->list)) { 136908ae27ddSKarsten Graul /* qentry is either a request from peer (send it back to 137008ae27ddSKarsten Graul * initiate the DELETE_LINK processing), or a locally 137108ae27ddSKarsten Graul * enqueued DELETE_LINK request (forward it) 137208ae27ddSKarsten Graul */ 137308ae27ddSKarsten Graul if (!smc_llc_send_message(lnk, &qentry->msg)) { 137408ae27ddSKarsten Graul struct smc_llc_qentry *qentry2; 137508ae27ddSKarsten Graul 137608ae27ddSKarsten Graul qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 137708ae27ddSKarsten Graul SMC_LLC_DELETE_LINK); 1378ca7e3edcSYueHaibing if (qentry2) 137908ae27ddSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 138008ae27ddSKarsten Graul } 138108ae27ddSKarsten Graul } 13820a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 138308ae27ddSKarsten Graul 138408ae27ddSKarsten Graul active_links = smc_llc_active_link_count(lgr); 138508ae27ddSKarsten Graul if (active_links == 1) { 1386ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 138708ae27ddSKarsten Graul } else if (!active_links) { 1388ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 138908ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 139008ae27ddSKarsten Graul } 139108ae27ddSKarsten Graul 139208ae27ddSKarsten Graul if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 139308ae27ddSKarsten Graul /* trigger setup of asymm alt link */ 1394c48254faSKarsten Graul smc_llc_add_link_local(lnk); 139508ae27ddSKarsten Graul } 139608ae27ddSKarsten Graul out: 139708ae27ddSKarsten Graul mutex_unlock(&lgr->llc_conf_mutex); 139808ae27ddSKarsten Graul kfree(qentry); 139908ae27ddSKarsten Graul } 140008ae27ddSKarsten Graul 14019ec6bf19SKarsten Graul static void smc_llc_delete_link_work(struct work_struct *work) 140252bedf37SKarsten Graul { 14039ec6bf19SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 14049ec6bf19SKarsten Graul llc_del_link_work); 140552bedf37SKarsten Graul 14069ec6bf19SKarsten Graul if (list_empty(&lgr->list)) { 14079ec6bf19SKarsten Graul /* link group is terminating */ 14089ec6bf19SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 14099ec6bf19SKarsten Graul goto out; 141052bedf37SKarsten Graul } 14119c416878SKarsten Graul 14129c416878SKarsten Graul if (lgr->role == SMC_CLNT) 14139c416878SKarsten Graul smc_llc_process_cli_delete_link(lgr); 141408ae27ddSKarsten Graul else 141508ae27ddSKarsten Graul smc_llc_process_srv_delete_link(lgr); 14169ec6bf19SKarsten Graul out: 14179ec6bf19SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 141852bedf37SKarsten Graul } 141952bedf37SKarsten Graul 14203bc67e09SKarsten Graul /* process a confirm_rkey request from peer, remote flow */ 14213bc67e09SKarsten Graul static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 14224ed75de5SKarsten Graul { 14233bc67e09SKarsten Graul struct smc_llc_msg_confirm_rkey *llc; 14243bc67e09SKarsten Graul struct smc_llc_qentry *qentry; 14253bc67e09SKarsten Graul struct smc_link *link; 14263bc67e09SKarsten Graul int num_entries; 14273bc67e09SKarsten Graul int rk_idx; 14283bc67e09SKarsten Graul int i; 14294ed75de5SKarsten Graul 14303bc67e09SKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 14313bc67e09SKarsten Graul llc = &qentry->msg.confirm_rkey; 14323bc67e09SKarsten Graul link = qentry->link; 14333bc67e09SKarsten Graul 14343bc67e09SKarsten Graul num_entries = llc->rtoken[0].num_rkeys; 14353bc67e09SKarsten Graul /* first rkey entry is for receiving link */ 14363bc67e09SKarsten Graul rk_idx = smc_rtoken_add(link, 14374ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 14384ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 14393bc67e09SKarsten Graul if (rk_idx < 0) 14403bc67e09SKarsten Graul goto out_err; 14414ed75de5SKarsten Graul 14423bc67e09SKarsten Graul for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 14433bc67e09SKarsten Graul smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 14443bc67e09SKarsten Graul llc->rtoken[i].rmb_vaddr, 14453bc67e09SKarsten Graul llc->rtoken[i].rmb_key); 14463bc67e09SKarsten Graul /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 14473bc67e09SKarsten Graul goto out; 14483bc67e09SKarsten Graul out_err: 14494ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 14503bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 14513bc67e09SKarsten Graul out: 14523bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 14533bc67e09SKarsten Graul smc_llc_send_message(link, &qentry->msg); 14543bc67e09SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 14554ed75de5SKarsten Graul } 14564ed75de5SKarsten Graul 1457218b24feSKarsten Graul /* process a delete_rkey request from peer, remote flow */ 1458218b24feSKarsten Graul static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 14594ed75de5SKarsten Graul { 1460218b24feSKarsten Graul struct smc_llc_msg_delete_rkey *llc; 1461218b24feSKarsten Graul struct smc_llc_qentry *qentry; 1462218b24feSKarsten Graul struct smc_link *link; 14634ed75de5SKarsten Graul u8 err_mask = 0; 14644ed75de5SKarsten Graul int i, max; 14654ed75de5SKarsten Graul 1466218b24feSKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 1467218b24feSKarsten Graul llc = &qentry->msg.delete_rkey; 1468218b24feSKarsten Graul link = qentry->link; 1469218b24feSKarsten Graul 14704ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 14714ed75de5SKarsten Graul for (i = 0; i < max; i++) { 1472387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 14734ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 14744ed75de5SKarsten Graul } 14754ed75de5SKarsten Graul if (err_mask) { 14764ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 14774ed75de5SKarsten Graul llc->err_mask = err_mask; 14784ed75de5SKarsten Graul } 1479218b24feSKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 1480218b24feSKarsten Graul smc_llc_send_message(link, &qentry->msg); 1481218b24feSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1482218b24feSKarsten Graul } 14834ed75de5SKarsten Graul 14843e0c40afSKarsten Graul static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 14853e0c40afSKarsten Graul { 14863e0c40afSKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " 14873e0c40afSKarsten Graul "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); 14883e0c40afSKarsten Graul smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 14893e0c40afSKarsten Graul smc_lgr_terminate_sched(lgr); 14903e0c40afSKarsten Graul } 14913e0c40afSKarsten Graul 14926c8968c4SKarsten Graul /* flush the llc event queue */ 149300a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 14949bf9abeaSUrsula Braun { 14956c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 14969bf9abeaSUrsula Braun 14976c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 14986c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 14996c8968c4SKarsten Graul list_del_init(&qentry->list); 15006c8968c4SKarsten Graul kfree(qentry); 15016c8968c4SKarsten Graul } 15026c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 15036c8968c4SKarsten Graul } 15046c8968c4SKarsten Graul 15056c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 15066c8968c4SKarsten Graul { 15076c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 15086c8968c4SKarsten Graul struct smc_link *link = qentry->link; 15090fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 15106c8968c4SKarsten Graul 1511d854fcbfSKarsten Graul if (!smc_link_usable(link)) 15126c8968c4SKarsten Graul goto out; 1513313164daSKarsten Graul 1514313164daSKarsten Graul switch (llc->raw.hdr.common.type) { 1515313164daSKarsten Graul case SMC_LLC_TEST_LINK: 151656e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 151756e8091cSKarsten Graul smc_llc_send_message(link, llc); 1518313164daSKarsten Graul break; 151952bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 15200fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 15210fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 15220fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 1523c48254faSKarsten Graul if (smc_llc_is_local_add_link(llc)) { 1524c48254faSKarsten Graul if (lgr->llc_flow_lcl.type == 1525c48254faSKarsten Graul SMC_LLC_FLOW_ADD_LINK) 1526c48254faSKarsten Graul break; /* add_link in progress */ 1527c48254faSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1528c48254faSKarsten Graul qentry)) { 1529c48254faSKarsten Graul schedule_work(&lgr->llc_add_link_work); 1530c48254faSKarsten Graul } 1531c48254faSKarsten Graul return; 1532c48254faSKarsten Graul } 1533c48254faSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1534c48254faSKarsten Graul !lgr->llc_flow_lcl.qentry) { 15350fb0b02bSKarsten Graul /* a flow is waiting for this message */ 15360fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 15370fb0b02bSKarsten Graul qentry); 15386778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 15390fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 15400fb0b02bSKarsten Graul qentry)) { 1541b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 15420fb0b02bSKarsten Graul } 15430fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 15440fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 1545b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 15460fb0b02bSKarsten Graul } 15470fb0b02bSKarsten Graul return; 15480fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 154987f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 15500fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 15510fb0b02bSKarsten Graul /* a flow is waiting for this message */ 15520fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 15536778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 15540fb0b02bSKarsten Graul return; 15550fb0b02bSKarsten Graul } 155652bedf37SKarsten Graul break; 155752bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 15589ec6bf19SKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 15599ec6bf19SKarsten Graul !lgr->llc_flow_lcl.qentry) { 15609ec6bf19SKarsten Graul /* DEL LINK REQ during ADD LINK SEQ */ 1561b9979c2eSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 15626778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 1563b9979c2eSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 15649ec6bf19SKarsten Graul schedule_work(&lgr->llc_del_link_work); 15659ec6bf19SKarsten Graul } 15669ec6bf19SKarsten Graul return; 15674ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 15683bc67e09SKarsten Graul /* new request from remote, assign to remote flow */ 15693bc67e09SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 15703bc67e09SKarsten Graul /* process here, does not wait for more llc msgs */ 15713bc67e09SKarsten Graul smc_llc_rmt_conf_rkey(lgr); 15723bc67e09SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 15733bc67e09SKarsten Graul } 15743bc67e09SKarsten Graul return; 15754ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 157642d18accSKarsten Graul /* not used because max links is 3, and 3 rkeys fit into 157742d18accSKarsten Graul * one CONFIRM_RKEY message 157842d18accSKarsten Graul */ 15794ed75de5SKarsten Graul break; 15804ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 1581218b24feSKarsten Graul /* new request from remote, assign to remote flow */ 1582218b24feSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1583218b24feSKarsten Graul /* process here, does not wait for more llc msgs */ 1584218b24feSKarsten Graul smc_llc_rmt_delete_rkey(lgr); 1585218b24feSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1586218b24feSKarsten Graul } 1587218b24feSKarsten Graul return; 15883e0c40afSKarsten Graul default: 15893e0c40afSKarsten Graul smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 15903e0c40afSKarsten Graul break; 1591313164daSKarsten Graul } 15926c8968c4SKarsten Graul out: 15936c8968c4SKarsten Graul kfree(qentry); 15946c8968c4SKarsten Graul } 15956c8968c4SKarsten Graul 15966c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 15976c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 15986c8968c4SKarsten Graul { 15996c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 16006c8968c4SKarsten Graul llc_event_work); 16016c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 16026c8968c4SKarsten Graul 1603555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1604555da9afSKarsten Graul qentry = lgr->delayed_event; 1605555da9afSKarsten Graul lgr->delayed_event = NULL; 1606*d535ca13SKarsten Graul if (smc_link_usable(qentry->link)) 1607*d535ca13SKarsten Graul smc_llc_event_handler(qentry); 1608*d535ca13SKarsten Graul else 1609555da9afSKarsten Graul kfree(qentry); 1610555da9afSKarsten Graul } 1611555da9afSKarsten Graul 16126c8968c4SKarsten Graul again: 16136c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 16146c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 16156c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 16166c8968c4SKarsten Graul struct smc_llc_qentry, list); 16176c8968c4SKarsten Graul list_del_init(&qentry->list); 16186c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 16196c8968c4SKarsten Graul smc_llc_event_handler(qentry); 16206c8968c4SKarsten Graul goto again; 16216c8968c4SKarsten Graul } 16226c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 16236c8968c4SKarsten Graul } 16246c8968c4SKarsten Graul 1625ef79d439SKarsten Graul /* process llc responses in tasklet context */ 1626a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 1627a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 1628ef79d439SKarsten Graul { 16292ff08678SKarsten Graul enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type; 16302ff08678SKarsten Graul struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl; 1631a6688d91SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.type; 1632ef79d439SKarsten Graul 1633a6688d91SKarsten Graul switch (llc_type) { 1634ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 1635741a49a4SKarsten Graul if (smc_link_active(link)) 1636ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 1637ef79d439SKarsten Graul break; 1638ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 163987f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 16402ff08678SKarsten Graul case SMC_LLC_CONFIRM_LINK: 16412ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry) 16422ff08678SKarsten Graul break; /* drop out-of-flow response */ 16432ff08678SKarsten Graul goto assign; 16442ff08678SKarsten Graul case SMC_LLC_DELETE_LINK: 16452ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry) 16462ff08678SKarsten Graul break; /* drop out-of-flow response */ 16472ff08678SKarsten Graul goto assign; 16483d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 16496d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 16502ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry) 16512ff08678SKarsten Graul break; /* drop out-of-flow response */ 16522ff08678SKarsten Graul goto assign; 1653ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 165442d18accSKarsten Graul /* not used because max links is 3 */ 1655ef79d439SKarsten Graul break; 16563e0c40afSKarsten Graul default: 16573e0c40afSKarsten Graul smc_llc_protocol_violation(link->lgr, llc_type); 16583e0c40afSKarsten Graul break; 1659ef79d439SKarsten Graul } 1660a6688d91SKarsten Graul kfree(qentry); 16612ff08678SKarsten Graul return; 16622ff08678SKarsten Graul assign: 16632ff08678SKarsten Graul /* assign responses to the local flow, we requested them */ 16642ff08678SKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 16652ff08678SKarsten Graul wake_up(&link->lgr->llc_msg_waiter); 1666ef79d439SKarsten Graul } 1667ef79d439SKarsten Graul 1668a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 16696c8968c4SKarsten Graul { 16706c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 16716c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 16726c8968c4SKarsten Graul unsigned long flags; 16736c8968c4SKarsten Graul 16746c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 16756c8968c4SKarsten Graul if (!qentry) 16766c8968c4SKarsten Graul return; 16776c8968c4SKarsten Graul qentry->link = link; 16786c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 16796c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 1680a6688d91SKarsten Graul 1681a6688d91SKarsten Graul /* process responses immediately */ 1682a6688d91SKarsten Graul if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 1683a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 1684a6688d91SKarsten Graul return; 1685a6688d91SKarsten Graul } 1686a6688d91SKarsten Graul 1687a6688d91SKarsten Graul /* add requests to event queue */ 16886c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 16896c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 16906c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 16916778a6beSKarsten Graul schedule_work(&lgr->llc_event_work); 16929bf9abeaSUrsula Braun } 16939bf9abeaSUrsula Braun 1694a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 1695a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 1696a6688d91SKarsten Graul { 1697a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 1698a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 1699a6688d91SKarsten Graul 1700a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 1701a6688d91SKarsten Graul return; /* short message */ 1702a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 1703a6688d91SKarsten Graul return; /* invalid message */ 1704a6688d91SKarsten Graul 1705a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 1706a6688d91SKarsten Graul } 1707a6688d91SKarsten Graul 170844aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 1709877ae5beSKarsten Graul 1710877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 1711877ae5beSKarsten Graul { 1712877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 1713877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 1714877ae5beSKarsten Graul unsigned long next_interval; 1715877ae5beSKarsten Graul unsigned long expire_time; 1716877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 1717877ae5beSKarsten Graul int rc; 1718877ae5beSKarsten Graul 1719741a49a4SKarsten Graul if (!smc_link_active(link)) 1720877ae5beSKarsten Graul return; /* don't reschedule worker */ 1721877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 1722877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 1723877ae5beSKarsten Graul next_interval = expire_time - jiffies; 1724877ae5beSKarsten Graul goto out; 1725877ae5beSKarsten Graul } 1726877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 1727d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 1728877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 1729877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 1730877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 1731741a49a4SKarsten Graul if (!smc_link_active(link)) 17321020e1efSKarsten Graul return; /* link state changed */ 1733877ae5beSKarsten Graul if (rc <= 0) { 173487523930SKarsten Graul smcr_link_down_cond_sched(link); 1735877ae5beSKarsten Graul return; 1736877ae5beSKarsten Graul } 1737877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 1738877ae5beSKarsten Graul out: 17391020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 1740877ae5beSKarsten Graul } 1741877ae5beSKarsten Graul 174200a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 174300a049cfSKarsten Graul { 174400a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 174500a049cfSKarsten Graul 174600a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 1747b45e7f98SKarsten Graul INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 17489ec6bf19SKarsten Graul INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 174900a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 175000a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 1751555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 17526778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_flow_waiter); 17536778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_msg_waiter); 1754d5500667SKarsten Graul mutex_init(&lgr->llc_conf_mutex); 175500a049cfSKarsten Graul lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 175600a049cfSKarsten Graul } 175700a049cfSKarsten Graul 175800a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 175900a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 176000a049cfSKarsten Graul { 176100a049cfSKarsten Graul smc_llc_event_flush(lgr); 17626778a6beSKarsten Graul wake_up_all(&lgr->llc_flow_waiter); 17636778a6beSKarsten Graul wake_up_all(&lgr->llc_msg_waiter); 176400a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 1765b45e7f98SKarsten Graul cancel_work_sync(&lgr->llc_add_link_work); 17669ec6bf19SKarsten Graul cancel_work_sync(&lgr->llc_del_link_work); 1767555da9afSKarsten Graul if (lgr->delayed_event) { 1768555da9afSKarsten Graul kfree(lgr->delayed_event); 1769555da9afSKarsten Graul lgr->delayed_event = NULL; 1770555da9afSKarsten Graul } 177100a049cfSKarsten Graul } 177200a049cfSKarsten Graul 17732a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 1774877ae5beSKarsten Graul { 1775877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 1776877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 17772a4c57a9SKarsten Graul return 0; 1778b32cf4abSKarsten Graul } 1779b32cf4abSKarsten Graul 178000a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 1781b32cf4abSKarsten Graul { 17820a99be43SKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " 17830a99be43SKarsten Graul "peerid %*phN, ibdev %s, ibport %d\n", 17840a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 17850a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 17860a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 17870a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 1788877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 178900a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 179000a049cfSKarsten Graul link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 17911020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 1792877ae5beSKarsten Graul link->llc_testlink_time); 1793877ae5beSKarsten Graul } 1794877ae5beSKarsten Graul } 1795877ae5beSKarsten Graul 1796877ae5beSKarsten Graul /* called in worker context */ 17970a99be43SKarsten Graul void smc_llc_link_clear(struct smc_link *link, bool log) 1798877ae5beSKarsten Graul { 17990a99be43SKarsten Graul if (log) 18000a99be43SKarsten Graul pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" 18010a99be43SKarsten Graul ", peerid %*phN, ibdev %s, ibport %d\n", 18020a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 18030a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 18040a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 18050a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 18062140ac26SKarsten Graul complete(&link->llc_testlink_resp); 18072140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 18082140ac26SKarsten Graul smc_wr_wakeup_reg_wait(link); 18092140ac26SKarsten Graul smc_wr_wakeup_tx_wait(link); 1810877ae5beSKarsten Graul } 1811877ae5beSKarsten Graul 18123d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 18133d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 181444aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 181544aa81ceSKarsten Graul { 18163d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 18173d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 18183d88a21bSKarsten Graul int rc = 0; 181944aa81ceSKarsten Graul 18203d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 18213d88a21bSKarsten Graul if (rc) 18223d88a21bSKarsten Graul goto out; 182344aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 18243d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 18253d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 18263d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 18273d88a21bSKarsten Graul rc = -EFAULT; 18283d88a21bSKarsten Graul out: 18293d88a21bSKarsten Graul if (qentry) 18303d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 18313d88a21bSKarsten Graul return rc; 183244aa81ceSKarsten Graul } 183344aa81ceSKarsten Graul 183460e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 18356d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 183660e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 183760e03c62SKarsten Graul { 18386d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 18396d74c3a8SKarsten Graul struct smc_link *send_link; 18400b29ec64SUrsula Braun int rc = 0; 184160e03c62SKarsten Graul 18426d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 18436d74c3a8SKarsten Graul if (!send_link) 18446d74c3a8SKarsten Graul return -ENOLINK; 18456d74c3a8SKarsten Graul 18466d74c3a8SKarsten Graul /* protected by llc_flow control */ 18476d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 184860e03c62SKarsten Graul if (rc) 184960e03c62SKarsten Graul goto out; 185060e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 18516d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 18526d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 18536d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 185460e03c62SKarsten Graul rc = -EFAULT; 185560e03c62SKarsten Graul out: 18566d74c3a8SKarsten Graul if (qentry) 18576d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 185860e03c62SKarsten Graul return rc; 185960e03c62SKarsten Graul } 186060e03c62SKarsten Graul 186145fa8da0SKarsten Graul void smc_llc_link_set_uid(struct smc_link *link) 186245fa8da0SKarsten Graul { 186345fa8da0SKarsten Graul __be32 link_uid; 186445fa8da0SKarsten Graul 186545fa8da0SKarsten Graul link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 186645fa8da0SKarsten Graul memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 186745fa8da0SKarsten Graul } 186845fa8da0SKarsten Graul 1869649758ffSKarsten Graul /* save peers link user id, used for debug purposes */ 1870649758ffSKarsten Graul void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 1871649758ffSKarsten Graul { 1872649758ffSKarsten Graul memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 1873649758ffSKarsten Graul SMC_LGR_ID_SIZE); 1874649758ffSKarsten Graul } 1875649758ffSKarsten Graul 187692334cfcSKarsten Graul /* evaluate confirm link request or response */ 187792334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 187892334cfcSKarsten Graul enum smc_llc_reqresp type) 187992334cfcSKarsten Graul { 188045fa8da0SKarsten Graul if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 188192334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 188245fa8da0SKarsten Graul smc_llc_link_set_uid(qentry->link); 188345fa8da0SKarsten Graul } 188492334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 188592334cfcSKarsten Graul return -ENOTSUPP; 188692334cfcSKarsten Graul return 0; 188792334cfcSKarsten Graul } 188892334cfcSKarsten Graul 18899bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 18909bf9abeaSUrsula Braun 18919bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 18929bf9abeaSUrsula Braun { 18939bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 18949bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 18959bf9abeaSUrsula Braun }, 18969bf9abeaSUrsula Braun { 1897313164daSKarsten Graul .handler = smc_llc_rx_handler, 1898313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 1899313164daSKarsten Graul }, 1900313164daSKarsten Graul { 19014ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 190252bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 190352bedf37SKarsten Graul }, 190452bedf37SKarsten Graul { 190552bedf37SKarsten Graul .handler = smc_llc_rx_handler, 190687f88cdaSKarsten Graul .type = SMC_LLC_ADD_LINK_CONT 190787f88cdaSKarsten Graul }, 190887f88cdaSKarsten Graul { 190987f88cdaSKarsten Graul .handler = smc_llc_rx_handler, 191052bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 191152bedf37SKarsten Graul }, 191252bedf37SKarsten Graul { 191352bedf37SKarsten Graul .handler = smc_llc_rx_handler, 19144ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 19154ed75de5SKarsten Graul }, 19164ed75de5SKarsten Graul { 19174ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 19184ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 19194ed75de5SKarsten Graul }, 19204ed75de5SKarsten Graul { 19214ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 19224ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 19234ed75de5SKarsten Graul }, 19244ed75de5SKarsten Graul { 19259bf9abeaSUrsula Braun .handler = NULL, 19269bf9abeaSUrsula Braun } 19279bf9abeaSUrsula Braun }; 19289bf9abeaSUrsula Braun 19299bf9abeaSUrsula Braun int __init smc_llc_init(void) 19309bf9abeaSUrsula Braun { 19319bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 19329bf9abeaSUrsula Braun int rc = 0; 19339bf9abeaSUrsula Braun 19349bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 19359bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 19369bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 19379bf9abeaSUrsula Braun if (rc) 19389bf9abeaSUrsula Braun break; 19399bf9abeaSUrsula Braun } 19409bf9abeaSUrsula Braun return rc; 19419bf9abeaSUrsula Braun } 1942