1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 209bf9abeaSUrsula Braun 210f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 220f627126SStefan Raspl 230f627126SStefan Raspl struct smc_llc_hdr { 240f627126SStefan Raspl struct smc_wr_rx_hdr common; 250f627126SStefan Raspl u8 length; /* 44 */ 2652bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 2752bedf37SKarsten Graul u8 reserved:4, 2852bedf37SKarsten Graul add_link_rej_rsn:4; 2952bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3052bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3152bedf37SKarsten Graul reserved:4; 3252bedf37SKarsten Graul #endif 330f627126SStefan Raspl u8 flags; 340f627126SStefan Raspl }; 350f627126SStefan Raspl 3675d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 3775d320d6SKarsten Graul 380f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 390f627126SStefan Raspl struct smc_llc_hdr hd; 400f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 410f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 420f627126SStefan Raspl u8 sender_qp_num[3]; 430f627126SStefan Raspl u8 link_num; 440f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 450f627126SStefan Raspl u8 max_links; 460f627126SStefan Raspl u8 reserved[9]; 470f627126SStefan Raspl }; 480f627126SStefan Raspl 4952bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5052bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 5152bedf37SKarsten Graul 5252bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 5352bedf37SKarsten Graul 5452bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 5552bedf37SKarsten Graul struct smc_llc_hdr hd; 5652bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 5752bedf37SKarsten Graul u8 reserved2[2]; 5852bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 5952bedf37SKarsten Graul u8 sender_qp_num[3]; 6052bedf37SKarsten Graul u8 link_num; 6152bedf37SKarsten Graul u8 flags2; /* QP mtu */ 6252bedf37SKarsten Graul u8 initial_psn[3]; 6352bedf37SKarsten Graul u8 reserved[8]; 6452bedf37SKarsten Graul }; 6552bedf37SKarsten Graul 6652bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 6752bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 6852bedf37SKarsten Graul 6952bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 7052bedf37SKarsten Graul struct smc_llc_hdr hd; 7152bedf37SKarsten Graul u8 link_num; 7252bedf37SKarsten Graul __be32 reason; 7352bedf37SKarsten Graul u8 reserved[35]; 7452bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 7552bedf37SKarsten Graul 76313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 77313164daSKarsten Graul struct smc_llc_hdr hd; 78313164daSKarsten Graul u8 user_data[16]; 79313164daSKarsten Graul u8 reserved[24]; 80313164daSKarsten Graul }; 81313164daSKarsten Graul 824ed75de5SKarsten Graul struct smc_rmb_rtoken { 834ed75de5SKarsten Graul union { 844ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 854ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 864ed75de5SKarsten Graul /* rtoken is always for the current link */ 874ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 884ed75de5SKarsten Graul }; 894ed75de5SKarsten Graul __be32 rmb_key; 904ed75de5SKarsten Graul __be64 rmb_vaddr; 914ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 924ed75de5SKarsten Graul 934ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 944ed75de5SKarsten Graul 954ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 964ed75de5SKarsten Graul struct smc_llc_hdr hd; 974ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 984ed75de5SKarsten Graul u8 reserved; 994ed75de5SKarsten Graul }; 1004ed75de5SKarsten Graul 1014ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */ 1024ed75de5SKarsten Graul struct smc_llc_hdr hd; 1034ed75de5SKarsten Graul u8 num_rkeys; 1044ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1054ed75de5SKarsten Graul }; 1064ed75de5SKarsten Graul 1074ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 1084ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1094ed75de5SKarsten Graul 1104ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1114ed75de5SKarsten Graul struct smc_llc_hdr hd; 1124ed75de5SKarsten Graul u8 num_rkeys; 1134ed75de5SKarsten Graul u8 err_mask; 1144ed75de5SKarsten Graul u8 reserved[2]; 1154ed75de5SKarsten Graul __be32 rkey[8]; 1164ed75de5SKarsten Graul u8 reserved2[4]; 1174ed75de5SKarsten Graul }; 1184ed75de5SKarsten Graul 1190f627126SStefan Raspl union smc_llc_msg { 1200f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 12152bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 12252bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1234ed75de5SKarsten Graul 1244ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1254ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont; 1264ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1274ed75de5SKarsten Graul 128313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1290f627126SStefan Raspl struct { 1300f627126SStefan Raspl struct smc_llc_hdr hdr; 1310f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1320f627126SStefan Raspl } raw; 1330f627126SStefan Raspl }; 1340f627126SStefan Raspl 1350f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1360f627126SStefan Raspl 1376c8968c4SKarsten Graul struct smc_llc_qentry { 1386c8968c4SKarsten Graul struct list_head list; 1396c8968c4SKarsten Graul struct smc_link *link; 1406c8968c4SKarsten Graul union smc_llc_msg msg; 1416c8968c4SKarsten Graul }; 1426c8968c4SKarsten Graul 143555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 144555da9afSKarsten Graul { 145555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 146555da9afSKarsten Graul 147555da9afSKarsten Graul flow->qentry = NULL; 148555da9afSKarsten Graul return qentry; 149555da9afSKarsten Graul } 150555da9afSKarsten Graul 151555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 152555da9afSKarsten Graul { 153555da9afSKarsten Graul struct smc_llc_qentry *qentry; 154555da9afSKarsten Graul 155555da9afSKarsten Graul if (flow->qentry) { 156555da9afSKarsten Graul qentry = flow->qentry; 157555da9afSKarsten Graul flow->qentry = NULL; 158555da9afSKarsten Graul kfree(qentry); 159555da9afSKarsten Graul } 160555da9afSKarsten Graul } 161555da9afSKarsten Graul 162555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 163555da9afSKarsten Graul struct smc_llc_qentry *qentry) 164555da9afSKarsten Graul { 165555da9afSKarsten Graul flow->qentry = qentry; 166555da9afSKarsten Graul } 167555da9afSKarsten Graul 168555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 169555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 170555da9afSKarsten Graul struct smc_llc_qentry *qentry) 171555da9afSKarsten Graul { 172555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 173555da9afSKarsten Graul 174555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 175555da9afSKarsten Graul if (flow->type) { 176555da9afSKarsten Graul /* a flow is already active */ 177555da9afSKarsten Graul if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || 178555da9afSKarsten Graul qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && 179555da9afSKarsten Graul !lgr->delayed_event) { 180555da9afSKarsten Graul lgr->delayed_event = qentry; 181555da9afSKarsten Graul } else { 182555da9afSKarsten Graul /* forget this llc request */ 183555da9afSKarsten Graul kfree(qentry); 184555da9afSKarsten Graul } 185555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 186555da9afSKarsten Graul return false; 187555da9afSKarsten Graul } 188555da9afSKarsten Graul switch (qentry->msg.raw.hdr.common.type) { 189555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 190555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 191555da9afSKarsten Graul break; 192555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 193555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 194555da9afSKarsten Graul break; 195555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 196555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 197555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 198555da9afSKarsten Graul break; 199555da9afSKarsten Graul default: 200555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 201555da9afSKarsten Graul } 202555da9afSKarsten Graul if (qentry == lgr->delayed_event) 203555da9afSKarsten Graul lgr->delayed_event = NULL; 204555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 205555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 206555da9afSKarsten Graul return true; 207555da9afSKarsten Graul } 208555da9afSKarsten Graul 209555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 210555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 211555da9afSKarsten Graul enum smc_llc_flowtype type) 212555da9afSKarsten Graul { 213555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 214555da9afSKarsten Graul int rc; 215555da9afSKarsten Graul 216555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 217555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 218555da9afSKarsten Graul */ 219555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 220555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 221555da9afSKarsten Graul again: 222555da9afSKarsten Graul if (list_empty(&lgr->list)) 223555da9afSKarsten Graul return -ENODEV; 224555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 225555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 226555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 227555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 228555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 229555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 230555da9afSKarsten Graul return 0; 231555da9afSKarsten Graul } 232555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 233555da9afSKarsten Graul rc = wait_event_interruptible_timeout(lgr->llc_waiter, 234555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 235555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 236555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)), 237555da9afSKarsten Graul SMC_LLC_WAIT_TIME); 238555da9afSKarsten Graul if (!rc) 239555da9afSKarsten Graul return -ETIMEDOUT; 240555da9afSKarsten Graul goto again; 241555da9afSKarsten Graul } 242555da9afSKarsten Graul 243555da9afSKarsten Graul /* finish the current llc flow */ 244555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 245555da9afSKarsten Graul { 246555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 247555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 248555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 249555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 250555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 251555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 252555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 253555da9afSKarsten Graul else 254555da9afSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 255555da9afSKarsten Graul } 256555da9afSKarsten Graul 257555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 258555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 259555da9afSKarsten Graul */ 260555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 261555da9afSKarsten Graul struct smc_link *lnk, 262555da9afSKarsten Graul int time_out, u8 exp_msg) 263555da9afSKarsten Graul { 264555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 265555da9afSKarsten Graul 266555da9afSKarsten Graul wait_event_interruptible_timeout(lgr->llc_waiter, 267555da9afSKarsten Graul (flow->qentry || 268555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 269555da9afSKarsten Graul list_empty(&lgr->list)), 270555da9afSKarsten Graul time_out); 271555da9afSKarsten Graul if (!flow->qentry || 272555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 273555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 274555da9afSKarsten Graul goto out; 275555da9afSKarsten Graul } 276555da9afSKarsten Graul if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { 277555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 278555da9afSKarsten Graul flow->qentry->msg.raw.hdr.common.type == 279555da9afSKarsten Graul SMC_LLC_DELETE_LINK) { 280555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 281555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 282555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 283555da9afSKarsten Graul return NULL; 284555da9afSKarsten Graul } 285555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 286555da9afSKarsten Graul } 287555da9afSKarsten Graul out: 288555da9afSKarsten Graul return flow->qentry; 289555da9afSKarsten Graul } 290555da9afSKarsten Graul 2919bf9abeaSUrsula Braun /********************************** send *************************************/ 2929bf9abeaSUrsula Braun 2939bf9abeaSUrsula Braun struct smc_llc_tx_pend { 2949bf9abeaSUrsula Braun }; 2959bf9abeaSUrsula Braun 2969bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 2979bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 2989bf9abeaSUrsula Braun struct smc_link *link, 2999bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3009bf9abeaSUrsula Braun { 3019bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3029bf9abeaSUrsula Braun } 3039bf9abeaSUrsula Braun 3049bf9abeaSUrsula Braun /** 3059bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3069bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3079bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3089bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3099bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3109bf9abeaSUrsula Braun * 3119bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3129bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3139bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3149bf9abeaSUrsula Braun * 3159bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 3169bf9abeaSUrsula Braun */ 3179bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 3189bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 3199bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 3209bf9abeaSUrsula Braun { 3219bf9abeaSUrsula Braun int rc; 3229bf9abeaSUrsula Braun 323ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 324ad6f317fSUrsula Braun pend); 3259bf9abeaSUrsula Braun if (rc < 0) 3269bf9abeaSUrsula Braun return rc; 3279bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3289bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 3299bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 3309bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3319bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 3329bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 3339bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3349bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 3359bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 3369bf9abeaSUrsula Braun return 0; 3379bf9abeaSUrsula Braun } 3389bf9abeaSUrsula Braun 3399bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 340947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 3419bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 3429bf9abeaSUrsula Braun { 34300e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 3449bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 3459bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 3469bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 3479bf9abeaSUrsula Braun int rc; 3489bf9abeaSUrsula Braun 3499bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 3509bf9abeaSUrsula Braun if (rc) 3519bf9abeaSUrsula Braun return rc; 3529bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 3539bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 3549bf9abeaSUrsula Braun confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 3559bf9abeaSUrsula Braun confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 35675d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 3579bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 3589bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 359947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 360947541f3SUrsula Braun ETH_ALEN); 3617005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 3629bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 3632be922f3SKarsten Graul confllc->link_num = link->link_id; 3649bf9abeaSUrsula Braun memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); 36552bedf37SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ 36652bedf37SKarsten Graul /* send llc message */ 36752bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 36852bedf37SKarsten Graul return rc; 36952bedf37SKarsten Graul } 37052bedf37SKarsten Graul 37144aa81ceSKarsten Graul /* send LLC confirm rkey request */ 3723d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 37344aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 37444aa81ceSKarsten Graul { 37544aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 37644aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 37744aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 3783d88a21bSKarsten Graul struct smc_link *link; 3793d88a21bSKarsten Graul int i, rc, rtok_ix; 38044aa81ceSKarsten Graul 3813d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 38244aa81ceSKarsten Graul if (rc) 38344aa81ceSKarsten Graul return rc; 38444aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 38544aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 38644aa81ceSKarsten Graul rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 38744aa81ceSKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 3883d88a21bSKarsten Graul 3893d88a21bSKarsten Graul rtok_ix = 1; 3903d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 3913d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 3923d88a21bSKarsten Graul if (link->state == SMC_LNK_ACTIVE && link != send_link) { 3933d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 3943d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 395387707fdSKarsten Graul htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 3963d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 3973d88a21bSKarsten Graul (u64)sg_dma_address( 3983d88a21bSKarsten Graul rmb_desc->sgt[link->link_idx].sgl)); 3993d88a21bSKarsten Graul rtok_ix++; 4003d88a21bSKarsten Graul } 4013d88a21bSKarsten Graul } 4023d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 4033d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 4043d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 4053d88a21bSKarsten Graul htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 40644aa81ceSKarsten Graul rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 4073d88a21bSKarsten Graul (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 40844aa81ceSKarsten Graul /* send llc message */ 4093d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 41044aa81ceSKarsten Graul return rc; 41144aa81ceSKarsten Graul } 41244aa81ceSKarsten Graul 41360e03c62SKarsten Graul /* send LLC delete rkey request */ 41460e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 41560e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 41660e03c62SKarsten Graul { 41760e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 41860e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 41960e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 42060e03c62SKarsten Graul int rc; 42160e03c62SKarsten Graul 42260e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 42360e03c62SKarsten Graul if (rc) 42460e03c62SKarsten Graul return rc; 42560e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 42660e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 42760e03c62SKarsten Graul rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 42860e03c62SKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 42960e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 430387707fdSKarsten Graul rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 43160e03c62SKarsten Graul /* send llc message */ 43260e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 43360e03c62SKarsten Graul return rc; 43460e03c62SKarsten Graul } 43560e03c62SKarsten Graul 4362a4c57a9SKarsten Graul /* prepare an add link message */ 4372a4c57a9SKarsten Graul static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc, 4387005ada6SUrsula Braun struct smc_link *link, u8 mac[], u8 gid[], 4392a4c57a9SKarsten Graul enum smc_llc_reqresp reqresp) 4402a4c57a9SKarsten Graul { 4412a4c57a9SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 4422a4c57a9SKarsten Graul addllc->hd.common.type = SMC_LLC_ADD_LINK; 4432a4c57a9SKarsten Graul addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 4442a4c57a9SKarsten Graul if (reqresp == SMC_LLC_RESP) { 4452a4c57a9SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 4462a4c57a9SKarsten Graul /* always reject more links for now */ 4472a4c57a9SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 4482a4c57a9SKarsten Graul addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 4492a4c57a9SKarsten Graul } 4502a4c57a9SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 4512a4c57a9SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 4522a4c57a9SKarsten Graul } 4532a4c57a9SKarsten Graul 45452bedf37SKarsten Graul /* send ADD LINK request or response */ 4557005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 45652bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 45752bedf37SKarsten Graul { 45852bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 45952bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 46052bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 46152bedf37SKarsten Graul int rc; 46252bedf37SKarsten Graul 46352bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 46452bedf37SKarsten Graul if (rc) 46552bedf37SKarsten Graul return rc; 46652bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 4672a4c57a9SKarsten Graul smc_llc_prep_add_link(addllc, link, mac, gid, reqresp); 46852bedf37SKarsten Graul /* send llc message */ 46952bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 47052bedf37SKarsten Graul return rc; 47152bedf37SKarsten Graul } 47252bedf37SKarsten Graul 4732a4c57a9SKarsten Graul /* prepare a delete link message */ 4742a4c57a9SKarsten Graul static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc, 4752a4c57a9SKarsten Graul struct smc_link *link, 4760d18a0cbSKarsten Graul enum smc_llc_reqresp reqresp, bool orderly) 4772a4c57a9SKarsten Graul { 4782a4c57a9SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 4792a4c57a9SKarsten Graul delllc->hd.common.type = SMC_LLC_DELETE_LINK; 4802a4c57a9SKarsten Graul delllc->hd.length = sizeof(struct smc_llc_msg_add_link); 4812a4c57a9SKarsten Graul if (reqresp == SMC_LLC_RESP) 4822a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 4832a4c57a9SKarsten Graul /* DEL_LINK_ALL because only 1 link supported */ 4842a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 4850d18a0cbSKarsten Graul if (orderly) 4862a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 4872a4c57a9SKarsten Graul delllc->link_num = link->link_id; 4882a4c57a9SKarsten Graul } 4892a4c57a9SKarsten Graul 49052bedf37SKarsten Graul /* send DELETE LINK request or response */ 49152bedf37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, 4920d18a0cbSKarsten Graul enum smc_llc_reqresp reqresp, bool orderly) 49352bedf37SKarsten Graul { 49452bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 49552bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 49652bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 49752bedf37SKarsten Graul int rc; 49852bedf37SKarsten Graul 49952bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 50052bedf37SKarsten Graul if (rc) 50152bedf37SKarsten Graul return rc; 50252bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 5030d18a0cbSKarsten Graul smc_llc_prep_delete_link(delllc, link, reqresp, orderly); 5049bf9abeaSUrsula Braun /* send llc message */ 5059bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 5069bf9abeaSUrsula Braun return rc; 5079bf9abeaSUrsula Braun } 5089bf9abeaSUrsula Braun 509d97935faSKarsten Graul /* send LLC test link request */ 510d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 511313164daSKarsten Graul { 512313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 513313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 514313164daSKarsten Graul struct smc_wr_buf *wr_buf; 515313164daSKarsten Graul int rc; 516313164daSKarsten Graul 517313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 518313164daSKarsten Graul if (rc) 519313164daSKarsten Graul return rc; 520313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 521313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 522313164daSKarsten Graul testllc->hd.common.type = SMC_LLC_TEST_LINK; 523313164daSKarsten Graul testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 524313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 525313164daSKarsten Graul /* send llc message */ 526313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 527313164daSKarsten Graul return rc; 528313164daSKarsten Graul } 529313164daSKarsten Graul 5306c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 5316c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 5324ed75de5SKarsten Graul { 5334ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 5344ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 5354ed75de5SKarsten Graul int rc; 5364ed75de5SKarsten Graul 5376c8968c4SKarsten Graul if (!smc_link_usable(link)) 5386c8968c4SKarsten Graul return -ENOLINK; 5396c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 5404ed75de5SKarsten Graul if (rc) 5416c8968c4SKarsten Graul return rc; 5426c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 5436c8968c4SKarsten Graul return smc_wr_tx_send(link, pend); 5444ed75de5SKarsten Graul } 5454ed75de5SKarsten Graul 5469bf9abeaSUrsula Braun /********************************* receive ***********************************/ 5479bf9abeaSUrsula Braun 54852bedf37SKarsten Graul static void smc_llc_rx_delete_link(struct smc_link *link, 54952bedf37SKarsten Graul struct smc_llc_msg_del_link *llc) 55052bedf37SKarsten Graul { 55100e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 55252bedf37SKarsten Graul 5539651b934SKarsten Graul smc_lgr_forget(lgr); 5540d18a0cbSKarsten Graul smc_llc_link_deleting(link); 5550d18a0cbSKarsten Graul if (lgr->role == SMC_SERV) { 5560d18a0cbSKarsten Graul /* client asks to delete this link, send request */ 5570d18a0cbSKarsten Graul smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true); 55852bedf37SKarsten Graul } else { 5590d18a0cbSKarsten Graul /* server requests to delete this link, send response */ 5600d18a0cbSKarsten Graul smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true); 56152bedf37SKarsten Graul } 5626c8968c4SKarsten Graul smc_llc_send_message(link, llc); 563f528ba24SUrsula Braun smc_lgr_terminate_sched(lgr); 56452bedf37SKarsten Graul } 56552bedf37SKarsten Graul 5664ed75de5SKarsten Graul static void smc_llc_rx_confirm_rkey(struct smc_link *link, 5674ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey *llc) 5684ed75de5SKarsten Graul { 5694ed75de5SKarsten Graul int rc; 5704ed75de5SKarsten Graul 571387707fdSKarsten Graul rc = smc_rtoken_add(link, 5724ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 5734ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 5744ed75de5SKarsten Graul 5754ed75de5SKarsten Graul /* ignore rtokens for other links, we have only one link */ 5764ed75de5SKarsten Graul 5774ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 5784ed75de5SKarsten Graul if (rc < 0) 5794ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 5806c8968c4SKarsten Graul smc_llc_send_message(link, llc); 5814ed75de5SKarsten Graul } 5824ed75de5SKarsten Graul 5834ed75de5SKarsten Graul static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link, 5844ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont *llc) 5854ed75de5SKarsten Graul { 5864ed75de5SKarsten Graul /* ignore rtokens for other links, we have only one link */ 5874ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 5886c8968c4SKarsten Graul smc_llc_send_message(link, llc); 5894ed75de5SKarsten Graul } 5904ed75de5SKarsten Graul 5914ed75de5SKarsten Graul static void smc_llc_rx_delete_rkey(struct smc_link *link, 5924ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey *llc) 5934ed75de5SKarsten Graul { 5944ed75de5SKarsten Graul u8 err_mask = 0; 5954ed75de5SKarsten Graul int i, max; 5964ed75de5SKarsten Graul 5974ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 5984ed75de5SKarsten Graul for (i = 0; i < max; i++) { 599387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 6004ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 6014ed75de5SKarsten Graul } 6024ed75de5SKarsten Graul 6034ed75de5SKarsten Graul if (err_mask) { 6044ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 6054ed75de5SKarsten Graul llc->err_mask = err_mask; 6064ed75de5SKarsten Graul } 6074ed75de5SKarsten Graul 6084ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 6096c8968c4SKarsten Graul smc_llc_send_message(link, llc); 6104ed75de5SKarsten Graul } 6114ed75de5SKarsten Graul 6126c8968c4SKarsten Graul /* flush the llc event queue */ 61300a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 6149bf9abeaSUrsula Braun { 6156c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 6169bf9abeaSUrsula Braun 6176c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 6186c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 6196c8968c4SKarsten Graul list_del_init(&qentry->list); 6206c8968c4SKarsten Graul kfree(qentry); 6216c8968c4SKarsten Graul } 6226c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 6236c8968c4SKarsten Graul } 6246c8968c4SKarsten Graul 6256c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 6266c8968c4SKarsten Graul { 6276c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 6286c8968c4SKarsten Graul struct smc_link *link = qentry->link; 6290fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 6306c8968c4SKarsten Graul 631d854fcbfSKarsten Graul if (!smc_link_usable(link)) 6326c8968c4SKarsten Graul goto out; 633313164daSKarsten Graul 634313164daSKarsten Graul switch (llc->raw.hdr.common.type) { 635313164daSKarsten Graul case SMC_LLC_TEST_LINK: 636*56e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 637*56e8091cSKarsten Graul smc_llc_send_message(link, llc); 638313164daSKarsten Graul break; 63952bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 6400fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 6410fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 6420fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 6430fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 6440fb0b02bSKarsten Graul /* a flow is waiting for this message */ 6450fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 6460fb0b02bSKarsten Graul qentry); 6470fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 6480fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 6490fb0b02bSKarsten Graul qentry)) { 6500fb0b02bSKarsten Graul /* tbd: schedule_work(&lgr->llc_add_link_work); */ 6510fb0b02bSKarsten Graul } 6520fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 6530fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 6540fb0b02bSKarsten Graul /* tbd: schedule_work(&lgr->llc_add_link_work); */ 6550fb0b02bSKarsten Graul } 6560fb0b02bSKarsten Graul return; 6570fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 6580fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 6590fb0b02bSKarsten Graul /* a flow is waiting for this message */ 6600fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 6610fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 6620fb0b02bSKarsten Graul return; 6630fb0b02bSKarsten Graul } 66452bedf37SKarsten Graul break; 66552bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 66652bedf37SKarsten Graul smc_llc_rx_delete_link(link, &llc->delete_link); 66752bedf37SKarsten Graul break; 6684ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 6694ed75de5SKarsten Graul smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey); 6704ed75de5SKarsten Graul break; 6714ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 6724ed75de5SKarsten Graul smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont); 6734ed75de5SKarsten Graul break; 6744ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 6754ed75de5SKarsten Graul smc_llc_rx_delete_rkey(link, &llc->delete_rkey); 6764ed75de5SKarsten Graul break; 677313164daSKarsten Graul } 6786c8968c4SKarsten Graul out: 6796c8968c4SKarsten Graul kfree(qentry); 6806c8968c4SKarsten Graul } 6816c8968c4SKarsten Graul 6826c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 6836c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 6846c8968c4SKarsten Graul { 6856c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 6866c8968c4SKarsten Graul llc_event_work); 6876c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 6886c8968c4SKarsten Graul 689555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 690555da9afSKarsten Graul if (smc_link_usable(lgr->delayed_event->link)) { 691555da9afSKarsten Graul smc_llc_event_handler(lgr->delayed_event); 692555da9afSKarsten Graul } else { 693555da9afSKarsten Graul qentry = lgr->delayed_event; 694555da9afSKarsten Graul lgr->delayed_event = NULL; 695555da9afSKarsten Graul kfree(qentry); 696555da9afSKarsten Graul } 697555da9afSKarsten Graul } 698555da9afSKarsten Graul 6996c8968c4SKarsten Graul again: 7006c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 7016c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 7026c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 7036c8968c4SKarsten Graul struct smc_llc_qentry, list); 7046c8968c4SKarsten Graul list_del_init(&qentry->list); 7056c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7066c8968c4SKarsten Graul smc_llc_event_handler(qentry); 7076c8968c4SKarsten Graul goto again; 7086c8968c4SKarsten Graul } 7096c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7106c8968c4SKarsten Graul } 7116c8968c4SKarsten Graul 712ef79d439SKarsten Graul /* process llc responses in tasklet context */ 713a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 714a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 715ef79d439SKarsten Graul { 716a6688d91SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.type; 717ef79d439SKarsten Graul 718a6688d91SKarsten Graul switch (llc_type) { 719ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 720ef79d439SKarsten Graul if (link->state == SMC_LNK_ACTIVE) 721ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 722ef79d439SKarsten Graul break; 723ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 7244667bb4aSKarsten Graul case SMC_LLC_CONFIRM_LINK: 7253d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 7266d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 7274667bb4aSKarsten Graul /* assign responses to the local flow, we requested them */ 7284667bb4aSKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 7294667bb4aSKarsten Graul wake_up_interruptible(&link->lgr->llc_waiter); 7304667bb4aSKarsten Graul return; 731ef79d439SKarsten Graul case SMC_LLC_DELETE_LINK: 732ef79d439SKarsten Graul if (link->lgr->role == SMC_SERV) 733ef79d439SKarsten Graul smc_lgr_schedule_free_work_fast(link->lgr); 734ef79d439SKarsten Graul break; 735ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 736ef79d439SKarsten Graul /* unused as long as we don't send this type of msg */ 737ef79d439SKarsten Graul break; 738ef79d439SKarsten Graul } 739a6688d91SKarsten Graul kfree(qentry); 740ef79d439SKarsten Graul } 741ef79d439SKarsten Graul 742a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 7436c8968c4SKarsten Graul { 7446c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 7456c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 7466c8968c4SKarsten Graul unsigned long flags; 7476c8968c4SKarsten Graul 7486c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 7496c8968c4SKarsten Graul if (!qentry) 7506c8968c4SKarsten Graul return; 7516c8968c4SKarsten Graul qentry->link = link; 7526c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 7536c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 754a6688d91SKarsten Graul 755a6688d91SKarsten Graul /* process responses immediately */ 756a6688d91SKarsten Graul if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 757a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 758a6688d91SKarsten Graul return; 759a6688d91SKarsten Graul } 760a6688d91SKarsten Graul 761a6688d91SKarsten Graul /* add requests to event queue */ 7626c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 7636c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 7646c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 7656c8968c4SKarsten Graul schedule_work(&link->lgr->llc_event_work); 7669bf9abeaSUrsula Braun } 7679bf9abeaSUrsula Braun 768a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 769a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 770a6688d91SKarsten Graul { 771a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 772a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 773a6688d91SKarsten Graul 774a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 775a6688d91SKarsten Graul return; /* short message */ 776a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 777a6688d91SKarsten Graul return; /* invalid message */ 778a6688d91SKarsten Graul 779a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 780a6688d91SKarsten Graul } 781a6688d91SKarsten Graul 78244aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 783877ae5beSKarsten Graul 784877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 785877ae5beSKarsten Graul { 786877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 787877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 788877ae5beSKarsten Graul unsigned long next_interval; 789877ae5beSKarsten Graul unsigned long expire_time; 790877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 791877ae5beSKarsten Graul int rc; 792877ae5beSKarsten Graul 793877ae5beSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 794877ae5beSKarsten Graul return; /* don't reschedule worker */ 795877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 796877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 797877ae5beSKarsten Graul next_interval = expire_time - jiffies; 798877ae5beSKarsten Graul goto out; 799877ae5beSKarsten Graul } 800877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 801d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 802877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 803877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 804877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 8051020e1efSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 8061020e1efSKarsten Graul return; /* link state changed */ 807877ae5beSKarsten Graul if (rc <= 0) { 8085f78fe96SKarsten Graul smc_lgr_terminate_sched(smc_get_lgr(link)); 809877ae5beSKarsten Graul return; 810877ae5beSKarsten Graul } 811877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 812877ae5beSKarsten Graul out: 8131020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 814877ae5beSKarsten Graul } 815877ae5beSKarsten Graul 81600a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 81700a049cfSKarsten Graul { 81800a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 81900a049cfSKarsten Graul 82000a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 82100a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 82200a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 823555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 824555da9afSKarsten Graul init_waitqueue_head(&lgr->llc_waiter); 82500a049cfSKarsten Graul lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 82600a049cfSKarsten Graul } 82700a049cfSKarsten Graul 82800a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 82900a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 83000a049cfSKarsten Graul { 83100a049cfSKarsten Graul smc_llc_event_flush(lgr); 832555da9afSKarsten Graul wake_up_interruptible_all(&lgr->llc_waiter); 83300a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 834555da9afSKarsten Graul if (lgr->delayed_event) { 835555da9afSKarsten Graul kfree(lgr->delayed_event); 836555da9afSKarsten Graul lgr->delayed_event = NULL; 837555da9afSKarsten Graul } 83800a049cfSKarsten Graul } 83900a049cfSKarsten Graul 8402a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 841877ae5beSKarsten Graul { 842877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 843877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 8442a4c57a9SKarsten Graul return 0; 845b32cf4abSKarsten Graul } 846b32cf4abSKarsten Graul 84700a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 848b32cf4abSKarsten Graul { 849877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 85000a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 85100a049cfSKarsten Graul link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 8521020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 853877ae5beSKarsten Graul link->llc_testlink_time); 854877ae5beSKarsten Graul } 855877ae5beSKarsten Graul } 856877ae5beSKarsten Graul 8570d18a0cbSKarsten Graul void smc_llc_link_deleting(struct smc_link *link) 8580d18a0cbSKarsten Graul { 8590d18a0cbSKarsten Graul link->state = SMC_LNK_DELETING; 86015e1b99aSUrsula Braun smc_wr_wakeup_tx_wait(link); 8610d18a0cbSKarsten Graul } 8620d18a0cbSKarsten Graul 863877ae5beSKarsten Graul /* called in worker context */ 8642a4c57a9SKarsten Graul void smc_llc_link_clear(struct smc_link *link) 865877ae5beSKarsten Graul { 8662140ac26SKarsten Graul complete(&link->llc_testlink_resp); 8672140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 8682140ac26SKarsten Graul smc_wr_wakeup_reg_wait(link); 8692140ac26SKarsten Graul smc_wr_wakeup_tx_wait(link); 870877ae5beSKarsten Graul } 871877ae5beSKarsten Graul 8723d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 8733d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 87444aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 87544aa81ceSKarsten Graul { 8763d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 8773d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 8783d88a21bSKarsten Graul int rc = 0; 87944aa81ceSKarsten Graul 8803d88a21bSKarsten Graul rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); 8814600cfc3SKarsten Graul if (rc) 8824600cfc3SKarsten Graul return rc; 8833d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 8843d88a21bSKarsten Graul if (rc) 8853d88a21bSKarsten Graul goto out; 88644aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 8873d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 8883d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 8893d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 8903d88a21bSKarsten Graul rc = -EFAULT; 8913d88a21bSKarsten Graul out: 8923d88a21bSKarsten Graul if (qentry) 8933d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 8943d88a21bSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 8953d88a21bSKarsten Graul return rc; 89644aa81ceSKarsten Graul } 89744aa81ceSKarsten Graul 89860e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 8996d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 90060e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 90160e03c62SKarsten Graul { 9026d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 9036d74c3a8SKarsten Graul struct smc_link *send_link; 9040b29ec64SUrsula Braun int rc = 0; 90560e03c62SKarsten Graul 9066d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 9076d74c3a8SKarsten Graul if (!send_link) 9086d74c3a8SKarsten Graul return -ENOLINK; 9096d74c3a8SKarsten Graul 9106d74c3a8SKarsten Graul rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); 9116d74c3a8SKarsten Graul if (rc) 9126d74c3a8SKarsten Graul return rc; 9136d74c3a8SKarsten Graul /* protected by llc_flow control */ 9146d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 91560e03c62SKarsten Graul if (rc) 91660e03c62SKarsten Graul goto out; 91760e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 9186d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 9196d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 9206d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 92160e03c62SKarsten Graul rc = -EFAULT; 92260e03c62SKarsten Graul out: 9236d74c3a8SKarsten Graul if (qentry) 9246d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 9256d74c3a8SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 92660e03c62SKarsten Graul return rc; 92760e03c62SKarsten Graul } 92860e03c62SKarsten Graul 92992334cfcSKarsten Graul /* evaluate confirm link request or response */ 93092334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 93192334cfcSKarsten Graul enum smc_llc_reqresp type) 93292334cfcSKarsten Graul { 93392334cfcSKarsten Graul if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ 93492334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 93592334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 93692334cfcSKarsten Graul return -ENOTSUPP; 93792334cfcSKarsten Graul return 0; 93892334cfcSKarsten Graul } 93992334cfcSKarsten Graul 9409bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 9419bf9abeaSUrsula Braun 9429bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 9439bf9abeaSUrsula Braun { 9449bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 9459bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 9469bf9abeaSUrsula Braun }, 9479bf9abeaSUrsula Braun { 948313164daSKarsten Graul .handler = smc_llc_rx_handler, 949313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 950313164daSKarsten Graul }, 951313164daSKarsten Graul { 9524ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 95352bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 95452bedf37SKarsten Graul }, 95552bedf37SKarsten Graul { 95652bedf37SKarsten Graul .handler = smc_llc_rx_handler, 95752bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 95852bedf37SKarsten Graul }, 95952bedf37SKarsten Graul { 96052bedf37SKarsten Graul .handler = smc_llc_rx_handler, 9614ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 9624ed75de5SKarsten Graul }, 9634ed75de5SKarsten Graul { 9644ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 9654ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 9664ed75de5SKarsten Graul }, 9674ed75de5SKarsten Graul { 9684ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 9694ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 9704ed75de5SKarsten Graul }, 9714ed75de5SKarsten Graul { 9729bf9abeaSUrsula Braun .handler = NULL, 9739bf9abeaSUrsula Braun } 9749bf9abeaSUrsula Braun }; 9759bf9abeaSUrsula Braun 9769bf9abeaSUrsula Braun int __init smc_llc_init(void) 9779bf9abeaSUrsula Braun { 9789bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 9799bf9abeaSUrsula Braun int rc = 0; 9809bf9abeaSUrsula Braun 9819bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 9829bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 9839bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 9849bf9abeaSUrsula Braun if (rc) 9859bf9abeaSUrsula Braun break; 9869bf9abeaSUrsula Braun } 9879bf9abeaSUrsula Braun return rc; 9889bf9abeaSUrsula Braun } 989