1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 209bf9abeaSUrsula Braun 210f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 220f627126SStefan Raspl 230f627126SStefan Raspl struct smc_llc_hdr { 240f627126SStefan Raspl struct smc_wr_rx_hdr common; 250f627126SStefan Raspl u8 length; /* 44 */ 2652bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 2752bedf37SKarsten Graul u8 reserved:4, 2852bedf37SKarsten Graul add_link_rej_rsn:4; 2952bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3052bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3152bedf37SKarsten Graul reserved:4; 3252bedf37SKarsten Graul #endif 330f627126SStefan Raspl u8 flags; 340f627126SStefan Raspl }; 350f627126SStefan Raspl 3675d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 3775d320d6SKarsten Graul 380f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 390f627126SStefan Raspl struct smc_llc_hdr hd; 400f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 410f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 420f627126SStefan Raspl u8 sender_qp_num[3]; 430f627126SStefan Raspl u8 link_num; 440f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 450f627126SStefan Raspl u8 max_links; 460f627126SStefan Raspl u8 reserved[9]; 470f627126SStefan Raspl }; 480f627126SStefan Raspl 4952bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5052bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 5152bedf37SKarsten Graul 5252bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 5352bedf37SKarsten Graul 5452bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 5552bedf37SKarsten Graul struct smc_llc_hdr hd; 5652bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 5752bedf37SKarsten Graul u8 reserved2[2]; 5852bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 5952bedf37SKarsten Graul u8 sender_qp_num[3]; 6052bedf37SKarsten Graul u8 link_num; 6152bedf37SKarsten Graul u8 flags2; /* QP mtu */ 6252bedf37SKarsten Graul u8 initial_psn[3]; 6352bedf37SKarsten Graul u8 reserved[8]; 6452bedf37SKarsten Graul }; 6552bedf37SKarsten Graul 6652bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 6752bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 6852bedf37SKarsten Graul 6952bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 7052bedf37SKarsten Graul struct smc_llc_hdr hd; 7152bedf37SKarsten Graul u8 link_num; 7252bedf37SKarsten Graul __be32 reason; 7352bedf37SKarsten Graul u8 reserved[35]; 7452bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 7552bedf37SKarsten Graul 76313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 77313164daSKarsten Graul struct smc_llc_hdr hd; 78313164daSKarsten Graul u8 user_data[16]; 79313164daSKarsten Graul u8 reserved[24]; 80313164daSKarsten Graul }; 81313164daSKarsten Graul 824ed75de5SKarsten Graul struct smc_rmb_rtoken { 834ed75de5SKarsten Graul union { 844ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 854ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 864ed75de5SKarsten Graul /* rtoken is always for the current link */ 874ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 884ed75de5SKarsten Graul }; 894ed75de5SKarsten Graul __be32 rmb_key; 904ed75de5SKarsten Graul __be64 rmb_vaddr; 914ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 924ed75de5SKarsten Graul 934ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 944ed75de5SKarsten Graul 954ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 964ed75de5SKarsten Graul struct smc_llc_hdr hd; 974ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 984ed75de5SKarsten Graul u8 reserved; 994ed75de5SKarsten Graul }; 1004ed75de5SKarsten Graul 1014ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */ 1024ed75de5SKarsten Graul struct smc_llc_hdr hd; 1034ed75de5SKarsten Graul u8 num_rkeys; 1044ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1054ed75de5SKarsten Graul }; 1064ed75de5SKarsten Graul 1074ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 108*3bc67e09SKarsten Graul #define SMC_LLC_FLAG_RKEY_RETRY 0x10 1094ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1104ed75de5SKarsten Graul 1114ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1124ed75de5SKarsten Graul struct smc_llc_hdr hd; 1134ed75de5SKarsten Graul u8 num_rkeys; 1144ed75de5SKarsten Graul u8 err_mask; 1154ed75de5SKarsten Graul u8 reserved[2]; 1164ed75de5SKarsten Graul __be32 rkey[8]; 1174ed75de5SKarsten Graul u8 reserved2[4]; 1184ed75de5SKarsten Graul }; 1194ed75de5SKarsten Graul 1200f627126SStefan Raspl union smc_llc_msg { 1210f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 12252bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 12352bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1244ed75de5SKarsten Graul 1254ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1264ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont; 1274ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1284ed75de5SKarsten Graul 129313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1300f627126SStefan Raspl struct { 1310f627126SStefan Raspl struct smc_llc_hdr hdr; 1320f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1330f627126SStefan Raspl } raw; 1340f627126SStefan Raspl }; 1350f627126SStefan Raspl 1360f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1370f627126SStefan Raspl 1386c8968c4SKarsten Graul struct smc_llc_qentry { 1396c8968c4SKarsten Graul struct list_head list; 1406c8968c4SKarsten Graul struct smc_link *link; 1416c8968c4SKarsten Graul union smc_llc_msg msg; 1426c8968c4SKarsten Graul }; 1436c8968c4SKarsten Graul 144555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 145555da9afSKarsten Graul { 146555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 147555da9afSKarsten Graul 148555da9afSKarsten Graul flow->qentry = NULL; 149555da9afSKarsten Graul return qentry; 150555da9afSKarsten Graul } 151555da9afSKarsten Graul 152555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 153555da9afSKarsten Graul { 154555da9afSKarsten Graul struct smc_llc_qentry *qentry; 155555da9afSKarsten Graul 156555da9afSKarsten Graul if (flow->qentry) { 157555da9afSKarsten Graul qentry = flow->qentry; 158555da9afSKarsten Graul flow->qentry = NULL; 159555da9afSKarsten Graul kfree(qentry); 160555da9afSKarsten Graul } 161555da9afSKarsten Graul } 162555da9afSKarsten Graul 163555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 164555da9afSKarsten Graul struct smc_llc_qentry *qentry) 165555da9afSKarsten Graul { 166555da9afSKarsten Graul flow->qentry = qentry; 167555da9afSKarsten Graul } 168555da9afSKarsten Graul 169555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 170555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 171555da9afSKarsten Graul struct smc_llc_qentry *qentry) 172555da9afSKarsten Graul { 173555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 174555da9afSKarsten Graul 175555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 176555da9afSKarsten Graul if (flow->type) { 177555da9afSKarsten Graul /* a flow is already active */ 178555da9afSKarsten Graul if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || 179555da9afSKarsten Graul qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && 180555da9afSKarsten Graul !lgr->delayed_event) { 181555da9afSKarsten Graul lgr->delayed_event = qentry; 182555da9afSKarsten Graul } else { 183555da9afSKarsten Graul /* forget this llc request */ 184555da9afSKarsten Graul kfree(qentry); 185555da9afSKarsten Graul } 186555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 187555da9afSKarsten Graul return false; 188555da9afSKarsten Graul } 189555da9afSKarsten Graul switch (qentry->msg.raw.hdr.common.type) { 190555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 191555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 192555da9afSKarsten Graul break; 193555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 194555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 195555da9afSKarsten Graul break; 196555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 197555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 198555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 199555da9afSKarsten Graul break; 200555da9afSKarsten Graul default: 201555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 202555da9afSKarsten Graul } 203555da9afSKarsten Graul if (qentry == lgr->delayed_event) 204555da9afSKarsten Graul lgr->delayed_event = NULL; 205555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 206555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 207555da9afSKarsten Graul return true; 208555da9afSKarsten Graul } 209555da9afSKarsten Graul 210555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 211555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 212555da9afSKarsten Graul enum smc_llc_flowtype type) 213555da9afSKarsten Graul { 214555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 215555da9afSKarsten Graul int rc; 216555da9afSKarsten Graul 217555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 218555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 219555da9afSKarsten Graul */ 220555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 221555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 222555da9afSKarsten Graul again: 223555da9afSKarsten Graul if (list_empty(&lgr->list)) 224555da9afSKarsten Graul return -ENODEV; 225555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 226555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 227555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 228555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 229555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 230555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 231555da9afSKarsten Graul return 0; 232555da9afSKarsten Graul } 233555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 234555da9afSKarsten Graul rc = wait_event_interruptible_timeout(lgr->llc_waiter, 235555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 236555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 237555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)), 238555da9afSKarsten Graul SMC_LLC_WAIT_TIME); 239555da9afSKarsten Graul if (!rc) 240555da9afSKarsten Graul return -ETIMEDOUT; 241555da9afSKarsten Graul goto again; 242555da9afSKarsten Graul } 243555da9afSKarsten Graul 244555da9afSKarsten Graul /* finish the current llc flow */ 245555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 246555da9afSKarsten Graul { 247555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 248555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 249555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 250555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 251555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 252555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 253555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 254555da9afSKarsten Graul else 255555da9afSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 256555da9afSKarsten Graul } 257555da9afSKarsten Graul 258555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 259555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 260555da9afSKarsten Graul */ 261555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 262555da9afSKarsten Graul struct smc_link *lnk, 263555da9afSKarsten Graul int time_out, u8 exp_msg) 264555da9afSKarsten Graul { 265555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 266555da9afSKarsten Graul 267555da9afSKarsten Graul wait_event_interruptible_timeout(lgr->llc_waiter, 268555da9afSKarsten Graul (flow->qentry || 269555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 270555da9afSKarsten Graul list_empty(&lgr->list)), 271555da9afSKarsten Graul time_out); 272555da9afSKarsten Graul if (!flow->qentry || 273555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 274555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 275555da9afSKarsten Graul goto out; 276555da9afSKarsten Graul } 277555da9afSKarsten Graul if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { 278555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 279555da9afSKarsten Graul flow->qentry->msg.raw.hdr.common.type == 280555da9afSKarsten Graul SMC_LLC_DELETE_LINK) { 281555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 282555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 283555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 284555da9afSKarsten Graul return NULL; 285555da9afSKarsten Graul } 286555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 287555da9afSKarsten Graul } 288555da9afSKarsten Graul out: 289555da9afSKarsten Graul return flow->qentry; 290555da9afSKarsten Graul } 291555da9afSKarsten Graul 2929bf9abeaSUrsula Braun /********************************** send *************************************/ 2939bf9abeaSUrsula Braun 2949bf9abeaSUrsula Braun struct smc_llc_tx_pend { 2959bf9abeaSUrsula Braun }; 2969bf9abeaSUrsula Braun 2979bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 2989bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 2999bf9abeaSUrsula Braun struct smc_link *link, 3009bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3019bf9abeaSUrsula Braun { 3029bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3039bf9abeaSUrsula Braun } 3049bf9abeaSUrsula Braun 3059bf9abeaSUrsula Braun /** 3069bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3079bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3089bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3099bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3109bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3119bf9abeaSUrsula Braun * 3129bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3139bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3149bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3159bf9abeaSUrsula Braun * 3169bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 3179bf9abeaSUrsula Braun */ 3189bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 3199bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 3209bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 3219bf9abeaSUrsula Braun { 3229bf9abeaSUrsula Braun int rc; 3239bf9abeaSUrsula Braun 324ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 325ad6f317fSUrsula Braun pend); 3269bf9abeaSUrsula Braun if (rc < 0) 3279bf9abeaSUrsula Braun return rc; 3289bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3299bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 3309bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 3319bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3329bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 3339bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 3349bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3359bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 3369bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 3379bf9abeaSUrsula Braun return 0; 3389bf9abeaSUrsula Braun } 3399bf9abeaSUrsula Braun 3409bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 341947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 3429bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 3439bf9abeaSUrsula Braun { 34400e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 3459bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 3469bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 3479bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 3489bf9abeaSUrsula Braun int rc; 3499bf9abeaSUrsula Braun 3509bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 3519bf9abeaSUrsula Braun if (rc) 3529bf9abeaSUrsula Braun return rc; 3539bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 3549bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 3559bf9abeaSUrsula Braun confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 3569bf9abeaSUrsula Braun confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 35775d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 3589bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 3599bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 360947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 361947541f3SUrsula Braun ETH_ALEN); 3627005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 3639bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 3642be922f3SKarsten Graul confllc->link_num = link->link_id; 3659bf9abeaSUrsula Braun memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); 36652bedf37SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ 36752bedf37SKarsten Graul /* send llc message */ 36852bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 36952bedf37SKarsten Graul return rc; 37052bedf37SKarsten Graul } 37152bedf37SKarsten Graul 37244aa81ceSKarsten Graul /* send LLC confirm rkey request */ 3733d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 37444aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 37544aa81ceSKarsten Graul { 37644aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 37744aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 37844aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 3793d88a21bSKarsten Graul struct smc_link *link; 3803d88a21bSKarsten Graul int i, rc, rtok_ix; 38144aa81ceSKarsten Graul 3823d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 38344aa81ceSKarsten Graul if (rc) 38444aa81ceSKarsten Graul return rc; 38544aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 38644aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 38744aa81ceSKarsten Graul rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 38844aa81ceSKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 3893d88a21bSKarsten Graul 3903d88a21bSKarsten Graul rtok_ix = 1; 3913d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 3923d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 3933d88a21bSKarsten Graul if (link->state == SMC_LNK_ACTIVE && link != send_link) { 3943d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 3953d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 396387707fdSKarsten Graul htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 3973d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 3983d88a21bSKarsten Graul (u64)sg_dma_address( 3993d88a21bSKarsten Graul rmb_desc->sgt[link->link_idx].sgl)); 4003d88a21bSKarsten Graul rtok_ix++; 4013d88a21bSKarsten Graul } 4023d88a21bSKarsten Graul } 4033d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 4043d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 4053d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 4063d88a21bSKarsten Graul htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 40744aa81ceSKarsten Graul rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 4083d88a21bSKarsten Graul (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 40944aa81ceSKarsten Graul /* send llc message */ 4103d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 41144aa81ceSKarsten Graul return rc; 41244aa81ceSKarsten Graul } 41344aa81ceSKarsten Graul 41460e03c62SKarsten Graul /* send LLC delete rkey request */ 41560e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 41660e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 41760e03c62SKarsten Graul { 41860e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 41960e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 42060e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 42160e03c62SKarsten Graul int rc; 42260e03c62SKarsten Graul 42360e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 42460e03c62SKarsten Graul if (rc) 42560e03c62SKarsten Graul return rc; 42660e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 42760e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 42860e03c62SKarsten Graul rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 42960e03c62SKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 43060e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 431387707fdSKarsten Graul rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 43260e03c62SKarsten Graul /* send llc message */ 43360e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 43460e03c62SKarsten Graul return rc; 43560e03c62SKarsten Graul } 43660e03c62SKarsten Graul 4372a4c57a9SKarsten Graul /* prepare an add link message */ 4382a4c57a9SKarsten Graul static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc, 4397005ada6SUrsula Braun struct smc_link *link, u8 mac[], u8 gid[], 4402a4c57a9SKarsten Graul enum smc_llc_reqresp reqresp) 4412a4c57a9SKarsten Graul { 4422a4c57a9SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 4432a4c57a9SKarsten Graul addllc->hd.common.type = SMC_LLC_ADD_LINK; 4442a4c57a9SKarsten Graul addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 4452a4c57a9SKarsten Graul if (reqresp == SMC_LLC_RESP) { 4462a4c57a9SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 4472a4c57a9SKarsten Graul /* always reject more links for now */ 4482a4c57a9SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 4492a4c57a9SKarsten Graul addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 4502a4c57a9SKarsten Graul } 4512a4c57a9SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 4522a4c57a9SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 4532a4c57a9SKarsten Graul } 4542a4c57a9SKarsten Graul 45552bedf37SKarsten Graul /* send ADD LINK request or response */ 4567005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 45752bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 45852bedf37SKarsten Graul { 45952bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 46052bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 46152bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 46252bedf37SKarsten Graul int rc; 46352bedf37SKarsten Graul 46452bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 46552bedf37SKarsten Graul if (rc) 46652bedf37SKarsten Graul return rc; 46752bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 4682a4c57a9SKarsten Graul smc_llc_prep_add_link(addllc, link, mac, gid, reqresp); 46952bedf37SKarsten Graul /* send llc message */ 47052bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 47152bedf37SKarsten Graul return rc; 47252bedf37SKarsten Graul } 47352bedf37SKarsten Graul 4742a4c57a9SKarsten Graul /* prepare a delete link message */ 4752a4c57a9SKarsten Graul static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc, 4762a4c57a9SKarsten Graul struct smc_link *link, 4770d18a0cbSKarsten Graul enum smc_llc_reqresp reqresp, bool orderly) 4782a4c57a9SKarsten Graul { 4792a4c57a9SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 4802a4c57a9SKarsten Graul delllc->hd.common.type = SMC_LLC_DELETE_LINK; 4812a4c57a9SKarsten Graul delllc->hd.length = sizeof(struct smc_llc_msg_add_link); 4822a4c57a9SKarsten Graul if (reqresp == SMC_LLC_RESP) 4832a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 4842a4c57a9SKarsten Graul /* DEL_LINK_ALL because only 1 link supported */ 4852a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 4860d18a0cbSKarsten Graul if (orderly) 4872a4c57a9SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 4882a4c57a9SKarsten Graul delllc->link_num = link->link_id; 4892a4c57a9SKarsten Graul } 4902a4c57a9SKarsten Graul 49152bedf37SKarsten Graul /* send DELETE LINK request or response */ 49252bedf37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, 4930d18a0cbSKarsten Graul enum smc_llc_reqresp reqresp, bool orderly) 49452bedf37SKarsten Graul { 49552bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 49652bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 49752bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 49852bedf37SKarsten Graul int rc; 49952bedf37SKarsten Graul 50052bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 50152bedf37SKarsten Graul if (rc) 50252bedf37SKarsten Graul return rc; 50352bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 5040d18a0cbSKarsten Graul smc_llc_prep_delete_link(delllc, link, reqresp, orderly); 5059bf9abeaSUrsula Braun /* send llc message */ 5069bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 5079bf9abeaSUrsula Braun return rc; 5089bf9abeaSUrsula Braun } 5099bf9abeaSUrsula Braun 510d97935faSKarsten Graul /* send LLC test link request */ 511d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 512313164daSKarsten Graul { 513313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 514313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 515313164daSKarsten Graul struct smc_wr_buf *wr_buf; 516313164daSKarsten Graul int rc; 517313164daSKarsten Graul 518313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 519313164daSKarsten Graul if (rc) 520313164daSKarsten Graul return rc; 521313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 522313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 523313164daSKarsten Graul testllc->hd.common.type = SMC_LLC_TEST_LINK; 524313164daSKarsten Graul testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 525313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 526313164daSKarsten Graul /* send llc message */ 527313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 528313164daSKarsten Graul return rc; 529313164daSKarsten Graul } 530313164daSKarsten Graul 5316c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 5326c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 5334ed75de5SKarsten Graul { 5344ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 5354ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 5364ed75de5SKarsten Graul int rc; 5374ed75de5SKarsten Graul 5386c8968c4SKarsten Graul if (!smc_link_usable(link)) 5396c8968c4SKarsten Graul return -ENOLINK; 5406c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 5414ed75de5SKarsten Graul if (rc) 5426c8968c4SKarsten Graul return rc; 5436c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 5446c8968c4SKarsten Graul return smc_wr_tx_send(link, pend); 5454ed75de5SKarsten Graul } 5464ed75de5SKarsten Graul 5479bf9abeaSUrsula Braun /********************************* receive ***********************************/ 5489bf9abeaSUrsula Braun 54952bedf37SKarsten Graul static void smc_llc_rx_delete_link(struct smc_link *link, 55052bedf37SKarsten Graul struct smc_llc_msg_del_link *llc) 55152bedf37SKarsten Graul { 55200e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 55352bedf37SKarsten Graul 5549651b934SKarsten Graul smc_lgr_forget(lgr); 5550d18a0cbSKarsten Graul smc_llc_link_deleting(link); 5560d18a0cbSKarsten Graul if (lgr->role == SMC_SERV) { 5570d18a0cbSKarsten Graul /* client asks to delete this link, send request */ 5580d18a0cbSKarsten Graul smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true); 55952bedf37SKarsten Graul } else { 5600d18a0cbSKarsten Graul /* server requests to delete this link, send response */ 5610d18a0cbSKarsten Graul smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true); 56252bedf37SKarsten Graul } 5636c8968c4SKarsten Graul smc_llc_send_message(link, llc); 564f528ba24SUrsula Braun smc_lgr_terminate_sched(lgr); 56552bedf37SKarsten Graul } 56652bedf37SKarsten Graul 567*3bc67e09SKarsten Graul /* process a confirm_rkey request from peer, remote flow */ 568*3bc67e09SKarsten Graul static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 5694ed75de5SKarsten Graul { 570*3bc67e09SKarsten Graul struct smc_llc_msg_confirm_rkey *llc; 571*3bc67e09SKarsten Graul struct smc_llc_qentry *qentry; 572*3bc67e09SKarsten Graul struct smc_link *link; 573*3bc67e09SKarsten Graul int num_entries; 574*3bc67e09SKarsten Graul int rk_idx; 575*3bc67e09SKarsten Graul int i; 5764ed75de5SKarsten Graul 577*3bc67e09SKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 578*3bc67e09SKarsten Graul llc = &qentry->msg.confirm_rkey; 579*3bc67e09SKarsten Graul link = qentry->link; 580*3bc67e09SKarsten Graul 581*3bc67e09SKarsten Graul num_entries = llc->rtoken[0].num_rkeys; 582*3bc67e09SKarsten Graul /* first rkey entry is for receiving link */ 583*3bc67e09SKarsten Graul rk_idx = smc_rtoken_add(link, 5844ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 5854ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 586*3bc67e09SKarsten Graul if (rk_idx < 0) 587*3bc67e09SKarsten Graul goto out_err; 5884ed75de5SKarsten Graul 589*3bc67e09SKarsten Graul for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 590*3bc67e09SKarsten Graul smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 591*3bc67e09SKarsten Graul llc->rtoken[i].rmb_vaddr, 592*3bc67e09SKarsten Graul llc->rtoken[i].rmb_key); 593*3bc67e09SKarsten Graul /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 594*3bc67e09SKarsten Graul goto out; 595*3bc67e09SKarsten Graul out_err: 5964ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 597*3bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 598*3bc67e09SKarsten Graul out: 599*3bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 600*3bc67e09SKarsten Graul smc_llc_send_message(link, &qentry->msg); 601*3bc67e09SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 6024ed75de5SKarsten Graul } 6034ed75de5SKarsten Graul 6044ed75de5SKarsten Graul static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link, 6054ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey_cont *llc) 6064ed75de5SKarsten Graul { 6074ed75de5SKarsten Graul /* ignore rtokens for other links, we have only one link */ 6084ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 6096c8968c4SKarsten Graul smc_llc_send_message(link, llc); 6104ed75de5SKarsten Graul } 6114ed75de5SKarsten Graul 6124ed75de5SKarsten Graul static void smc_llc_rx_delete_rkey(struct smc_link *link, 6134ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey *llc) 6144ed75de5SKarsten Graul { 6154ed75de5SKarsten Graul u8 err_mask = 0; 6164ed75de5SKarsten Graul int i, max; 6174ed75de5SKarsten Graul 6184ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 6194ed75de5SKarsten Graul for (i = 0; i < max; i++) { 620387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 6214ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 6224ed75de5SKarsten Graul } 6234ed75de5SKarsten Graul 6244ed75de5SKarsten Graul if (err_mask) { 6254ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 6264ed75de5SKarsten Graul llc->err_mask = err_mask; 6274ed75de5SKarsten Graul } 6284ed75de5SKarsten Graul 6294ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 6306c8968c4SKarsten Graul smc_llc_send_message(link, llc); 6314ed75de5SKarsten Graul } 6324ed75de5SKarsten Graul 6336c8968c4SKarsten Graul /* flush the llc event queue */ 63400a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 6359bf9abeaSUrsula Braun { 6366c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 6379bf9abeaSUrsula Braun 6386c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 6396c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 6406c8968c4SKarsten Graul list_del_init(&qentry->list); 6416c8968c4SKarsten Graul kfree(qentry); 6426c8968c4SKarsten Graul } 6436c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 6446c8968c4SKarsten Graul } 6456c8968c4SKarsten Graul 6466c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 6476c8968c4SKarsten Graul { 6486c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 6496c8968c4SKarsten Graul struct smc_link *link = qentry->link; 6500fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 6516c8968c4SKarsten Graul 652d854fcbfSKarsten Graul if (!smc_link_usable(link)) 6536c8968c4SKarsten Graul goto out; 654313164daSKarsten Graul 655313164daSKarsten Graul switch (llc->raw.hdr.common.type) { 656313164daSKarsten Graul case SMC_LLC_TEST_LINK: 65756e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 65856e8091cSKarsten Graul smc_llc_send_message(link, llc); 659313164daSKarsten Graul break; 66052bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 6610fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 6620fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 6630fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 6640fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 6650fb0b02bSKarsten Graul /* a flow is waiting for this message */ 6660fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 6670fb0b02bSKarsten Graul qentry); 6680fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 6690fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 6700fb0b02bSKarsten Graul qentry)) { 6710fb0b02bSKarsten Graul /* tbd: schedule_work(&lgr->llc_add_link_work); */ 6720fb0b02bSKarsten Graul } 6730fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 6740fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 6750fb0b02bSKarsten Graul /* tbd: schedule_work(&lgr->llc_add_link_work); */ 6760fb0b02bSKarsten Graul } 6770fb0b02bSKarsten Graul return; 6780fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 6790fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 6800fb0b02bSKarsten Graul /* a flow is waiting for this message */ 6810fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 6820fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 6830fb0b02bSKarsten Graul return; 6840fb0b02bSKarsten Graul } 68552bedf37SKarsten Graul break; 68652bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 68752bedf37SKarsten Graul smc_llc_rx_delete_link(link, &llc->delete_link); 68852bedf37SKarsten Graul break; 6894ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 690*3bc67e09SKarsten Graul /* new request from remote, assign to remote flow */ 691*3bc67e09SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 692*3bc67e09SKarsten Graul /* process here, does not wait for more llc msgs */ 693*3bc67e09SKarsten Graul smc_llc_rmt_conf_rkey(lgr); 694*3bc67e09SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 695*3bc67e09SKarsten Graul } 696*3bc67e09SKarsten Graul return; 6974ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 6984ed75de5SKarsten Graul smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont); 6994ed75de5SKarsten Graul break; 7004ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 7014ed75de5SKarsten Graul smc_llc_rx_delete_rkey(link, &llc->delete_rkey); 7024ed75de5SKarsten Graul break; 703313164daSKarsten Graul } 7046c8968c4SKarsten Graul out: 7056c8968c4SKarsten Graul kfree(qentry); 7066c8968c4SKarsten Graul } 7076c8968c4SKarsten Graul 7086c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 7096c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 7106c8968c4SKarsten Graul { 7116c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 7126c8968c4SKarsten Graul llc_event_work); 7136c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 7146c8968c4SKarsten Graul 715555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 716555da9afSKarsten Graul if (smc_link_usable(lgr->delayed_event->link)) { 717555da9afSKarsten Graul smc_llc_event_handler(lgr->delayed_event); 718555da9afSKarsten Graul } else { 719555da9afSKarsten Graul qentry = lgr->delayed_event; 720555da9afSKarsten Graul lgr->delayed_event = NULL; 721555da9afSKarsten Graul kfree(qentry); 722555da9afSKarsten Graul } 723555da9afSKarsten Graul } 724555da9afSKarsten Graul 7256c8968c4SKarsten Graul again: 7266c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 7276c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 7286c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 7296c8968c4SKarsten Graul struct smc_llc_qentry, list); 7306c8968c4SKarsten Graul list_del_init(&qentry->list); 7316c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7326c8968c4SKarsten Graul smc_llc_event_handler(qentry); 7336c8968c4SKarsten Graul goto again; 7346c8968c4SKarsten Graul } 7356c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7366c8968c4SKarsten Graul } 7376c8968c4SKarsten Graul 738ef79d439SKarsten Graul /* process llc responses in tasklet context */ 739a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 740a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 741ef79d439SKarsten Graul { 742a6688d91SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.type; 743ef79d439SKarsten Graul 744a6688d91SKarsten Graul switch (llc_type) { 745ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 746ef79d439SKarsten Graul if (link->state == SMC_LNK_ACTIVE) 747ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 748ef79d439SKarsten Graul break; 749ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 7504667bb4aSKarsten Graul case SMC_LLC_CONFIRM_LINK: 7513d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 7526d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 7534667bb4aSKarsten Graul /* assign responses to the local flow, we requested them */ 7544667bb4aSKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 7554667bb4aSKarsten Graul wake_up_interruptible(&link->lgr->llc_waiter); 7564667bb4aSKarsten Graul return; 757ef79d439SKarsten Graul case SMC_LLC_DELETE_LINK: 758ef79d439SKarsten Graul if (link->lgr->role == SMC_SERV) 759ef79d439SKarsten Graul smc_lgr_schedule_free_work_fast(link->lgr); 760ef79d439SKarsten Graul break; 761ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 762ef79d439SKarsten Graul /* unused as long as we don't send this type of msg */ 763ef79d439SKarsten Graul break; 764ef79d439SKarsten Graul } 765a6688d91SKarsten Graul kfree(qentry); 766ef79d439SKarsten Graul } 767ef79d439SKarsten Graul 768a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 7696c8968c4SKarsten Graul { 7706c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 7716c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 7726c8968c4SKarsten Graul unsigned long flags; 7736c8968c4SKarsten Graul 7746c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 7756c8968c4SKarsten Graul if (!qentry) 7766c8968c4SKarsten Graul return; 7776c8968c4SKarsten Graul qentry->link = link; 7786c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 7796c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 780a6688d91SKarsten Graul 781a6688d91SKarsten Graul /* process responses immediately */ 782a6688d91SKarsten Graul if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 783a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 784a6688d91SKarsten Graul return; 785a6688d91SKarsten Graul } 786a6688d91SKarsten Graul 787a6688d91SKarsten Graul /* add requests to event queue */ 7886c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 7896c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 7906c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 7916c8968c4SKarsten Graul schedule_work(&link->lgr->llc_event_work); 7929bf9abeaSUrsula Braun } 7939bf9abeaSUrsula Braun 794a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 795a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 796a6688d91SKarsten Graul { 797a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 798a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 799a6688d91SKarsten Graul 800a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 801a6688d91SKarsten Graul return; /* short message */ 802a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 803a6688d91SKarsten Graul return; /* invalid message */ 804a6688d91SKarsten Graul 805a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 806a6688d91SKarsten Graul } 807a6688d91SKarsten Graul 80844aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 809877ae5beSKarsten Graul 810877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 811877ae5beSKarsten Graul { 812877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 813877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 814877ae5beSKarsten Graul unsigned long next_interval; 815877ae5beSKarsten Graul unsigned long expire_time; 816877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 817877ae5beSKarsten Graul int rc; 818877ae5beSKarsten Graul 819877ae5beSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 820877ae5beSKarsten Graul return; /* don't reschedule worker */ 821877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 822877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 823877ae5beSKarsten Graul next_interval = expire_time - jiffies; 824877ae5beSKarsten Graul goto out; 825877ae5beSKarsten Graul } 826877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 827d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 828877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 829877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 830877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 8311020e1efSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 8321020e1efSKarsten Graul return; /* link state changed */ 833877ae5beSKarsten Graul if (rc <= 0) { 8345f78fe96SKarsten Graul smc_lgr_terminate_sched(smc_get_lgr(link)); 835877ae5beSKarsten Graul return; 836877ae5beSKarsten Graul } 837877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 838877ae5beSKarsten Graul out: 8391020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 840877ae5beSKarsten Graul } 841877ae5beSKarsten Graul 84200a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 84300a049cfSKarsten Graul { 84400a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 84500a049cfSKarsten Graul 84600a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 84700a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 84800a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 849555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 850555da9afSKarsten Graul init_waitqueue_head(&lgr->llc_waiter); 85100a049cfSKarsten Graul lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 85200a049cfSKarsten Graul } 85300a049cfSKarsten Graul 85400a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 85500a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 85600a049cfSKarsten Graul { 85700a049cfSKarsten Graul smc_llc_event_flush(lgr); 858555da9afSKarsten Graul wake_up_interruptible_all(&lgr->llc_waiter); 85900a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 860555da9afSKarsten Graul if (lgr->delayed_event) { 861555da9afSKarsten Graul kfree(lgr->delayed_event); 862555da9afSKarsten Graul lgr->delayed_event = NULL; 863555da9afSKarsten Graul } 86400a049cfSKarsten Graul } 86500a049cfSKarsten Graul 8662a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 867877ae5beSKarsten Graul { 868877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 869877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 8702a4c57a9SKarsten Graul return 0; 871b32cf4abSKarsten Graul } 872b32cf4abSKarsten Graul 87300a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 874b32cf4abSKarsten Graul { 875877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 87600a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 87700a049cfSKarsten Graul link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 8781020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 879877ae5beSKarsten Graul link->llc_testlink_time); 880877ae5beSKarsten Graul } 881877ae5beSKarsten Graul } 882877ae5beSKarsten Graul 8830d18a0cbSKarsten Graul void smc_llc_link_deleting(struct smc_link *link) 8840d18a0cbSKarsten Graul { 8850d18a0cbSKarsten Graul link->state = SMC_LNK_DELETING; 88615e1b99aSUrsula Braun smc_wr_wakeup_tx_wait(link); 8870d18a0cbSKarsten Graul } 8880d18a0cbSKarsten Graul 889877ae5beSKarsten Graul /* called in worker context */ 8902a4c57a9SKarsten Graul void smc_llc_link_clear(struct smc_link *link) 891877ae5beSKarsten Graul { 8922140ac26SKarsten Graul complete(&link->llc_testlink_resp); 8932140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 8942140ac26SKarsten Graul smc_wr_wakeup_reg_wait(link); 8952140ac26SKarsten Graul smc_wr_wakeup_tx_wait(link); 896877ae5beSKarsten Graul } 897877ae5beSKarsten Graul 8983d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 8993d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 90044aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 90144aa81ceSKarsten Graul { 9023d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 9033d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 9043d88a21bSKarsten Graul int rc = 0; 90544aa81ceSKarsten Graul 9063d88a21bSKarsten Graul rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); 9074600cfc3SKarsten Graul if (rc) 9084600cfc3SKarsten Graul return rc; 9093d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 9103d88a21bSKarsten Graul if (rc) 9113d88a21bSKarsten Graul goto out; 91244aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 9133d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 9143d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 9153d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 9163d88a21bSKarsten Graul rc = -EFAULT; 9173d88a21bSKarsten Graul out: 9183d88a21bSKarsten Graul if (qentry) 9193d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 9203d88a21bSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 9213d88a21bSKarsten Graul return rc; 92244aa81ceSKarsten Graul } 92344aa81ceSKarsten Graul 92460e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 9256d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 92660e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 92760e03c62SKarsten Graul { 9286d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 9296d74c3a8SKarsten Graul struct smc_link *send_link; 9300b29ec64SUrsula Braun int rc = 0; 93160e03c62SKarsten Graul 9326d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 9336d74c3a8SKarsten Graul if (!send_link) 9346d74c3a8SKarsten Graul return -ENOLINK; 9356d74c3a8SKarsten Graul 9366d74c3a8SKarsten Graul rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); 9376d74c3a8SKarsten Graul if (rc) 9386d74c3a8SKarsten Graul return rc; 9396d74c3a8SKarsten Graul /* protected by llc_flow control */ 9406d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 94160e03c62SKarsten Graul if (rc) 94260e03c62SKarsten Graul goto out; 94360e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 9446d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 9456d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 9466d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 94760e03c62SKarsten Graul rc = -EFAULT; 94860e03c62SKarsten Graul out: 9496d74c3a8SKarsten Graul if (qentry) 9506d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 9516d74c3a8SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 95260e03c62SKarsten Graul return rc; 95360e03c62SKarsten Graul } 95460e03c62SKarsten Graul 95592334cfcSKarsten Graul /* evaluate confirm link request or response */ 95692334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 95792334cfcSKarsten Graul enum smc_llc_reqresp type) 95892334cfcSKarsten Graul { 95992334cfcSKarsten Graul if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ 96092334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 96192334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 96292334cfcSKarsten Graul return -ENOTSUPP; 96392334cfcSKarsten Graul return 0; 96492334cfcSKarsten Graul } 96592334cfcSKarsten Graul 9669bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 9679bf9abeaSUrsula Braun 9689bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 9699bf9abeaSUrsula Braun { 9709bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 9719bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 9729bf9abeaSUrsula Braun }, 9739bf9abeaSUrsula Braun { 974313164daSKarsten Graul .handler = smc_llc_rx_handler, 975313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 976313164daSKarsten Graul }, 977313164daSKarsten Graul { 9784ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 97952bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 98052bedf37SKarsten Graul }, 98152bedf37SKarsten Graul { 98252bedf37SKarsten Graul .handler = smc_llc_rx_handler, 98352bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 98452bedf37SKarsten Graul }, 98552bedf37SKarsten Graul { 98652bedf37SKarsten Graul .handler = smc_llc_rx_handler, 9874ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 9884ed75de5SKarsten Graul }, 9894ed75de5SKarsten Graul { 9904ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 9914ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 9924ed75de5SKarsten Graul }, 9934ed75de5SKarsten Graul { 9944ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 9954ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 9964ed75de5SKarsten Graul }, 9974ed75de5SKarsten Graul { 9989bf9abeaSUrsula Braun .handler = NULL, 9999bf9abeaSUrsula Braun } 10009bf9abeaSUrsula Braun }; 10019bf9abeaSUrsula Braun 10029bf9abeaSUrsula Braun int __init smc_llc_init(void) 10039bf9abeaSUrsula Braun { 10049bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 10059bf9abeaSUrsula Braun int rc = 0; 10069bf9abeaSUrsula Braun 10079bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 10089bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 10099bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 10109bf9abeaSUrsula Braun if (rc) 10119bf9abeaSUrsula Braun break; 10129bf9abeaSUrsula Braun } 10139bf9abeaSUrsula Braun return rc; 10149bf9abeaSUrsula Braun } 1015