1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 209bf9abeaSUrsula Braun 210f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 220f627126SStefan Raspl 230f627126SStefan Raspl struct smc_llc_hdr { 240f627126SStefan Raspl struct smc_wr_rx_hdr common; 250f627126SStefan Raspl u8 length; /* 44 */ 2652bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 2752bedf37SKarsten Graul u8 reserved:4, 2852bedf37SKarsten Graul add_link_rej_rsn:4; 2952bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3052bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3152bedf37SKarsten Graul reserved:4; 3252bedf37SKarsten Graul #endif 330f627126SStefan Raspl u8 flags; 340f627126SStefan Raspl }; 350f627126SStefan Raspl 3675d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 3775d320d6SKarsten Graul 380f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 390f627126SStefan Raspl struct smc_llc_hdr hd; 400f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 410f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 420f627126SStefan Raspl u8 sender_qp_num[3]; 430f627126SStefan Raspl u8 link_num; 440f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 450f627126SStefan Raspl u8 max_links; 460f627126SStefan Raspl u8 reserved[9]; 470f627126SStefan Raspl }; 480f627126SStefan Raspl 4952bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5052bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 5152bedf37SKarsten Graul 5252bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 5352bedf37SKarsten Graul 5452bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 5552bedf37SKarsten Graul struct smc_llc_hdr hd; 5652bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 5752bedf37SKarsten Graul u8 reserved2[2]; 5852bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 5952bedf37SKarsten Graul u8 sender_qp_num[3]; 6052bedf37SKarsten Graul u8 link_num; 61fbed3b37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 62fbed3b37SKarsten Graul u8 reserved3 : 4, 63fbed3b37SKarsten Graul qp_mtu : 4; 64fbed3b37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 65fbed3b37SKarsten Graul u8 qp_mtu : 4, 66fbed3b37SKarsten Graul reserved3 : 4; 67fbed3b37SKarsten Graul #endif 6852bedf37SKarsten Graul u8 initial_psn[3]; 6952bedf37SKarsten Graul u8 reserved[8]; 7052bedf37SKarsten Graul }; 7152bedf37SKarsten Graul 7252bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 7352bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 7452bedf37SKarsten Graul 7552bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 7652bedf37SKarsten Graul struct smc_llc_hdr hd; 7752bedf37SKarsten Graul u8 link_num; 7852bedf37SKarsten Graul __be32 reason; 7952bedf37SKarsten Graul u8 reserved[35]; 8052bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 8152bedf37SKarsten Graul 82313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 83313164daSKarsten Graul struct smc_llc_hdr hd; 84313164daSKarsten Graul u8 user_data[16]; 85313164daSKarsten Graul u8 reserved[24]; 86313164daSKarsten Graul }; 87313164daSKarsten Graul 884ed75de5SKarsten Graul struct smc_rmb_rtoken { 894ed75de5SKarsten Graul union { 904ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 914ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 924ed75de5SKarsten Graul /* rtoken is always for the current link */ 934ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 944ed75de5SKarsten Graul }; 954ed75de5SKarsten Graul __be32 rmb_key; 964ed75de5SKarsten Graul __be64 rmb_vaddr; 974ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 984ed75de5SKarsten Graul 994ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 1004ed75de5SKarsten Graul 1014ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 1024ed75de5SKarsten Graul struct smc_llc_hdr hd; 1034ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1044ed75de5SKarsten Graul u8 reserved; 1054ed75de5SKarsten Graul }; 1064ed75de5SKarsten Graul 1074ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 1083bc67e09SKarsten Graul #define SMC_LLC_FLAG_RKEY_RETRY 0x10 1094ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1104ed75de5SKarsten Graul 1114ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1124ed75de5SKarsten Graul struct smc_llc_hdr hd; 1134ed75de5SKarsten Graul u8 num_rkeys; 1144ed75de5SKarsten Graul u8 err_mask; 1154ed75de5SKarsten Graul u8 reserved[2]; 1164ed75de5SKarsten Graul __be32 rkey[8]; 1174ed75de5SKarsten Graul u8 reserved2[4]; 1184ed75de5SKarsten Graul }; 1194ed75de5SKarsten Graul 1200f627126SStefan Raspl union smc_llc_msg { 1210f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 12252bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 12352bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1244ed75de5SKarsten Graul 1254ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1264ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1274ed75de5SKarsten Graul 128313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1290f627126SStefan Raspl struct { 1300f627126SStefan Raspl struct smc_llc_hdr hdr; 1310f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1320f627126SStefan Raspl } raw; 1330f627126SStefan Raspl }; 1340f627126SStefan Raspl 1350f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1360f627126SStefan Raspl 1376c8968c4SKarsten Graul struct smc_llc_qentry { 1386c8968c4SKarsten Graul struct list_head list; 1396c8968c4SKarsten Graul struct smc_link *link; 1406c8968c4SKarsten Graul union smc_llc_msg msg; 1416c8968c4SKarsten Graul }; 1426c8968c4SKarsten Graul 143555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 144555da9afSKarsten Graul { 145555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 146555da9afSKarsten Graul 147555da9afSKarsten Graul flow->qentry = NULL; 148555da9afSKarsten Graul return qentry; 149555da9afSKarsten Graul } 150555da9afSKarsten Graul 151555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 152555da9afSKarsten Graul { 153555da9afSKarsten Graul struct smc_llc_qentry *qentry; 154555da9afSKarsten Graul 155555da9afSKarsten Graul if (flow->qentry) { 156555da9afSKarsten Graul qentry = flow->qentry; 157555da9afSKarsten Graul flow->qentry = NULL; 158555da9afSKarsten Graul kfree(qentry); 159555da9afSKarsten Graul } 160555da9afSKarsten Graul } 161555da9afSKarsten Graul 162555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 163555da9afSKarsten Graul struct smc_llc_qentry *qentry) 164555da9afSKarsten Graul { 165555da9afSKarsten Graul flow->qentry = qentry; 166555da9afSKarsten Graul } 167555da9afSKarsten Graul 168555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 169555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 170555da9afSKarsten Graul struct smc_llc_qentry *qentry) 171555da9afSKarsten Graul { 172555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 173555da9afSKarsten Graul 174555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 175555da9afSKarsten Graul if (flow->type) { 176555da9afSKarsten Graul /* a flow is already active */ 177555da9afSKarsten Graul if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || 178555da9afSKarsten Graul qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && 179555da9afSKarsten Graul !lgr->delayed_event) { 180555da9afSKarsten Graul lgr->delayed_event = qentry; 181555da9afSKarsten Graul } else { 182555da9afSKarsten Graul /* forget this llc request */ 183555da9afSKarsten Graul kfree(qentry); 184555da9afSKarsten Graul } 185555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 186555da9afSKarsten Graul return false; 187555da9afSKarsten Graul } 188555da9afSKarsten Graul switch (qentry->msg.raw.hdr.common.type) { 189555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 190555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 191555da9afSKarsten Graul break; 192555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 193555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 194555da9afSKarsten Graul break; 195555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 196555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 197555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 198555da9afSKarsten Graul break; 199555da9afSKarsten Graul default: 200555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 201555da9afSKarsten Graul } 202555da9afSKarsten Graul if (qentry == lgr->delayed_event) 203555da9afSKarsten Graul lgr->delayed_event = NULL; 204555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 205555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 206555da9afSKarsten Graul return true; 207555da9afSKarsten Graul } 208555da9afSKarsten Graul 209555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 210555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 211555da9afSKarsten Graul enum smc_llc_flowtype type) 212555da9afSKarsten Graul { 213555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 214555da9afSKarsten Graul int rc; 215555da9afSKarsten Graul 216555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 217555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 218555da9afSKarsten Graul */ 219555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 220555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 221555da9afSKarsten Graul again: 222555da9afSKarsten Graul if (list_empty(&lgr->list)) 223555da9afSKarsten Graul return -ENODEV; 224555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 225555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 226555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 227555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 228555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 229555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 230555da9afSKarsten Graul return 0; 231555da9afSKarsten Graul } 232555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 233555da9afSKarsten Graul rc = wait_event_interruptible_timeout(lgr->llc_waiter, 234555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 235555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 236555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)), 237555da9afSKarsten Graul SMC_LLC_WAIT_TIME); 238555da9afSKarsten Graul if (!rc) 239555da9afSKarsten Graul return -ETIMEDOUT; 240555da9afSKarsten Graul goto again; 241555da9afSKarsten Graul } 242555da9afSKarsten Graul 243555da9afSKarsten Graul /* finish the current llc flow */ 244555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 245555da9afSKarsten Graul { 246555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 247555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 248555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 249555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 250555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 251555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 252555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 253555da9afSKarsten Graul else 254555da9afSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 255555da9afSKarsten Graul } 256555da9afSKarsten Graul 257555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 258555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 259555da9afSKarsten Graul */ 260555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 261555da9afSKarsten Graul struct smc_link *lnk, 262555da9afSKarsten Graul int time_out, u8 exp_msg) 263555da9afSKarsten Graul { 264555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 265555da9afSKarsten Graul 266555da9afSKarsten Graul wait_event_interruptible_timeout(lgr->llc_waiter, 267555da9afSKarsten Graul (flow->qentry || 268555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 269555da9afSKarsten Graul list_empty(&lgr->list)), 270555da9afSKarsten Graul time_out); 271555da9afSKarsten Graul if (!flow->qentry || 272555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 273555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 274555da9afSKarsten Graul goto out; 275555da9afSKarsten Graul } 276555da9afSKarsten Graul if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { 277555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 278555da9afSKarsten Graul flow->qentry->msg.raw.hdr.common.type == 279555da9afSKarsten Graul SMC_LLC_DELETE_LINK) { 280555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 281555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 282555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 283555da9afSKarsten Graul return NULL; 284555da9afSKarsten Graul } 285555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 286555da9afSKarsten Graul } 287555da9afSKarsten Graul out: 288555da9afSKarsten Graul return flow->qentry; 289555da9afSKarsten Graul } 290555da9afSKarsten Graul 2919bf9abeaSUrsula Braun /********************************** send *************************************/ 2929bf9abeaSUrsula Braun 2939bf9abeaSUrsula Braun struct smc_llc_tx_pend { 2949bf9abeaSUrsula Braun }; 2959bf9abeaSUrsula Braun 2969bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 2979bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 2989bf9abeaSUrsula Braun struct smc_link *link, 2999bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3009bf9abeaSUrsula Braun { 3019bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3029bf9abeaSUrsula Braun } 3039bf9abeaSUrsula Braun 3049bf9abeaSUrsula Braun /** 3059bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3069bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3079bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3089bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3099bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3109bf9abeaSUrsula Braun * 3119bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3129bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3139bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3149bf9abeaSUrsula Braun * 3159bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 3169bf9abeaSUrsula Braun */ 3179bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 3189bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 3199bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 3209bf9abeaSUrsula Braun { 3219bf9abeaSUrsula Braun int rc; 3229bf9abeaSUrsula Braun 323ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 324ad6f317fSUrsula Braun pend); 3259bf9abeaSUrsula Braun if (rc < 0) 3269bf9abeaSUrsula Braun return rc; 3279bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3289bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 3299bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 3309bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3319bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 3329bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 3339bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 3349bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 3359bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 3369bf9abeaSUrsula Braun return 0; 3379bf9abeaSUrsula Braun } 3389bf9abeaSUrsula Braun 3399bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 340947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 3419bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 3429bf9abeaSUrsula Braun { 34300e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 3449bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 3459bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 3469bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 3479bf9abeaSUrsula Braun int rc; 3489bf9abeaSUrsula Braun 3499bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 3509bf9abeaSUrsula Braun if (rc) 3519bf9abeaSUrsula Braun return rc; 3529bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 3539bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 3549bf9abeaSUrsula Braun confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; 3559bf9abeaSUrsula Braun confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); 35675d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 3579bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 3589bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 359947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 360947541f3SUrsula Braun ETH_ALEN); 3617005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 3629bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 3632be922f3SKarsten Graul confllc->link_num = link->link_id; 3649bf9abeaSUrsula Braun memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); 36552bedf37SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ 36652bedf37SKarsten Graul /* send llc message */ 36752bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 36852bedf37SKarsten Graul return rc; 36952bedf37SKarsten Graul } 37052bedf37SKarsten Graul 37144aa81ceSKarsten Graul /* send LLC confirm rkey request */ 3723d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 37344aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 37444aa81ceSKarsten Graul { 37544aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 37644aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 37744aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 3783d88a21bSKarsten Graul struct smc_link *link; 3793d88a21bSKarsten Graul int i, rc, rtok_ix; 38044aa81ceSKarsten Graul 3813d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 38244aa81ceSKarsten Graul if (rc) 38344aa81ceSKarsten Graul return rc; 38444aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 38544aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 38644aa81ceSKarsten Graul rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; 38744aa81ceSKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); 3883d88a21bSKarsten Graul 3893d88a21bSKarsten Graul rtok_ix = 1; 3903d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 3913d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 3923d88a21bSKarsten Graul if (link->state == SMC_LNK_ACTIVE && link != send_link) { 3933d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 3943d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 395387707fdSKarsten Graul htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 3963d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( 3973d88a21bSKarsten Graul (u64)sg_dma_address( 3983d88a21bSKarsten Graul rmb_desc->sgt[link->link_idx].sgl)); 3993d88a21bSKarsten Graul rtok_ix++; 4003d88a21bSKarsten Graul } 4013d88a21bSKarsten Graul } 4023d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 4033d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 4043d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 4053d88a21bSKarsten Graul htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); 40644aa81ceSKarsten Graul rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( 4073d88a21bSKarsten Graul (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); 40844aa81ceSKarsten Graul /* send llc message */ 4093d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 41044aa81ceSKarsten Graul return rc; 41144aa81ceSKarsten Graul } 41244aa81ceSKarsten Graul 41360e03c62SKarsten Graul /* send LLC delete rkey request */ 41460e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 41560e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 41660e03c62SKarsten Graul { 41760e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 41860e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 41960e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 42060e03c62SKarsten Graul int rc; 42160e03c62SKarsten Graul 42260e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 42360e03c62SKarsten Graul if (rc) 42460e03c62SKarsten Graul return rc; 42560e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 42660e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 42760e03c62SKarsten Graul rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; 42860e03c62SKarsten Graul rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); 42960e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 430387707fdSKarsten Graul rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); 43160e03c62SKarsten Graul /* send llc message */ 43260e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 43360e03c62SKarsten Graul return rc; 43460e03c62SKarsten Graul } 43560e03c62SKarsten Graul 43652bedf37SKarsten Graul /* send ADD LINK request or response */ 4377005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 438fbed3b37SKarsten Graul struct smc_link *link_new, 43952bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 44052bedf37SKarsten Graul { 44152bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 44252bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 44352bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 44452bedf37SKarsten Graul int rc; 44552bedf37SKarsten Graul 44652bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 44752bedf37SKarsten Graul if (rc) 44852bedf37SKarsten Graul return rc; 44952bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 450fbed3b37SKarsten Graul 451fbed3b37SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 452fbed3b37SKarsten Graul addllc->hd.common.type = SMC_LLC_ADD_LINK; 453fbed3b37SKarsten Graul addllc->hd.length = sizeof(struct smc_llc_msg_add_link); 454fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 455fbed3b37SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 456fbed3b37SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 457fbed3b37SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 458fbed3b37SKarsten Graul if (link_new) { 459fbed3b37SKarsten Graul addllc->link_num = link_new->link_id; 460fbed3b37SKarsten Graul hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 461fbed3b37SKarsten Graul hton24(addllc->initial_psn, link_new->psn_initial); 462fbed3b37SKarsten Graul if (reqresp == SMC_LLC_REQ) 463fbed3b37SKarsten Graul addllc->qp_mtu = link_new->path_mtu; 464fbed3b37SKarsten Graul else 465fbed3b37SKarsten Graul addllc->qp_mtu = min(link_new->path_mtu, 466fbed3b37SKarsten Graul link_new->peer_mtu); 467fbed3b37SKarsten Graul } 46852bedf37SKarsten Graul /* send llc message */ 46952bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 47052bedf37SKarsten Graul return rc; 47152bedf37SKarsten Graul } 47252bedf37SKarsten Graul 47352bedf37SKarsten Graul /* send DELETE LINK request or response */ 474fbed3b37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 475fbed3b37SKarsten Graul enum smc_llc_reqresp reqresp, bool orderly, 476fbed3b37SKarsten Graul u32 reason) 47752bedf37SKarsten Graul { 47852bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 47952bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 48052bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 48152bedf37SKarsten Graul int rc; 48252bedf37SKarsten Graul 48352bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 48452bedf37SKarsten Graul if (rc) 48552bedf37SKarsten Graul return rc; 48652bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 487fbed3b37SKarsten Graul 488fbed3b37SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 489fbed3b37SKarsten Graul delllc->hd.common.type = SMC_LLC_DELETE_LINK; 490fbed3b37SKarsten Graul delllc->hd.length = sizeof(struct smc_llc_msg_del_link); 491fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 492fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 493fbed3b37SKarsten Graul if (orderly) 494fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 495fbed3b37SKarsten Graul if (link_del_id) 496fbed3b37SKarsten Graul delllc->link_num = link_del_id; 497fbed3b37SKarsten Graul else 498fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 499fbed3b37SKarsten Graul delllc->reason = htonl(reason); 5009bf9abeaSUrsula Braun /* send llc message */ 5019bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 5029bf9abeaSUrsula Braun return rc; 5039bf9abeaSUrsula Braun } 5049bf9abeaSUrsula Braun 505d97935faSKarsten Graul /* send LLC test link request */ 506d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 507313164daSKarsten Graul { 508313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 509313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 510313164daSKarsten Graul struct smc_wr_buf *wr_buf; 511313164daSKarsten Graul int rc; 512313164daSKarsten Graul 513313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 514313164daSKarsten Graul if (rc) 515313164daSKarsten Graul return rc; 516313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 517313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 518313164daSKarsten Graul testllc->hd.common.type = SMC_LLC_TEST_LINK; 519313164daSKarsten Graul testllc->hd.length = sizeof(struct smc_llc_msg_test_link); 520313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 521313164daSKarsten Graul /* send llc message */ 522313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 523313164daSKarsten Graul return rc; 524313164daSKarsten Graul } 525313164daSKarsten Graul 5266c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 5276c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 5284ed75de5SKarsten Graul { 5294ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 5304ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 5314ed75de5SKarsten Graul int rc; 5324ed75de5SKarsten Graul 5336c8968c4SKarsten Graul if (!smc_link_usable(link)) 5346c8968c4SKarsten Graul return -ENOLINK; 5356c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 5364ed75de5SKarsten Graul if (rc) 5376c8968c4SKarsten Graul return rc; 5386c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 5396c8968c4SKarsten Graul return smc_wr_tx_send(link, pend); 5404ed75de5SKarsten Graul } 5414ed75de5SKarsten Graul 5429bf9abeaSUrsula Braun /********************************* receive ***********************************/ 5439bf9abeaSUrsula Braun 5448574cf40SKarsten Graul static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 5458574cf40SKarsten Graul enum smc_lgr_type lgr_new_t) 5468574cf40SKarsten Graul { 5478574cf40SKarsten Graul int i; 5488574cf40SKarsten Graul 5498574cf40SKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 5508574cf40SKarsten Graul (lgr->type != SMC_LGR_SINGLE && 5518574cf40SKarsten Graul (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 5528574cf40SKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 5538574cf40SKarsten Graul return -EMLINK; 5548574cf40SKarsten Graul 5558574cf40SKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 5568574cf40SKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 5578574cf40SKarsten Graul for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 5588574cf40SKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 5598574cf40SKarsten Graul return i; 5608574cf40SKarsten Graul } else { 5618574cf40SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 5628574cf40SKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 5638574cf40SKarsten Graul return i; 5648574cf40SKarsten Graul } 5658574cf40SKarsten Graul return -EMLINK; 5668574cf40SKarsten Graul } 5678574cf40SKarsten Graul 568*b45e7f98SKarsten Graul /* worker to process an add link message */ 569*b45e7f98SKarsten Graul static void smc_llc_add_link_work(struct work_struct *work) 570*b45e7f98SKarsten Graul { 571*b45e7f98SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 572*b45e7f98SKarsten Graul llc_add_link_work); 573*b45e7f98SKarsten Graul 574*b45e7f98SKarsten Graul if (list_empty(&lgr->list)) { 575*b45e7f98SKarsten Graul /* link group is terminating */ 576*b45e7f98SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 577*b45e7f98SKarsten Graul goto out; 578*b45e7f98SKarsten Graul } 579*b45e7f98SKarsten Graul 580*b45e7f98SKarsten Graul /* tbd: call smc_llc_process_cli_add_link(lgr); */ 581*b45e7f98SKarsten Graul /* tbd: call smc_llc_process_srv_add_link(lgr); */ 582*b45e7f98SKarsten Graul out: 583*b45e7f98SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 584*b45e7f98SKarsten Graul } 585*b45e7f98SKarsten Graul 58652bedf37SKarsten Graul static void smc_llc_rx_delete_link(struct smc_link *link, 58752bedf37SKarsten Graul struct smc_llc_msg_del_link *llc) 58852bedf37SKarsten Graul { 58900e5fb26SStefan Raspl struct smc_link_group *lgr = smc_get_lgr(link); 59052bedf37SKarsten Graul 5919651b934SKarsten Graul smc_lgr_forget(lgr); 5920d18a0cbSKarsten Graul if (lgr->role == SMC_SERV) { 5930d18a0cbSKarsten Graul /* client asks to delete this link, send request */ 594fbed3b37SKarsten Graul smc_llc_send_delete_link(link, 0, SMC_LLC_REQ, true, 595fbed3b37SKarsten Graul SMC_LLC_DEL_PROG_INIT_TERM); 59652bedf37SKarsten Graul } else { 5970d18a0cbSKarsten Graul /* server requests to delete this link, send response */ 598fbed3b37SKarsten Graul smc_llc_send_delete_link(link, 0, SMC_LLC_RESP, true, 599fbed3b37SKarsten Graul SMC_LLC_DEL_PROG_INIT_TERM); 60052bedf37SKarsten Graul } 60187523930SKarsten Graul smcr_link_down_cond(link); 60252bedf37SKarsten Graul } 60352bedf37SKarsten Graul 6043bc67e09SKarsten Graul /* process a confirm_rkey request from peer, remote flow */ 6053bc67e09SKarsten Graul static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 6064ed75de5SKarsten Graul { 6073bc67e09SKarsten Graul struct smc_llc_msg_confirm_rkey *llc; 6083bc67e09SKarsten Graul struct smc_llc_qentry *qentry; 6093bc67e09SKarsten Graul struct smc_link *link; 6103bc67e09SKarsten Graul int num_entries; 6113bc67e09SKarsten Graul int rk_idx; 6123bc67e09SKarsten Graul int i; 6134ed75de5SKarsten Graul 6143bc67e09SKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 6153bc67e09SKarsten Graul llc = &qentry->msg.confirm_rkey; 6163bc67e09SKarsten Graul link = qentry->link; 6173bc67e09SKarsten Graul 6183bc67e09SKarsten Graul num_entries = llc->rtoken[0].num_rkeys; 6193bc67e09SKarsten Graul /* first rkey entry is for receiving link */ 6203bc67e09SKarsten Graul rk_idx = smc_rtoken_add(link, 6214ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 6224ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 6233bc67e09SKarsten Graul if (rk_idx < 0) 6243bc67e09SKarsten Graul goto out_err; 6254ed75de5SKarsten Graul 6263bc67e09SKarsten Graul for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 6273bc67e09SKarsten Graul smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 6283bc67e09SKarsten Graul llc->rtoken[i].rmb_vaddr, 6293bc67e09SKarsten Graul llc->rtoken[i].rmb_key); 6303bc67e09SKarsten Graul /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 6313bc67e09SKarsten Graul goto out; 6323bc67e09SKarsten Graul out_err: 6334ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 6343bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 6353bc67e09SKarsten Graul out: 6363bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 6373bc67e09SKarsten Graul smc_llc_send_message(link, &qentry->msg); 6383bc67e09SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 6394ed75de5SKarsten Graul } 6404ed75de5SKarsten Graul 641218b24feSKarsten Graul /* process a delete_rkey request from peer, remote flow */ 642218b24feSKarsten Graul static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 6434ed75de5SKarsten Graul { 644218b24feSKarsten Graul struct smc_llc_msg_delete_rkey *llc; 645218b24feSKarsten Graul struct smc_llc_qentry *qentry; 646218b24feSKarsten Graul struct smc_link *link; 6474ed75de5SKarsten Graul u8 err_mask = 0; 6484ed75de5SKarsten Graul int i, max; 6494ed75de5SKarsten Graul 650218b24feSKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 651218b24feSKarsten Graul llc = &qentry->msg.delete_rkey; 652218b24feSKarsten Graul link = qentry->link; 653218b24feSKarsten Graul 6544ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 6554ed75de5SKarsten Graul for (i = 0; i < max; i++) { 656387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 6574ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 6584ed75de5SKarsten Graul } 6594ed75de5SKarsten Graul if (err_mask) { 6604ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 6614ed75de5SKarsten Graul llc->err_mask = err_mask; 6624ed75de5SKarsten Graul } 663218b24feSKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 664218b24feSKarsten Graul smc_llc_send_message(link, &qentry->msg); 665218b24feSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 666218b24feSKarsten Graul } 6674ed75de5SKarsten Graul 6686c8968c4SKarsten Graul /* flush the llc event queue */ 66900a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 6709bf9abeaSUrsula Braun { 6716c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 6729bf9abeaSUrsula Braun 6736c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 6746c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 6756c8968c4SKarsten Graul list_del_init(&qentry->list); 6766c8968c4SKarsten Graul kfree(qentry); 6776c8968c4SKarsten Graul } 6786c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 6796c8968c4SKarsten Graul } 6806c8968c4SKarsten Graul 6816c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 6826c8968c4SKarsten Graul { 6836c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 6846c8968c4SKarsten Graul struct smc_link *link = qentry->link; 6850fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 6866c8968c4SKarsten Graul 687d854fcbfSKarsten Graul if (!smc_link_usable(link)) 6886c8968c4SKarsten Graul goto out; 689313164daSKarsten Graul 690313164daSKarsten Graul switch (llc->raw.hdr.common.type) { 691313164daSKarsten Graul case SMC_LLC_TEST_LINK: 69256e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 69356e8091cSKarsten Graul smc_llc_send_message(link, llc); 694313164daSKarsten Graul break; 69552bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 6960fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 6970fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 6980fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 6990fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { 7000fb0b02bSKarsten Graul /* a flow is waiting for this message */ 7010fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 7020fb0b02bSKarsten Graul qentry); 7030fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 7040fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, 7050fb0b02bSKarsten Graul qentry)) { 706*b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 7070fb0b02bSKarsten Graul } 7080fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 7090fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 710*b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 7110fb0b02bSKarsten Graul } 7120fb0b02bSKarsten Graul return; 7130fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 7140fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 7150fb0b02bSKarsten Graul /* a flow is waiting for this message */ 7160fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 7170fb0b02bSKarsten Graul wake_up_interruptible(&lgr->llc_waiter); 7180fb0b02bSKarsten Graul return; 7190fb0b02bSKarsten Graul } 72052bedf37SKarsten Graul break; 72152bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 72252bedf37SKarsten Graul smc_llc_rx_delete_link(link, &llc->delete_link); 72352bedf37SKarsten Graul break; 7244ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 7253bc67e09SKarsten Graul /* new request from remote, assign to remote flow */ 7263bc67e09SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 7273bc67e09SKarsten Graul /* process here, does not wait for more llc msgs */ 7283bc67e09SKarsten Graul smc_llc_rmt_conf_rkey(lgr); 7293bc67e09SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 7303bc67e09SKarsten Graul } 7313bc67e09SKarsten Graul return; 7324ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 73342d18accSKarsten Graul /* not used because max links is 3, and 3 rkeys fit into 73442d18accSKarsten Graul * one CONFIRM_RKEY message 73542d18accSKarsten Graul */ 7364ed75de5SKarsten Graul break; 7374ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 738218b24feSKarsten Graul /* new request from remote, assign to remote flow */ 739218b24feSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 740218b24feSKarsten Graul /* process here, does not wait for more llc msgs */ 741218b24feSKarsten Graul smc_llc_rmt_delete_rkey(lgr); 742218b24feSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 743218b24feSKarsten Graul } 744218b24feSKarsten Graul return; 745313164daSKarsten Graul } 7466c8968c4SKarsten Graul out: 7476c8968c4SKarsten Graul kfree(qentry); 7486c8968c4SKarsten Graul } 7496c8968c4SKarsten Graul 7506c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 7516c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 7526c8968c4SKarsten Graul { 7536c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 7546c8968c4SKarsten Graul llc_event_work); 7556c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 7566c8968c4SKarsten Graul 757555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 758555da9afSKarsten Graul if (smc_link_usable(lgr->delayed_event->link)) { 759555da9afSKarsten Graul smc_llc_event_handler(lgr->delayed_event); 760555da9afSKarsten Graul } else { 761555da9afSKarsten Graul qentry = lgr->delayed_event; 762555da9afSKarsten Graul lgr->delayed_event = NULL; 763555da9afSKarsten Graul kfree(qentry); 764555da9afSKarsten Graul } 765555da9afSKarsten Graul } 766555da9afSKarsten Graul 7676c8968c4SKarsten Graul again: 7686c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 7696c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 7706c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 7716c8968c4SKarsten Graul struct smc_llc_qentry, list); 7726c8968c4SKarsten Graul list_del_init(&qentry->list); 7736c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7746c8968c4SKarsten Graul smc_llc_event_handler(qentry); 7756c8968c4SKarsten Graul goto again; 7766c8968c4SKarsten Graul } 7776c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 7786c8968c4SKarsten Graul } 7796c8968c4SKarsten Graul 780ef79d439SKarsten Graul /* process llc responses in tasklet context */ 781a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 782a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 783ef79d439SKarsten Graul { 784a6688d91SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.type; 785ef79d439SKarsten Graul 786a6688d91SKarsten Graul switch (llc_type) { 787ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 788ef79d439SKarsten Graul if (link->state == SMC_LNK_ACTIVE) 789ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 790ef79d439SKarsten Graul break; 791ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 7924667bb4aSKarsten Graul case SMC_LLC_CONFIRM_LINK: 7933d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 7946d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 7954667bb4aSKarsten Graul /* assign responses to the local flow, we requested them */ 7964667bb4aSKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 7974667bb4aSKarsten Graul wake_up_interruptible(&link->lgr->llc_waiter); 7984667bb4aSKarsten Graul return; 799ef79d439SKarsten Graul case SMC_LLC_DELETE_LINK: 800ef79d439SKarsten Graul if (link->lgr->role == SMC_SERV) 801ef79d439SKarsten Graul smc_lgr_schedule_free_work_fast(link->lgr); 802ef79d439SKarsten Graul break; 803ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 80442d18accSKarsten Graul /* not used because max links is 3 */ 805ef79d439SKarsten Graul break; 806ef79d439SKarsten Graul } 807a6688d91SKarsten Graul kfree(qentry); 808ef79d439SKarsten Graul } 809ef79d439SKarsten Graul 810a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 8116c8968c4SKarsten Graul { 8126c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 8136c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 8146c8968c4SKarsten Graul unsigned long flags; 8156c8968c4SKarsten Graul 8166c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 8176c8968c4SKarsten Graul if (!qentry) 8186c8968c4SKarsten Graul return; 8196c8968c4SKarsten Graul qentry->link = link; 8206c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 8216c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 822a6688d91SKarsten Graul 823a6688d91SKarsten Graul /* process responses immediately */ 824a6688d91SKarsten Graul if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { 825a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 826a6688d91SKarsten Graul return; 827a6688d91SKarsten Graul } 828a6688d91SKarsten Graul 829a6688d91SKarsten Graul /* add requests to event queue */ 8306c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 8316c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 8326c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 8336c8968c4SKarsten Graul schedule_work(&link->lgr->llc_event_work); 8349bf9abeaSUrsula Braun } 8359bf9abeaSUrsula Braun 836a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 837a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 838a6688d91SKarsten Graul { 839a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 840a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 841a6688d91SKarsten Graul 842a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 843a6688d91SKarsten Graul return; /* short message */ 844a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 845a6688d91SKarsten Graul return; /* invalid message */ 846a6688d91SKarsten Graul 847a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 848a6688d91SKarsten Graul } 849a6688d91SKarsten Graul 85044aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 851877ae5beSKarsten Graul 852877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 853877ae5beSKarsten Graul { 854877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 855877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 856877ae5beSKarsten Graul unsigned long next_interval; 857877ae5beSKarsten Graul unsigned long expire_time; 858877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 859877ae5beSKarsten Graul int rc; 860877ae5beSKarsten Graul 861877ae5beSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 862877ae5beSKarsten Graul return; /* don't reschedule worker */ 863877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 864877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 865877ae5beSKarsten Graul next_interval = expire_time - jiffies; 866877ae5beSKarsten Graul goto out; 867877ae5beSKarsten Graul } 868877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 869d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 870877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 871877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 872877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 8731020e1efSKarsten Graul if (link->state != SMC_LNK_ACTIVE) 8741020e1efSKarsten Graul return; /* link state changed */ 875877ae5beSKarsten Graul if (rc <= 0) { 87687523930SKarsten Graul smcr_link_down_cond_sched(link); 877877ae5beSKarsten Graul return; 878877ae5beSKarsten Graul } 879877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 880877ae5beSKarsten Graul out: 8811020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 882877ae5beSKarsten Graul } 883877ae5beSKarsten Graul 88400a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 88500a049cfSKarsten Graul { 88600a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 88700a049cfSKarsten Graul 88800a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 889*b45e7f98SKarsten Graul INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 89000a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 89100a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 892555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 893555da9afSKarsten Graul init_waitqueue_head(&lgr->llc_waiter); 894d5500667SKarsten Graul mutex_init(&lgr->llc_conf_mutex); 89500a049cfSKarsten Graul lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; 89600a049cfSKarsten Graul } 89700a049cfSKarsten Graul 89800a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 89900a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 90000a049cfSKarsten Graul { 90100a049cfSKarsten Graul smc_llc_event_flush(lgr); 902555da9afSKarsten Graul wake_up_interruptible_all(&lgr->llc_waiter); 90300a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 904*b45e7f98SKarsten Graul cancel_work_sync(&lgr->llc_add_link_work); 905555da9afSKarsten Graul if (lgr->delayed_event) { 906555da9afSKarsten Graul kfree(lgr->delayed_event); 907555da9afSKarsten Graul lgr->delayed_event = NULL; 908555da9afSKarsten Graul } 90900a049cfSKarsten Graul } 91000a049cfSKarsten Graul 9112a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 912877ae5beSKarsten Graul { 913877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 914877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 9152a4c57a9SKarsten Graul return 0; 916b32cf4abSKarsten Graul } 917b32cf4abSKarsten Graul 91800a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 919b32cf4abSKarsten Graul { 920877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 92100a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 92200a049cfSKarsten Graul link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; 9231020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 924877ae5beSKarsten Graul link->llc_testlink_time); 925877ae5beSKarsten Graul } 926877ae5beSKarsten Graul } 927877ae5beSKarsten Graul 928877ae5beSKarsten Graul /* called in worker context */ 9292a4c57a9SKarsten Graul void smc_llc_link_clear(struct smc_link *link) 930877ae5beSKarsten Graul { 9312140ac26SKarsten Graul complete(&link->llc_testlink_resp); 9322140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 9332140ac26SKarsten Graul smc_wr_wakeup_reg_wait(link); 9342140ac26SKarsten Graul smc_wr_wakeup_tx_wait(link); 935877ae5beSKarsten Graul } 936877ae5beSKarsten Graul 9373d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 9383d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 93944aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 94044aa81ceSKarsten Graul { 9413d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 9423d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 9433d88a21bSKarsten Graul int rc = 0; 94444aa81ceSKarsten Graul 9453d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 9463d88a21bSKarsten Graul if (rc) 9473d88a21bSKarsten Graul goto out; 94844aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 9493d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 9503d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 9513d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 9523d88a21bSKarsten Graul rc = -EFAULT; 9533d88a21bSKarsten Graul out: 9543d88a21bSKarsten Graul if (qentry) 9553d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 9563d88a21bSKarsten Graul return rc; 95744aa81ceSKarsten Graul } 95844aa81ceSKarsten Graul 95960e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 9606d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 96160e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 96260e03c62SKarsten Graul { 9636d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 9646d74c3a8SKarsten Graul struct smc_link *send_link; 9650b29ec64SUrsula Braun int rc = 0; 96660e03c62SKarsten Graul 9676d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 9686d74c3a8SKarsten Graul if (!send_link) 9696d74c3a8SKarsten Graul return -ENOLINK; 9706d74c3a8SKarsten Graul 9716d74c3a8SKarsten Graul /* protected by llc_flow control */ 9726d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 97360e03c62SKarsten Graul if (rc) 97460e03c62SKarsten Graul goto out; 97560e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 9766d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 9776d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 9786d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 97960e03c62SKarsten Graul rc = -EFAULT; 98060e03c62SKarsten Graul out: 9816d74c3a8SKarsten Graul if (qentry) 9826d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 98360e03c62SKarsten Graul return rc; 98460e03c62SKarsten Graul } 98560e03c62SKarsten Graul 98692334cfcSKarsten Graul /* evaluate confirm link request or response */ 98792334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 98892334cfcSKarsten Graul enum smc_llc_reqresp type) 98992334cfcSKarsten Graul { 99092334cfcSKarsten Graul if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ 99192334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 99292334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 99392334cfcSKarsten Graul return -ENOTSUPP; 99492334cfcSKarsten Graul return 0; 99592334cfcSKarsten Graul } 99692334cfcSKarsten Graul 9979bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 9989bf9abeaSUrsula Braun 9999bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 10009bf9abeaSUrsula Braun { 10019bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 10029bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 10039bf9abeaSUrsula Braun }, 10049bf9abeaSUrsula Braun { 1005313164daSKarsten Graul .handler = smc_llc_rx_handler, 1006313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 1007313164daSKarsten Graul }, 1008313164daSKarsten Graul { 10094ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 101052bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 101152bedf37SKarsten Graul }, 101252bedf37SKarsten Graul { 101352bedf37SKarsten Graul .handler = smc_llc_rx_handler, 101452bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 101552bedf37SKarsten Graul }, 101652bedf37SKarsten Graul { 101752bedf37SKarsten Graul .handler = smc_llc_rx_handler, 10184ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 10194ed75de5SKarsten Graul }, 10204ed75de5SKarsten Graul { 10214ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 10224ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 10234ed75de5SKarsten Graul }, 10244ed75de5SKarsten Graul { 10254ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 10264ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 10274ed75de5SKarsten Graul }, 10284ed75de5SKarsten Graul { 10299bf9abeaSUrsula Braun .handler = NULL, 10309bf9abeaSUrsula Braun } 10319bf9abeaSUrsula Braun }; 10329bf9abeaSUrsula Braun 10339bf9abeaSUrsula Braun int __init smc_llc_init(void) 10349bf9abeaSUrsula Braun { 10359bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 10369bf9abeaSUrsula Braun int rc = 0; 10379bf9abeaSUrsula Braun 10389bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 10399bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 10409bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 10419bf9abeaSUrsula Braun if (rc) 10429bf9abeaSUrsula Braun break; 10439bf9abeaSUrsula Braun } 10449bf9abeaSUrsula Braun return rc; 10459bf9abeaSUrsula Braun } 1046