1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 29bf9abeaSUrsula Braun /* 39bf9abeaSUrsula Braun * Shared Memory Communications over RDMA (SMC-R) and RoCE 49bf9abeaSUrsula Braun * 59bf9abeaSUrsula Braun * Link Layer Control (LLC) 69bf9abeaSUrsula Braun * 79bf9abeaSUrsula Braun * Copyright IBM Corp. 2016 89bf9abeaSUrsula Braun * 99bf9abeaSUrsula Braun * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> 109bf9abeaSUrsula Braun * Ursula Braun <ubraun@linux.vnet.ibm.com> 119bf9abeaSUrsula Braun */ 129bf9abeaSUrsula Braun 139bf9abeaSUrsula Braun #include <net/tcp.h> 149bf9abeaSUrsula Braun #include <rdma/ib_verbs.h> 159bf9abeaSUrsula Braun 169bf9abeaSUrsula Braun #include "smc.h" 179bf9abeaSUrsula Braun #include "smc_core.h" 189bf9abeaSUrsula Braun #include "smc_clc.h" 199bf9abeaSUrsula Braun #include "smc_llc.h" 20336ba09fSKarsten Graul #include "smc_pnet.h" 219bf9abeaSUrsula Braun 220f627126SStefan Raspl #define SMC_LLC_DATA_LEN 40 230f627126SStefan Raspl 240f627126SStefan Raspl struct smc_llc_hdr { 250f627126SStefan Raspl struct smc_wr_rx_hdr common; 26b4ba4652SKarsten Graul union { 27b4ba4652SKarsten Graul struct { 280f627126SStefan Raspl u8 length; /* 44 */ 2952bedf37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 3052bedf37SKarsten Graul u8 reserved:4, 3152bedf37SKarsten Graul add_link_rej_rsn:4; 3252bedf37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 3352bedf37SKarsten Graul u8 add_link_rej_rsn:4, 3452bedf37SKarsten Graul reserved:4; 3552bedf37SKarsten Graul #endif 360f627126SStefan Raspl }; 37b4ba4652SKarsten Graul u16 length_v2; /* 44 - 8192*/ 38b4ba4652SKarsten Graul }; 39b4ba4652SKarsten Graul u8 flags; 40b4ba4652SKarsten Graul } __packed; /* format defined in 41b4ba4652SKarsten Graul * IBM Shared Memory Communications Version 2 42b4ba4652SKarsten Graul * (https://www.ibm.com/support/pages/node/6326337) 43b4ba4652SKarsten Graul */ 440f627126SStefan Raspl 4575d320d6SKarsten Graul #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 4675d320d6SKarsten Graul 470f627126SStefan Raspl struct smc_llc_msg_confirm_link { /* type 0x01 */ 480f627126SStefan Raspl struct smc_llc_hdr hd; 490f627126SStefan Raspl u8 sender_mac[ETH_ALEN]; 500f627126SStefan Raspl u8 sender_gid[SMC_GID_SIZE]; 510f627126SStefan Raspl u8 sender_qp_num[3]; 520f627126SStefan Raspl u8 link_num; 530f627126SStefan Raspl u8 link_uid[SMC_LGR_ID_SIZE]; 540f627126SStefan Raspl u8 max_links; 550f627126SStefan Raspl u8 reserved[9]; 560f627126SStefan Raspl }; 570f627126SStefan Raspl 5852bedf37SKarsten Graul #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 5952bedf37SKarsten Graul #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 6052bedf37SKarsten Graul 6152bedf37SKarsten Graul #define SMC_LLC_ADD_LNK_MAX_LINKS 2 6252bedf37SKarsten Graul 6352bedf37SKarsten Graul struct smc_llc_msg_add_link { /* type 0x02 */ 6452bedf37SKarsten Graul struct smc_llc_hdr hd; 6552bedf37SKarsten Graul u8 sender_mac[ETH_ALEN]; 6652bedf37SKarsten Graul u8 reserved2[2]; 6752bedf37SKarsten Graul u8 sender_gid[SMC_GID_SIZE]; 6852bedf37SKarsten Graul u8 sender_qp_num[3]; 6952bedf37SKarsten Graul u8 link_num; 70fbed3b37SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 71fbed3b37SKarsten Graul u8 reserved3 : 4, 72fbed3b37SKarsten Graul qp_mtu : 4; 73fbed3b37SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 74fbed3b37SKarsten Graul u8 qp_mtu : 4, 75fbed3b37SKarsten Graul reserved3 : 4; 76fbed3b37SKarsten Graul #endif 7752bedf37SKarsten Graul u8 initial_psn[3]; 7852bedf37SKarsten Graul u8 reserved[8]; 7952bedf37SKarsten Graul }; 8052bedf37SKarsten Graul 8187f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt { 8287f88cdaSKarsten Graul __be32 rmb_key; 8387f88cdaSKarsten Graul __be32 rmb_key_new; 8487f88cdaSKarsten Graul __be64 rmb_vaddr_new; 8587f88cdaSKarsten Graul }; 8687f88cdaSKarsten Graul 87b4ba4652SKarsten Graul struct smc_llc_msg_add_link_v2_ext { 88b4ba4652SKarsten Graul #if defined(__BIG_ENDIAN_BITFIELD) 89b4ba4652SKarsten Graul u8 v2_direct : 1, 90b4ba4652SKarsten Graul reserved : 7; 91b4ba4652SKarsten Graul #elif defined(__LITTLE_ENDIAN_BITFIELD) 92b4ba4652SKarsten Graul u8 reserved : 7, 93b4ba4652SKarsten Graul v2_direct : 1; 94b4ba4652SKarsten Graul #endif 95b4ba4652SKarsten Graul u8 reserved2; 96b4ba4652SKarsten Graul u8 client_target_gid[SMC_GID_SIZE]; 97b4ba4652SKarsten Graul u8 reserved3[8]; 98b4ba4652SKarsten Graul u16 num_rkeys; 99b4ba4652SKarsten Graul struct smc_llc_msg_add_link_cont_rt rt[]; 100b4ba4652SKarsten Graul } __packed; /* format defined in 101b4ba4652SKarsten Graul * IBM Shared Memory Communications Version 2 102b4ba4652SKarsten Graul * (https://www.ibm.com/support/pages/node/6326337) 103b4ba4652SKarsten Graul */ 104b4ba4652SKarsten Graul 105b4ba4652SKarsten Graul struct smc_llc_msg_req_add_link_v2 { 106b4ba4652SKarsten Graul struct smc_llc_hdr hd; 107b4ba4652SKarsten Graul u8 reserved[20]; 108b4ba4652SKarsten Graul u8 gid_cnt; 109b4ba4652SKarsten Graul u8 reserved2[3]; 110b4ba4652SKarsten Graul u8 gid[][SMC_GID_SIZE]; 111b4ba4652SKarsten Graul }; 112b4ba4652SKarsten Graul 11387f88cdaSKarsten Graul #define SMC_LLC_RKEYS_PER_CONT_MSG 2 11487f88cdaSKarsten Graul 11587f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont { /* type 0x03 */ 11687f88cdaSKarsten Graul struct smc_llc_hdr hd; 11787f88cdaSKarsten Graul u8 link_num; 11887f88cdaSKarsten Graul u8 num_rkeys; 11987f88cdaSKarsten Graul u8 reserved2[2]; 12087f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; 12187f88cdaSKarsten Graul u8 reserved[4]; 12287f88cdaSKarsten Graul } __packed; /* format defined in RFC7609 */ 12387f88cdaSKarsten Graul 12452bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 12552bedf37SKarsten Graul #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 12652bedf37SKarsten Graul 12752bedf37SKarsten Graul struct smc_llc_msg_del_link { /* type 0x04 */ 12852bedf37SKarsten Graul struct smc_llc_hdr hd; 12952bedf37SKarsten Graul u8 link_num; 13052bedf37SKarsten Graul __be32 reason; 13152bedf37SKarsten Graul u8 reserved[35]; 13252bedf37SKarsten Graul } __packed; /* format defined in RFC7609 */ 13352bedf37SKarsten Graul 134313164daSKarsten Graul struct smc_llc_msg_test_link { /* type 0x07 */ 135313164daSKarsten Graul struct smc_llc_hdr hd; 136313164daSKarsten Graul u8 user_data[16]; 137313164daSKarsten Graul u8 reserved[24]; 138313164daSKarsten Graul }; 139313164daSKarsten Graul 1404ed75de5SKarsten Graul struct smc_rmb_rtoken { 1414ed75de5SKarsten Graul union { 1424ed75de5SKarsten Graul u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ 1434ed75de5SKarsten Graul /* is actually the num of rtokens, first */ 1444ed75de5SKarsten Graul /* rtoken is always for the current link */ 1454ed75de5SKarsten Graul u8 link_id; /* link id of the rtoken */ 1464ed75de5SKarsten Graul }; 1474ed75de5SKarsten Graul __be32 rmb_key; 1484ed75de5SKarsten Graul __be64 rmb_vaddr; 1494ed75de5SKarsten Graul } __packed; /* format defined in RFC7609 */ 1504ed75de5SKarsten Graul 1514ed75de5SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG 3 152b4ba4652SKarsten Graul #define SMC_LLC_RKEYS_PER_MSG_V2 255 1534ed75de5SKarsten Graul 1544ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey { /* type 0x06 */ 1554ed75de5SKarsten Graul struct smc_llc_hdr hd; 1564ed75de5SKarsten Graul struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; 1574ed75de5SKarsten Graul u8 reserved; 1584ed75de5SKarsten Graul }; 1594ed75de5SKarsten Graul 1604ed75de5SKarsten Graul #define SMC_LLC_DEL_RKEY_MAX 8 1613bc67e09SKarsten Graul #define SMC_LLC_FLAG_RKEY_RETRY 0x10 1624ed75de5SKarsten Graul #define SMC_LLC_FLAG_RKEY_NEG 0x20 1634ed75de5SKarsten Graul 1644ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey { /* type 0x09 */ 1654ed75de5SKarsten Graul struct smc_llc_hdr hd; 1664ed75de5SKarsten Graul u8 num_rkeys; 1674ed75de5SKarsten Graul u8 err_mask; 1684ed75de5SKarsten Graul u8 reserved[2]; 1694ed75de5SKarsten Graul __be32 rkey[8]; 1704ed75de5SKarsten Graul u8 reserved2[4]; 1714ed75de5SKarsten Graul }; 1724ed75de5SKarsten Graul 173b4ba4652SKarsten Graul struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ 174b4ba4652SKarsten Graul struct smc_llc_hdr hd; 175b4ba4652SKarsten Graul u8 num_rkeys; 176b4ba4652SKarsten Graul u8 num_inval_rkeys; 177b4ba4652SKarsten Graul u8 reserved[2]; 178b4ba4652SKarsten Graul __be32 rkey[]; 179b4ba4652SKarsten Graul }; 180b4ba4652SKarsten Graul 1810f627126SStefan Raspl union smc_llc_msg { 1820f627126SStefan Raspl struct smc_llc_msg_confirm_link confirm_link; 18352bedf37SKarsten Graul struct smc_llc_msg_add_link add_link; 184b4ba4652SKarsten Graul struct smc_llc_msg_req_add_link_v2 req_add_link; 18587f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont add_link_cont; 18652bedf37SKarsten Graul struct smc_llc_msg_del_link delete_link; 1874ed75de5SKarsten Graul 1884ed75de5SKarsten Graul struct smc_llc_msg_confirm_rkey confirm_rkey; 1894ed75de5SKarsten Graul struct smc_llc_msg_delete_rkey delete_rkey; 1904ed75de5SKarsten Graul 191313164daSKarsten Graul struct smc_llc_msg_test_link test_link; 1920f627126SStefan Raspl struct { 1930f627126SStefan Raspl struct smc_llc_hdr hdr; 1940f627126SStefan Raspl u8 data[SMC_LLC_DATA_LEN]; 1950f627126SStefan Raspl } raw; 1960f627126SStefan Raspl }; 1970f627126SStefan Raspl 1980f627126SStefan Raspl #define SMC_LLC_FLAG_RESP 0x80 1990f627126SStefan Raspl 2006c8968c4SKarsten Graul struct smc_llc_qentry { 2016c8968c4SKarsten Graul struct list_head list; 2026c8968c4SKarsten Graul struct smc_link *link; 2036c8968c4SKarsten Graul union smc_llc_msg msg; 2046c8968c4SKarsten Graul }; 2056c8968c4SKarsten Graul 2064dadd151SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); 2074dadd151SKarsten Graul 208555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) 209555da9afSKarsten Graul { 210555da9afSKarsten Graul struct smc_llc_qentry *qentry = flow->qentry; 211555da9afSKarsten Graul 212555da9afSKarsten Graul flow->qentry = NULL; 213555da9afSKarsten Graul return qentry; 214555da9afSKarsten Graul } 215555da9afSKarsten Graul 216555da9afSKarsten Graul void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) 217555da9afSKarsten Graul { 218555da9afSKarsten Graul struct smc_llc_qentry *qentry; 219555da9afSKarsten Graul 220555da9afSKarsten Graul if (flow->qentry) { 221555da9afSKarsten Graul qentry = flow->qentry; 222555da9afSKarsten Graul flow->qentry = NULL; 223555da9afSKarsten Graul kfree(qentry); 224555da9afSKarsten Graul } 225555da9afSKarsten Graul } 226555da9afSKarsten Graul 227555da9afSKarsten Graul static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, 228555da9afSKarsten Graul struct smc_llc_qentry *qentry) 229555da9afSKarsten Graul { 230555da9afSKarsten Graul flow->qentry = qentry; 231555da9afSKarsten Graul } 232555da9afSKarsten Graul 2336778a6beSKarsten Graul static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, 2346778a6beSKarsten Graul struct smc_llc_qentry *qentry) 2356778a6beSKarsten Graul { 236b4ba4652SKarsten Graul u8 msg_type = qentry->msg.raw.hdr.common.llc_type; 2376778a6beSKarsten Graul 2386778a6beSKarsten Graul if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && 2396778a6beSKarsten Graul flow_type != msg_type && !lgr->delayed_event) { 2406778a6beSKarsten Graul lgr->delayed_event = qentry; 2416778a6beSKarsten Graul return; 2426778a6beSKarsten Graul } 2436778a6beSKarsten Graul /* drop parallel or already-in-progress llc requests */ 2446778a6beSKarsten Graul if (flow_type != msg_type) 245de2fea7bSTony Lu pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel " 2466778a6beSKarsten Graul "LLC msg: msg %d flow %d role %d\n", 2476778a6beSKarsten Graul SMC_LGR_ID_SIZE, &lgr->id, 248de2fea7bSTony Lu lgr->net->net_cookie, 2496778a6beSKarsten Graul qentry->msg.raw.hdr.common.type, 2506778a6beSKarsten Graul flow_type, lgr->role); 2516778a6beSKarsten Graul kfree(qentry); 2526778a6beSKarsten Graul } 2536778a6beSKarsten Graul 254555da9afSKarsten Graul /* try to start a new llc flow, initiated by an incoming llc msg */ 255555da9afSKarsten Graul static bool smc_llc_flow_start(struct smc_llc_flow *flow, 256555da9afSKarsten Graul struct smc_llc_qentry *qentry) 257555da9afSKarsten Graul { 258555da9afSKarsten Graul struct smc_link_group *lgr = qentry->link->lgr; 259555da9afSKarsten Graul 260555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 261555da9afSKarsten Graul if (flow->type) { 262555da9afSKarsten Graul /* a flow is already active */ 2636778a6beSKarsten Graul smc_llc_flow_parallel(lgr, flow->type, qentry); 264555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 265555da9afSKarsten Graul return false; 266555da9afSKarsten Graul } 267b4ba4652SKarsten Graul switch (qentry->msg.raw.hdr.common.llc_type) { 268555da9afSKarsten Graul case SMC_LLC_ADD_LINK: 269555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_ADD_LINK; 270555da9afSKarsten Graul break; 271555da9afSKarsten Graul case SMC_LLC_DELETE_LINK: 272555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_DEL_LINK; 273555da9afSKarsten Graul break; 274555da9afSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 275555da9afSKarsten Graul case SMC_LLC_DELETE_RKEY: 276555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_RKEY; 277555da9afSKarsten Graul break; 278555da9afSKarsten Graul default: 279555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 280555da9afSKarsten Graul } 281555da9afSKarsten Graul smc_llc_flow_qentry_set(flow, qentry); 2826778a6beSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 283555da9afSKarsten Graul return true; 284555da9afSKarsten Graul } 285555da9afSKarsten Graul 286555da9afSKarsten Graul /* start a new local llc flow, wait till current flow finished */ 287555da9afSKarsten Graul int smc_llc_flow_initiate(struct smc_link_group *lgr, 288555da9afSKarsten Graul enum smc_llc_flowtype type) 289555da9afSKarsten Graul { 290555da9afSKarsten Graul enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; 291555da9afSKarsten Graul int rc; 292555da9afSKarsten Graul 293555da9afSKarsten Graul /* all flows except confirm_rkey and delete_rkey are exclusive, 294555da9afSKarsten Graul * confirm/delete rkey flows can run concurrently (local and remote) 295555da9afSKarsten Graul */ 296555da9afSKarsten Graul if (type == SMC_LLC_FLOW_RKEY) 297555da9afSKarsten Graul allowed_remote = SMC_LLC_FLOW_RKEY; 298555da9afSKarsten Graul again: 299555da9afSKarsten Graul if (list_empty(&lgr->list)) 300555da9afSKarsten Graul return -ENODEV; 301555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 302555da9afSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 303555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 304555da9afSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote)) { 305555da9afSKarsten Graul lgr->llc_flow_lcl.type = type; 306555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 307555da9afSKarsten Graul return 0; 308555da9afSKarsten Graul } 309555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 3106778a6beSKarsten Graul rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || 311555da9afSKarsten Graul (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && 312555da9afSKarsten Graul (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || 3136778a6beSKarsten Graul lgr->llc_flow_rmt.type == allowed_remote))), 3146778a6beSKarsten Graul SMC_LLC_WAIT_TIME * 10); 315555da9afSKarsten Graul if (!rc) 316555da9afSKarsten Graul return -ETIMEDOUT; 317555da9afSKarsten Graul goto again; 318555da9afSKarsten Graul } 319555da9afSKarsten Graul 320555da9afSKarsten Graul /* finish the current llc flow */ 321555da9afSKarsten Graul void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) 322555da9afSKarsten Graul { 323555da9afSKarsten Graul spin_lock_bh(&lgr->llc_flow_lock); 324555da9afSKarsten Graul memset(flow, 0, sizeof(*flow)); 325555da9afSKarsten Graul flow->type = SMC_LLC_FLOW_NONE; 326555da9afSKarsten Graul spin_unlock_bh(&lgr->llc_flow_lock); 327555da9afSKarsten Graul if (!list_empty(&lgr->list) && lgr->delayed_event && 328555da9afSKarsten Graul flow == &lgr->llc_flow_lcl) 329555da9afSKarsten Graul schedule_work(&lgr->llc_event_work); 330555da9afSKarsten Graul else 3316778a6beSKarsten Graul wake_up(&lgr->llc_flow_waiter); 332555da9afSKarsten Graul } 333555da9afSKarsten Graul 334555da9afSKarsten Graul /* lnk is optional and used for early wakeup when link goes down, useful in 335555da9afSKarsten Graul * cases where we wait for a response on the link after we sent a request 336555da9afSKarsten Graul */ 337555da9afSKarsten Graul struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, 338555da9afSKarsten Graul struct smc_link *lnk, 339555da9afSKarsten Graul int time_out, u8 exp_msg) 340555da9afSKarsten Graul { 341555da9afSKarsten Graul struct smc_llc_flow *flow = &lgr->llc_flow_lcl; 3426778a6beSKarsten Graul u8 rcv_msg; 343555da9afSKarsten Graul 3446778a6beSKarsten Graul wait_event_timeout(lgr->llc_msg_waiter, 345555da9afSKarsten Graul (flow->qentry || 346555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || 347555da9afSKarsten Graul list_empty(&lgr->list)), 348555da9afSKarsten Graul time_out); 349555da9afSKarsten Graul if (!flow->qentry || 350555da9afSKarsten Graul (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { 351555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 352555da9afSKarsten Graul goto out; 353555da9afSKarsten Graul } 354b4ba4652SKarsten Graul rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type; 3556778a6beSKarsten Graul if (exp_msg && rcv_msg != exp_msg) { 356555da9afSKarsten Graul if (exp_msg == SMC_LLC_ADD_LINK && 3576778a6beSKarsten Graul rcv_msg == SMC_LLC_DELETE_LINK) { 358555da9afSKarsten Graul /* flow_start will delay the unexpected msg */ 359555da9afSKarsten Graul smc_llc_flow_start(&lgr->llc_flow_lcl, 360555da9afSKarsten Graul smc_llc_flow_qentry_clr(flow)); 361555da9afSKarsten Graul return NULL; 362555da9afSKarsten Graul } 363de2fea7bSTony Lu pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: " 3646778a6beSKarsten Graul "msg %d exp %d flow %d role %d flags %x\n", 365de2fea7bSTony Lu SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie, 366de2fea7bSTony Lu rcv_msg, exp_msg, 3676778a6beSKarsten Graul flow->type, lgr->role, 3686778a6beSKarsten Graul flow->qentry->msg.raw.hdr.flags); 369555da9afSKarsten Graul smc_llc_flow_qentry_del(flow); 370555da9afSKarsten Graul } 371555da9afSKarsten Graul out: 372555da9afSKarsten Graul return flow->qentry; 373555da9afSKarsten Graul } 374555da9afSKarsten Graul 3759bf9abeaSUrsula Braun /********************************** send *************************************/ 3769bf9abeaSUrsula Braun 3779bf9abeaSUrsula Braun struct smc_llc_tx_pend { 3789bf9abeaSUrsula Braun }; 3799bf9abeaSUrsula Braun 3809bf9abeaSUrsula Braun /* handler for send/transmission completion of an LLC msg */ 3819bf9abeaSUrsula Braun static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, 3829bf9abeaSUrsula Braun struct smc_link *link, 3839bf9abeaSUrsula Braun enum ib_wc_status wc_status) 3849bf9abeaSUrsula Braun { 3859bf9abeaSUrsula Braun /* future work: handle wc_status error for recovery and failover */ 3869bf9abeaSUrsula Braun } 3879bf9abeaSUrsula Braun 3889bf9abeaSUrsula Braun /** 3899bf9abeaSUrsula Braun * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits 3909bf9abeaSUrsula Braun * @link: Pointer to SMC link used for sending LLC control message. 3919bf9abeaSUrsula Braun * @wr_buf: Out variable returning pointer to work request payload buffer. 3929bf9abeaSUrsula Braun * @pend: Out variable returning pointer to private pending WR tracking. 3939bf9abeaSUrsula Braun * It's the context the transmit complete handler will get. 3949bf9abeaSUrsula Braun * 3959bf9abeaSUrsula Braun * Reserves and pre-fills an entry for a pending work request send/tx. 3969bf9abeaSUrsula Braun * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. 3979bf9abeaSUrsula Braun * Can sleep due to smc_get_ctrl_buf (if not in softirq context). 3989bf9abeaSUrsula Braun * 3999bf9abeaSUrsula Braun * Return: 0 on success, otherwise an error value. 4009bf9abeaSUrsula Braun */ 4019bf9abeaSUrsula Braun static int smc_llc_add_pending_send(struct smc_link *link, 4029bf9abeaSUrsula Braun struct smc_wr_buf **wr_buf, 4039bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv **pend) 4049bf9abeaSUrsula Braun { 4059bf9abeaSUrsula Braun int rc; 4069bf9abeaSUrsula Braun 407ad6f317fSUrsula Braun rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, 408ad6f317fSUrsula Braun pend); 4099bf9abeaSUrsula Braun if (rc < 0) 4109bf9abeaSUrsula Braun return rc; 4119bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 4129bf9abeaSUrsula Braun sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, 4139bf9abeaSUrsula Braun "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); 4149bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 4159bf9abeaSUrsula Braun sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, 4169bf9abeaSUrsula Braun "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); 4179bf9abeaSUrsula Braun BUILD_BUG_ON_MSG( 4189bf9abeaSUrsula Braun sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, 4199bf9abeaSUrsula Braun "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); 4209bf9abeaSUrsula Braun return 0; 4219bf9abeaSUrsula Braun } 4229bf9abeaSUrsula Braun 423b4ba4652SKarsten Graul static int smc_llc_add_pending_send_v2(struct smc_link *link, 424b4ba4652SKarsten Graul struct smc_wr_v2_buf **wr_buf, 425b4ba4652SKarsten Graul struct smc_wr_tx_pend_priv **pend) 426b4ba4652SKarsten Graul { 427b4ba4652SKarsten Graul int rc; 428b4ba4652SKarsten Graul 429b4ba4652SKarsten Graul rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend); 430b4ba4652SKarsten Graul if (rc < 0) 431b4ba4652SKarsten Graul return rc; 432b4ba4652SKarsten Graul return 0; 433b4ba4652SKarsten Graul } 434b4ba4652SKarsten Graul 435b4ba4652SKarsten Graul static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr, 436b4ba4652SKarsten Graul struct smc_link_group *lgr, size_t len) 437b4ba4652SKarsten Graul { 438b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 439b4ba4652SKarsten Graul hdr->common.llc_version = SMC_V2; 440b4ba4652SKarsten Graul hdr->length_v2 = len; 441b4ba4652SKarsten Graul } else { 442b4ba4652SKarsten Graul hdr->common.llc_version = 0; 443b4ba4652SKarsten Graul hdr->length = len; 444b4ba4652SKarsten Graul } 445b4ba4652SKarsten Graul } 446b4ba4652SKarsten Graul 4479bf9abeaSUrsula Braun /* high-level API to send LLC confirm link */ 448947541f3SUrsula Braun int smc_llc_send_confirm_link(struct smc_link *link, 4499bf9abeaSUrsula Braun enum smc_llc_reqresp reqresp) 4509bf9abeaSUrsula Braun { 4519bf9abeaSUrsula Braun struct smc_llc_msg_confirm_link *confllc; 4529bf9abeaSUrsula Braun struct smc_wr_tx_pend_priv *pend; 4539bf9abeaSUrsula Braun struct smc_wr_buf *wr_buf; 4549bf9abeaSUrsula Braun int rc; 4559bf9abeaSUrsula Braun 45695f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 45795f7f3e7SKarsten Graul return -ENOLINK; 4589bf9abeaSUrsula Braun rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 4599bf9abeaSUrsula Braun if (rc) 46095f7f3e7SKarsten Graul goto put_out; 4619bf9abeaSUrsula Braun confllc = (struct smc_llc_msg_confirm_link *)wr_buf; 4629bf9abeaSUrsula Braun memset(confllc, 0, sizeof(*confllc)); 463b4ba4652SKarsten Graul confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK; 464b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc)); 46575d320d6SKarsten Graul confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; 4669bf9abeaSUrsula Braun if (reqresp == SMC_LLC_RESP) 4679bf9abeaSUrsula Braun confllc->hd.flags |= SMC_LLC_FLAG_RESP; 468947541f3SUrsula Braun memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], 469947541f3SUrsula Braun ETH_ALEN); 4707005ada6SUrsula Braun memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); 4719bf9abeaSUrsula Braun hton24(confllc->sender_qp_num, link->roce_qp->qp_num); 4722be922f3SKarsten Graul confllc->link_num = link->link_id; 47345fa8da0SKarsten Graul memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); 474b1570a87SKarsten Graul confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; 47552bedf37SKarsten Graul /* send llc message */ 47652bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 47795f7f3e7SKarsten Graul put_out: 47895f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 47952bedf37SKarsten Graul return rc; 48052bedf37SKarsten Graul } 48152bedf37SKarsten Graul 48244aa81ceSKarsten Graul /* send LLC confirm rkey request */ 4833d88a21bSKarsten Graul static int smc_llc_send_confirm_rkey(struct smc_link *send_link, 48444aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 48544aa81ceSKarsten Graul { 48644aa81ceSKarsten Graul struct smc_llc_msg_confirm_rkey *rkeyllc; 48744aa81ceSKarsten Graul struct smc_wr_tx_pend_priv *pend; 48844aa81ceSKarsten Graul struct smc_wr_buf *wr_buf; 4893d88a21bSKarsten Graul struct smc_link *link; 4903d88a21bSKarsten Graul int i, rc, rtok_ix; 49144aa81ceSKarsten Graul 49295f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(send_link)) 49395f7f3e7SKarsten Graul return -ENOLINK; 4943d88a21bSKarsten Graul rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); 49544aa81ceSKarsten Graul if (rc) 49695f7f3e7SKarsten Graul goto put_out; 49744aa81ceSKarsten Graul rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; 49844aa81ceSKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 499b4ba4652SKarsten Graul rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY; 500b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc)); 5013d88a21bSKarsten Graul 5023d88a21bSKarsten Graul rtok_ix = 1; 5033d88a21bSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 5043d88a21bSKarsten Graul link = &send_link->lgr->lnk[i]; 505741a49a4SKarsten Graul if (smc_link_active(link) && link != send_link) { 5063d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].link_id = link->link_id; 5073d88a21bSKarsten Graul rkeyllc->rtoken[rtok_ix].rmb_key = 508b8d19945SWen Gu htonl(rmb_desc->mr[link->link_idx]->rkey); 509b8d19945SWen Gu rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ? 510b8d19945SWen Gu cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : 511b8d19945SWen Gu cpu_to_be64((u64)sg_dma_address 512b8d19945SWen Gu (rmb_desc->sgt[link->link_idx].sgl)); 5133d88a21bSKarsten Graul rtok_ix++; 5143d88a21bSKarsten Graul } 5153d88a21bSKarsten Graul } 5163d88a21bSKarsten Graul /* rkey of send_link is in rtoken[0] */ 5173d88a21bSKarsten Graul rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; 5183d88a21bSKarsten Graul rkeyllc->rtoken[0].rmb_key = 519b8d19945SWen Gu htonl(rmb_desc->mr[send_link->link_idx]->rkey); 520b8d19945SWen Gu rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ? 521b8d19945SWen Gu cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) : 522b8d19945SWen Gu cpu_to_be64((u64)sg_dma_address 523b8d19945SWen Gu (rmb_desc->sgt[send_link->link_idx].sgl)); 52444aa81ceSKarsten Graul /* send llc message */ 5253d88a21bSKarsten Graul rc = smc_wr_tx_send(send_link, pend); 52695f7f3e7SKarsten Graul put_out: 52795f7f3e7SKarsten Graul smc_wr_tx_link_put(send_link); 52844aa81ceSKarsten Graul return rc; 52944aa81ceSKarsten Graul } 53044aa81ceSKarsten Graul 53160e03c62SKarsten Graul /* send LLC delete rkey request */ 53260e03c62SKarsten Graul static int smc_llc_send_delete_rkey(struct smc_link *link, 53360e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 53460e03c62SKarsten Graul { 53560e03c62SKarsten Graul struct smc_llc_msg_delete_rkey *rkeyllc; 53660e03c62SKarsten Graul struct smc_wr_tx_pend_priv *pend; 53760e03c62SKarsten Graul struct smc_wr_buf *wr_buf; 53860e03c62SKarsten Graul int rc; 53960e03c62SKarsten Graul 54095f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 54195f7f3e7SKarsten Graul return -ENOLINK; 54260e03c62SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 54360e03c62SKarsten Graul if (rc) 54495f7f3e7SKarsten Graul goto put_out; 54560e03c62SKarsten Graul rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; 54660e03c62SKarsten Graul memset(rkeyllc, 0, sizeof(*rkeyllc)); 547b4ba4652SKarsten Graul rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY; 548b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc)); 54960e03c62SKarsten Graul rkeyllc->num_rkeys = 1; 550b8d19945SWen Gu rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey); 55160e03c62SKarsten Graul /* send llc message */ 55260e03c62SKarsten Graul rc = smc_wr_tx_send(link, pend); 55395f7f3e7SKarsten Graul put_out: 55495f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 55560e03c62SKarsten Graul return rc; 55660e03c62SKarsten Graul } 55760e03c62SKarsten Graul 558b4ba4652SKarsten Graul /* return first buffer from any of the next buf lists */ 559b4ba4652SKarsten Graul static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, 560b4ba4652SKarsten Graul int *buf_lst) 561b4ba4652SKarsten Graul { 562b4ba4652SKarsten Graul struct smc_buf_desc *buf_pos; 563b4ba4652SKarsten Graul 564b4ba4652SKarsten Graul while (*buf_lst < SMC_RMBE_SIZES) { 565b4ba4652SKarsten Graul buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], 566b4ba4652SKarsten Graul struct smc_buf_desc, list); 567b4ba4652SKarsten Graul if (buf_pos) 568b4ba4652SKarsten Graul return buf_pos; 569b4ba4652SKarsten Graul (*buf_lst)++; 570b4ba4652SKarsten Graul } 571b4ba4652SKarsten Graul return NULL; 572b4ba4652SKarsten Graul } 573b4ba4652SKarsten Graul 574b4ba4652SKarsten Graul /* return next rmb from buffer lists */ 575b4ba4652SKarsten Graul static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, 576b4ba4652SKarsten Graul int *buf_lst, 577b4ba4652SKarsten Graul struct smc_buf_desc *buf_pos) 578b4ba4652SKarsten Graul { 579b4ba4652SKarsten Graul struct smc_buf_desc *buf_next; 580b4ba4652SKarsten Graul 581*b24aa141SWen Gu if (!buf_pos) 582*b24aa141SWen Gu return _smc_llc_get_next_rmb(lgr, buf_lst); 583*b24aa141SWen Gu 584*b24aa141SWen Gu if (list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { 585b4ba4652SKarsten Graul (*buf_lst)++; 586b4ba4652SKarsten Graul return _smc_llc_get_next_rmb(lgr, buf_lst); 587b4ba4652SKarsten Graul } 588b4ba4652SKarsten Graul buf_next = list_next_entry(buf_pos, list); 589b4ba4652SKarsten Graul return buf_next; 590b4ba4652SKarsten Graul } 591b4ba4652SKarsten Graul 592b4ba4652SKarsten Graul static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, 593b4ba4652SKarsten Graul int *buf_lst) 594b4ba4652SKarsten Graul { 595b4ba4652SKarsten Graul *buf_lst = 0; 596b4ba4652SKarsten Graul return smc_llc_get_next_rmb(lgr, buf_lst, NULL); 597b4ba4652SKarsten Graul } 598b4ba4652SKarsten Graul 599b4ba4652SKarsten Graul static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext, 600b4ba4652SKarsten Graul struct smc_link *link, struct smc_link *link_new) 601b4ba4652SKarsten Graul { 602b4ba4652SKarsten Graul struct smc_link_group *lgr = link->lgr; 603b4ba4652SKarsten Graul struct smc_buf_desc *buf_pos; 604b4ba4652SKarsten Graul int prim_lnk_idx, lnk_idx, i; 605b4ba4652SKarsten Graul struct smc_buf_desc *rmb; 606b4ba4652SKarsten Graul int len = sizeof(*ext); 607b4ba4652SKarsten Graul int buf_lst; 608b4ba4652SKarsten Graul 609b4ba4652SKarsten Graul ext->v2_direct = !lgr->uses_gateway; 610b4ba4652SKarsten Graul memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE); 611b4ba4652SKarsten Graul 612b4ba4652SKarsten Graul prim_lnk_idx = link->link_idx; 613b4ba4652SKarsten Graul lnk_idx = link_new->link_idx; 614aff7bfedSD. Wythe down_write(&lgr->rmbs_lock); 615b4ba4652SKarsten Graul ext->num_rkeys = lgr->conns_num; 616b4ba4652SKarsten Graul if (!ext->num_rkeys) 617b4ba4652SKarsten Graul goto out; 618b4ba4652SKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 619b4ba4652SKarsten Graul for (i = 0; i < ext->num_rkeys; i++) { 620b4ba4652SKarsten Graul if (!buf_pos) 621b4ba4652SKarsten Graul break; 622b4ba4652SKarsten Graul rmb = buf_pos; 623b8d19945SWen Gu ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); 624b8d19945SWen Gu ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); 625b8d19945SWen Gu ext->rt[i].rmb_vaddr_new = rmb->is_vm ? 626b8d19945SWen Gu cpu_to_be64((uintptr_t)rmb->cpu_addr) : 627b4ba4652SKarsten Graul cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 628b4ba4652SKarsten Graul buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 629b4ba4652SKarsten Graul while (buf_pos && !(buf_pos)->used) 630b4ba4652SKarsten Graul buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos); 631b4ba4652SKarsten Graul } 632b4ba4652SKarsten Graul len += i * sizeof(ext->rt[0]); 633b4ba4652SKarsten Graul out: 634aff7bfedSD. Wythe up_write(&lgr->rmbs_lock); 635b4ba4652SKarsten Graul return len; 636b4ba4652SKarsten Graul } 637b4ba4652SKarsten Graul 63852bedf37SKarsten Graul /* send ADD LINK request or response */ 6397005ada6SUrsula Braun int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], 640fbed3b37SKarsten Graul struct smc_link *link_new, 64152bedf37SKarsten Graul enum smc_llc_reqresp reqresp) 64252bedf37SKarsten Graul { 643b4ba4652SKarsten Graul struct smc_llc_msg_add_link_v2_ext *ext = NULL; 64452bedf37SKarsten Graul struct smc_llc_msg_add_link *addllc; 64552bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 646b4ba4652SKarsten Graul int len = sizeof(*addllc); 64752bedf37SKarsten Graul int rc; 64852bedf37SKarsten Graul 64995f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 65095f7f3e7SKarsten Graul return -ENOLINK; 651b4ba4652SKarsten Graul if (link->lgr->smc_version == SMC_V2) { 652b4ba4652SKarsten Graul struct smc_wr_v2_buf *wr_buf; 653b4ba4652SKarsten Graul 654b4ba4652SKarsten Graul rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 655b4ba4652SKarsten Graul if (rc) 656b4ba4652SKarsten Graul goto put_out; 657b4ba4652SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 658b4ba4652SKarsten Graul ext = (struct smc_llc_msg_add_link_v2_ext *) 659b4ba4652SKarsten Graul &wr_buf->raw[sizeof(*addllc)]; 660b4ba4652SKarsten Graul memset(ext, 0, SMC_WR_TX_SIZE); 661b4ba4652SKarsten Graul } else { 662b4ba4652SKarsten Graul struct smc_wr_buf *wr_buf; 663b4ba4652SKarsten Graul 66452bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 66552bedf37SKarsten Graul if (rc) 66695f7f3e7SKarsten Graul goto put_out; 66752bedf37SKarsten Graul addllc = (struct smc_llc_msg_add_link *)wr_buf; 668b4ba4652SKarsten Graul } 669fbed3b37SKarsten Graul 670fbed3b37SKarsten Graul memset(addllc, 0, sizeof(*addllc)); 671b4ba4652SKarsten Graul addllc->hd.common.llc_type = SMC_LLC_ADD_LINK; 672fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 673fbed3b37SKarsten Graul addllc->hd.flags |= SMC_LLC_FLAG_RESP; 674fbed3b37SKarsten Graul memcpy(addllc->sender_mac, mac, ETH_ALEN); 675fbed3b37SKarsten Graul memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); 676fbed3b37SKarsten Graul if (link_new) { 677fbed3b37SKarsten Graul addllc->link_num = link_new->link_id; 678fbed3b37SKarsten Graul hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); 679fbed3b37SKarsten Graul hton24(addllc->initial_psn, link_new->psn_initial); 680fbed3b37SKarsten Graul if (reqresp == SMC_LLC_REQ) 681fbed3b37SKarsten Graul addllc->qp_mtu = link_new->path_mtu; 682fbed3b37SKarsten Graul else 683fbed3b37SKarsten Graul addllc->qp_mtu = min(link_new->path_mtu, 684fbed3b37SKarsten Graul link_new->peer_mtu); 685fbed3b37SKarsten Graul } 686b4ba4652SKarsten Graul if (ext && link_new) 687b4ba4652SKarsten Graul len += smc_llc_fill_ext_v2(ext, link, link_new); 688b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len); 68952bedf37SKarsten Graul /* send llc message */ 690b4ba4652SKarsten Graul if (link->lgr->smc_version == SMC_V2) 691b4ba4652SKarsten Graul rc = smc_wr_tx_v2_send(link, pend, len); 692b4ba4652SKarsten Graul else 69352bedf37SKarsten Graul rc = smc_wr_tx_send(link, pend); 69495f7f3e7SKarsten Graul put_out: 69595f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 69652bedf37SKarsten Graul return rc; 69752bedf37SKarsten Graul } 69852bedf37SKarsten Graul 69952bedf37SKarsten Graul /* send DELETE LINK request or response */ 700fbed3b37SKarsten Graul int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, 701fbed3b37SKarsten Graul enum smc_llc_reqresp reqresp, bool orderly, 702fbed3b37SKarsten Graul u32 reason) 70352bedf37SKarsten Graul { 70452bedf37SKarsten Graul struct smc_llc_msg_del_link *delllc; 70552bedf37SKarsten Graul struct smc_wr_tx_pend_priv *pend; 70652bedf37SKarsten Graul struct smc_wr_buf *wr_buf; 70752bedf37SKarsten Graul int rc; 70852bedf37SKarsten Graul 70995f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 71095f7f3e7SKarsten Graul return -ENOLINK; 71152bedf37SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 71252bedf37SKarsten Graul if (rc) 71395f7f3e7SKarsten Graul goto put_out; 71452bedf37SKarsten Graul delllc = (struct smc_llc_msg_del_link *)wr_buf; 715fbed3b37SKarsten Graul 716fbed3b37SKarsten Graul memset(delllc, 0, sizeof(*delllc)); 717b4ba4652SKarsten Graul delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK; 718b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc)); 719fbed3b37SKarsten Graul if (reqresp == SMC_LLC_RESP) 720fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_RESP; 721fbed3b37SKarsten Graul if (orderly) 722fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 723fbed3b37SKarsten Graul if (link_del_id) 724fbed3b37SKarsten Graul delllc->link_num = link_del_id; 725fbed3b37SKarsten Graul else 726fbed3b37SKarsten Graul delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 727fbed3b37SKarsten Graul delllc->reason = htonl(reason); 7289bf9abeaSUrsula Braun /* send llc message */ 7299bf9abeaSUrsula Braun rc = smc_wr_tx_send(link, pend); 73095f7f3e7SKarsten Graul put_out: 73195f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 7329bf9abeaSUrsula Braun return rc; 7339bf9abeaSUrsula Braun } 7349bf9abeaSUrsula Braun 735d97935faSKarsten Graul /* send LLC test link request */ 736d97935faSKarsten Graul static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) 737313164daSKarsten Graul { 738313164daSKarsten Graul struct smc_llc_msg_test_link *testllc; 739313164daSKarsten Graul struct smc_wr_tx_pend_priv *pend; 740313164daSKarsten Graul struct smc_wr_buf *wr_buf; 741313164daSKarsten Graul int rc; 742313164daSKarsten Graul 74395f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 74495f7f3e7SKarsten Graul return -ENOLINK; 745313164daSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 746313164daSKarsten Graul if (rc) 74795f7f3e7SKarsten Graul goto put_out; 748313164daSKarsten Graul testllc = (struct smc_llc_msg_test_link *)wr_buf; 749313164daSKarsten Graul memset(testllc, 0, sizeof(*testllc)); 750b4ba4652SKarsten Graul testllc->hd.common.llc_type = SMC_LLC_TEST_LINK; 751b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc)); 752313164daSKarsten Graul memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); 753313164daSKarsten Graul /* send llc message */ 754313164daSKarsten Graul rc = smc_wr_tx_send(link, pend); 75595f7f3e7SKarsten Graul put_out: 75695f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 757313164daSKarsten Graul return rc; 758313164daSKarsten Graul } 759313164daSKarsten Graul 7606c8968c4SKarsten Graul /* schedule an llc send on link, may wait for buffers */ 7616c8968c4SKarsten Graul static int smc_llc_send_message(struct smc_link *link, void *llcbuf) 7624ed75de5SKarsten Graul { 7634ed75de5SKarsten Graul struct smc_wr_tx_pend_priv *pend; 7644ed75de5SKarsten Graul struct smc_wr_buf *wr_buf; 7654ed75de5SKarsten Graul int rc; 7664ed75de5SKarsten Graul 76795f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 7686c8968c4SKarsten Graul return -ENOLINK; 7696c8968c4SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 7704ed75de5SKarsten Graul if (rc) 77195f7f3e7SKarsten Graul goto put_out; 7726c8968c4SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 77395f7f3e7SKarsten Graul rc = smc_wr_tx_send(link, pend); 77495f7f3e7SKarsten Graul put_out: 77595f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 77695f7f3e7SKarsten Graul return rc; 7774ed75de5SKarsten Graul } 7784ed75de5SKarsten Graul 779f3811fd7SKarsten Graul /* schedule an llc send on link, may wait for buffers, 780f3811fd7SKarsten Graul * and wait for send completion notification. 781f3811fd7SKarsten Graul * @return 0 on success 782f3811fd7SKarsten Graul */ 783f3811fd7SKarsten Graul static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) 784f3811fd7SKarsten Graul { 785f3811fd7SKarsten Graul struct smc_wr_tx_pend_priv *pend; 786f3811fd7SKarsten Graul struct smc_wr_buf *wr_buf; 787f3811fd7SKarsten Graul int rc; 788f3811fd7SKarsten Graul 78995f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 790f3811fd7SKarsten Graul return -ENOLINK; 791f3811fd7SKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 792f3811fd7SKarsten Graul if (rc) 79395f7f3e7SKarsten Graul goto put_out; 794f3811fd7SKarsten Graul memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); 79595f7f3e7SKarsten Graul rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); 79695f7f3e7SKarsten Graul put_out: 79795f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 79895f7f3e7SKarsten Graul return rc; 799f3811fd7SKarsten Graul } 800f3811fd7SKarsten Graul 8019bf9abeaSUrsula Braun /********************************* receive ***********************************/ 8029bf9abeaSUrsula Braun 803336ba09fSKarsten Graul static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, 804336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t) 805336ba09fSKarsten Graul { 806336ba09fSKarsten Graul int i; 807336ba09fSKarsten Graul 808336ba09fSKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 809336ba09fSKarsten Graul (lgr->type != SMC_LGR_SINGLE && 810336ba09fSKarsten Graul (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 811336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) 812336ba09fSKarsten Graul return -EMLINK; 813336ba09fSKarsten Graul 814336ba09fSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 815336ba09fSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { 816336ba09fSKarsten Graul for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) 817336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 818336ba09fSKarsten Graul return i; 819336ba09fSKarsten Graul } else { 820336ba09fSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) 821336ba09fSKarsten Graul if (lgr->lnk[i].state == SMC_LNK_UNUSED) 822336ba09fSKarsten Graul return i; 823336ba09fSKarsten Graul } 824336ba09fSKarsten Graul return -EMLINK; 825336ba09fSKarsten Graul } 826336ba09fSKarsten Graul 82787f88cdaSKarsten Graul /* send one add_link_continue msg */ 82887f88cdaSKarsten Graul static int smc_llc_add_link_cont(struct smc_link *link, 82987f88cdaSKarsten Graul struct smc_link *link_new, u8 *num_rkeys_todo, 83087f88cdaSKarsten Graul int *buf_lst, struct smc_buf_desc **buf_pos) 83187f88cdaSKarsten Graul { 83287f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 83387f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 83487f88cdaSKarsten Graul int prim_lnk_idx, lnk_idx, i, rc; 83587f88cdaSKarsten Graul struct smc_wr_tx_pend_priv *pend; 83687f88cdaSKarsten Graul struct smc_wr_buf *wr_buf; 83787f88cdaSKarsten Graul struct smc_buf_desc *rmb; 83887f88cdaSKarsten Graul u8 n; 83987f88cdaSKarsten Graul 84095f7f3e7SKarsten Graul if (!smc_wr_tx_link_hold(link)) 84195f7f3e7SKarsten Graul return -ENOLINK; 84287f88cdaSKarsten Graul rc = smc_llc_add_pending_send(link, &wr_buf, &pend); 84387f88cdaSKarsten Graul if (rc) 84495f7f3e7SKarsten Graul goto put_out; 84587f88cdaSKarsten Graul addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; 84687f88cdaSKarsten Graul memset(addc_llc, 0, sizeof(*addc_llc)); 84787f88cdaSKarsten Graul 84887f88cdaSKarsten Graul prim_lnk_idx = link->link_idx; 84987f88cdaSKarsten Graul lnk_idx = link_new->link_idx; 85087f88cdaSKarsten Graul addc_llc->link_num = link_new->link_id; 85187f88cdaSKarsten Graul addc_llc->num_rkeys = *num_rkeys_todo; 85287f88cdaSKarsten Graul n = *num_rkeys_todo; 85387f88cdaSKarsten Graul for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { 85487f88cdaSKarsten Graul if (!*buf_pos) { 85587f88cdaSKarsten Graul addc_llc->num_rkeys = addc_llc->num_rkeys - 85687f88cdaSKarsten Graul *num_rkeys_todo; 85787f88cdaSKarsten Graul *num_rkeys_todo = 0; 85887f88cdaSKarsten Graul break; 85987f88cdaSKarsten Graul } 86087f88cdaSKarsten Graul rmb = *buf_pos; 86187f88cdaSKarsten Graul 862b8d19945SWen Gu addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey); 863b8d19945SWen Gu addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey); 864b8d19945SWen Gu addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ? 865b8d19945SWen Gu cpu_to_be64((uintptr_t)rmb->cpu_addr) : 86687f88cdaSKarsten Graul cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); 86787f88cdaSKarsten Graul 86887f88cdaSKarsten Graul (*num_rkeys_todo)--; 86987f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 87087f88cdaSKarsten Graul while (*buf_pos && !(*buf_pos)->used) 87187f88cdaSKarsten Graul *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); 87287f88cdaSKarsten Graul } 873b4ba4652SKarsten Graul addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT; 87487f88cdaSKarsten Graul addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); 87587f88cdaSKarsten Graul if (lgr->role == SMC_CLNT) 87687f88cdaSKarsten Graul addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; 87795f7f3e7SKarsten Graul rc = smc_wr_tx_send(link, pend); 87895f7f3e7SKarsten Graul put_out: 87995f7f3e7SKarsten Graul smc_wr_tx_link_put(link); 88095f7f3e7SKarsten Graul return rc; 88187f88cdaSKarsten Graul } 88287f88cdaSKarsten Graul 88387f88cdaSKarsten Graul static int smc_llc_cli_rkey_exchange(struct smc_link *link, 88487f88cdaSKarsten Graul struct smc_link *link_new) 88587f88cdaSKarsten Graul { 88687f88cdaSKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 88787f88cdaSKarsten Graul struct smc_link_group *lgr = link->lgr; 88887f88cdaSKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 88987f88cdaSKarsten Graul struct smc_llc_qentry *qentry; 89087f88cdaSKarsten Graul struct smc_buf_desc *buf_pos; 89187f88cdaSKarsten Graul int buf_lst; 89287f88cdaSKarsten Graul int rc = 0; 89387f88cdaSKarsten Graul int i; 89487f88cdaSKarsten Graul 895aff7bfedSD. Wythe down_write(&lgr->rmbs_lock); 89687f88cdaSKarsten Graul num_rkeys_send = lgr->conns_num; 89787f88cdaSKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 89887f88cdaSKarsten Graul do { 89987f88cdaSKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, 90087f88cdaSKarsten Graul SMC_LLC_ADD_LINK_CONT); 90187f88cdaSKarsten Graul if (!qentry) { 90287f88cdaSKarsten Graul rc = -ETIMEDOUT; 90387f88cdaSKarsten Graul break; 90487f88cdaSKarsten Graul } 90587f88cdaSKarsten Graul addc_llc = &qentry->msg.add_link_cont; 90687f88cdaSKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 90787f88cdaSKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 90887f88cdaSKarsten Graul for (i = 0; i < max; i++) { 90987f88cdaSKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 91087f88cdaSKarsten Graul addc_llc->rt[i].rmb_key, 91187f88cdaSKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 91287f88cdaSKarsten Graul addc_llc->rt[i].rmb_key_new); 91387f88cdaSKarsten Graul num_rkeys_recv--; 91487f88cdaSKarsten Graul } 91587f88cdaSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 91687f88cdaSKarsten Graul rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 91787f88cdaSKarsten Graul &buf_lst, &buf_pos); 91887f88cdaSKarsten Graul if (rc) 91987f88cdaSKarsten Graul break; 92087f88cdaSKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 92187f88cdaSKarsten Graul 922aff7bfedSD. Wythe up_write(&lgr->rmbs_lock); 92387f88cdaSKarsten Graul return rc; 92487f88cdaSKarsten Graul } 92587f88cdaSKarsten Graul 926336ba09fSKarsten Graul /* prepare and send an add link reject response */ 927336ba09fSKarsten Graul static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) 928336ba09fSKarsten Graul { 929336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 930336ba09fSKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; 931336ba09fSKarsten Graul qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; 932b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 933b4ba4652SKarsten Graul sizeof(qentry->msg)); 934336ba09fSKarsten Graul return smc_llc_send_message(qentry->link, &qentry->msg); 935336ba09fSKarsten Graul } 936336ba09fSKarsten Graul 937b1570a87SKarsten Graul static int smc_llc_cli_conf_link(struct smc_link *link, 938b1570a87SKarsten Graul struct smc_init_info *ini, 939b1570a87SKarsten Graul struct smc_link *link_new, 940b1570a87SKarsten Graul enum smc_lgr_type lgr_new_t) 941b1570a87SKarsten Graul { 942b1570a87SKarsten Graul struct smc_link_group *lgr = link->lgr; 943b1570a87SKarsten Graul struct smc_llc_qentry *qentry = NULL; 944b1570a87SKarsten Graul int rc = 0; 945b1570a87SKarsten Graul 946b1570a87SKarsten Graul /* receive CONFIRM LINK request over RoCE fabric */ 947b1570a87SKarsten Graul qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); 948b1570a87SKarsten Graul if (!qentry) { 949b1570a87SKarsten Graul rc = smc_llc_send_delete_link(link, link_new->link_id, 950b1570a87SKarsten Graul SMC_LLC_REQ, false, 951b1570a87SKarsten Graul SMC_LLC_DEL_LOST_PATH); 952b1570a87SKarsten Graul return -ENOLINK; 953b1570a87SKarsten Graul } 954b4ba4652SKarsten Graul if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 955b1570a87SKarsten Graul /* received DELETE_LINK instead */ 956b1570a87SKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 957b1570a87SKarsten Graul smc_llc_send_message(link, &qentry->msg); 958b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 959b1570a87SKarsten Graul return -ENOLINK; 960b1570a87SKarsten Graul } 961649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 962b1570a87SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 963b1570a87SKarsten Graul 964b1570a87SKarsten Graul rc = smc_ib_modify_qp_rts(link_new); 965b1570a87SKarsten Graul if (rc) { 966b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 967b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 968b1570a87SKarsten Graul return -ENOLINK; 969b1570a87SKarsten Graul } 970b1570a87SKarsten Graul smc_wr_remember_qp_attr(link_new); 971b1570a87SKarsten Graul 972b1570a87SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 973b1570a87SKarsten Graul if (rc) { 974b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 975b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 976b1570a87SKarsten Graul return -ENOLINK; 977b1570a87SKarsten Graul } 978b1570a87SKarsten Graul 979b1570a87SKarsten Graul /* send CONFIRM LINK response over RoCE fabric */ 980b1570a87SKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); 981b1570a87SKarsten Graul if (rc) { 982b1570a87SKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 983b1570a87SKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 984b1570a87SKarsten Graul return -ENOLINK; 985b1570a87SKarsten Graul } 986b1570a87SKarsten Graul smc_llc_link_active(link_new); 987ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 988ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 989ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 990ad6c111bSKarsten Graul else 991ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 992b1570a87SKarsten Graul return 0; 993b1570a87SKarsten Graul } 994b1570a87SKarsten Graul 995b4ba4652SKarsten Graul static void smc_llc_save_add_link_rkeys(struct smc_link *link, 996b4ba4652SKarsten Graul struct smc_link *link_new) 997b4ba4652SKarsten Graul { 998b4ba4652SKarsten Graul struct smc_llc_msg_add_link_v2_ext *ext; 999b4ba4652SKarsten Graul struct smc_link_group *lgr = link->lgr; 1000b4ba4652SKarsten Graul int max, i; 1001b4ba4652SKarsten Graul 1002b4ba4652SKarsten Graul ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 + 1003b4ba4652SKarsten Graul SMC_WR_TX_SIZE); 1004b4ba4652SKarsten Graul max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 1005aff7bfedSD. Wythe down_write(&lgr->rmbs_lock); 1006b4ba4652SKarsten Graul for (i = 0; i < max; i++) { 1007b4ba4652SKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 1008b4ba4652SKarsten Graul ext->rt[i].rmb_key, 1009b4ba4652SKarsten Graul ext->rt[i].rmb_vaddr_new, 1010b4ba4652SKarsten Graul ext->rt[i].rmb_key_new); 1011b4ba4652SKarsten Graul } 1012aff7bfedSD. Wythe up_write(&lgr->rmbs_lock); 1013b4ba4652SKarsten Graul } 1014b4ba4652SKarsten Graul 1015336ba09fSKarsten Graul static void smc_llc_save_add_link_info(struct smc_link *link, 1016336ba09fSKarsten Graul struct smc_llc_msg_add_link *add_llc) 1017336ba09fSKarsten Graul { 1018336ba09fSKarsten Graul link->peer_qpn = ntoh24(add_llc->sender_qp_num); 1019336ba09fSKarsten Graul memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); 1020336ba09fSKarsten Graul memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); 1021336ba09fSKarsten Graul link->peer_psn = ntoh24(add_llc->initial_psn); 1022336ba09fSKarsten Graul link->peer_mtu = add_llc->qp_mtu; 1023336ba09fSKarsten Graul } 1024336ba09fSKarsten Graul 1025336ba09fSKarsten Graul /* as an SMC client, process an add link request */ 1026336ba09fSKarsten Graul int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) 1027336ba09fSKarsten Graul { 1028336ba09fSKarsten Graul struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; 1029336ba09fSKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 1030336ba09fSKarsten Graul struct smc_link_group *lgr = smc_get_lgr(link); 1031ed990df2SKarsten Graul struct smc_init_info *ini = NULL; 1032336ba09fSKarsten Graul struct smc_link *lnk_new = NULL; 1033336ba09fSKarsten Graul int lnk_idx, rc = 0; 1034336ba09fSKarsten Graul 1035fffe83c8SKarsten Graul if (!llc->qp_mtu) 1036fffe83c8SKarsten Graul goto out_reject; 1037fffe83c8SKarsten Graul 1038ed990df2SKarsten Graul ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1039ed990df2SKarsten Graul if (!ini) { 1040ed990df2SKarsten Graul rc = -ENOMEM; 1041ed990df2SKarsten Graul goto out_reject; 1042ed990df2SKarsten Graul } 1043ed990df2SKarsten Graul 1044ed990df2SKarsten Graul ini->vlan_id = lgr->vlan_id; 1045b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1046b4ba4652SKarsten Graul ini->check_smcrv2 = true; 1047b4ba4652SKarsten Graul ini->smcrv2.saddr = lgr->saddr; 1048b4ba4652SKarsten Graul ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid); 1049b4ba4652SKarsten Graul } 1050ed990df2SKarsten Graul smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1051336ba09fSKarsten Graul if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1052b4ba4652SKarsten Graul (lgr->smc_version == SMC_V2 || 1053b4ba4652SKarsten Graul !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) { 1054b4ba4652SKarsten Graul if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2) 1055336ba09fSKarsten Graul goto out_reject; 1056336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 1057336ba09fSKarsten Graul } 1058b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1059b4ba4652SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1060b4ba4652SKarsten Graul ini->smcrv2.ib_dev_v2 = link->smcibdev; 1061b4ba4652SKarsten Graul ini->smcrv2.ib_port_v2 = link->ibport; 1062b4ba4652SKarsten Graul } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 1063336ba09fSKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1064ed990df2SKarsten Graul ini->ib_dev = link->smcibdev; 1065ed990df2SKarsten Graul ini->ib_port = link->ibport; 1066336ba09fSKarsten Graul } 1067336ba09fSKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1068336ba09fSKarsten Graul if (lnk_idx < 0) 1069336ba09fSKarsten Graul goto out_reject; 1070336ba09fSKarsten Graul lnk_new = &lgr->lnk[lnk_idx]; 1071ed990df2SKarsten Graul rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini); 1072336ba09fSKarsten Graul if (rc) 1073336ba09fSKarsten Graul goto out_reject; 1074336ba09fSKarsten Graul smc_llc_save_add_link_info(lnk_new, llc); 107545fa8da0SKarsten Graul lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ 107645fa8da0SKarsten Graul smc_llc_link_set_uid(lnk_new); 1077336ba09fSKarsten Graul 1078336ba09fSKarsten Graul rc = smc_ib_ready_link(lnk_new); 1079336ba09fSKarsten Graul if (rc) 1080336ba09fSKarsten Graul goto out_clear_lnk; 1081336ba09fSKarsten Graul 1082336ba09fSKarsten Graul rc = smcr_buf_map_lgr(lnk_new); 1083336ba09fSKarsten Graul if (rc) 1084336ba09fSKarsten Graul goto out_clear_lnk; 1085336ba09fSKarsten Graul 1086336ba09fSKarsten Graul rc = smc_llc_send_add_link(link, 1087b4ba4652SKarsten Graul lnk_new->smcibdev->mac[lnk_new->ibport - 1], 1088336ba09fSKarsten Graul lnk_new->gid, lnk_new, SMC_LLC_RESP); 1089336ba09fSKarsten Graul if (rc) 1090336ba09fSKarsten Graul goto out_clear_lnk; 1091b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1092b4ba4652SKarsten Graul smc_llc_save_add_link_rkeys(link, lnk_new); 1093b4ba4652SKarsten Graul } else { 109487f88cdaSKarsten Graul rc = smc_llc_cli_rkey_exchange(link, lnk_new); 1095336ba09fSKarsten Graul if (rc) { 1096336ba09fSKarsten Graul rc = 0; 1097336ba09fSKarsten Graul goto out_clear_lnk; 1098336ba09fSKarsten Graul } 1099b4ba4652SKarsten Graul } 1100ed990df2SKarsten Graul rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t); 1101336ba09fSKarsten Graul if (!rc) 1102336ba09fSKarsten Graul goto out; 1103336ba09fSKarsten Graul out_clear_lnk: 11048f3d65c1SKarsten Graul lnk_new->state = SMC_LNK_INACTIVE; 11050a99be43SKarsten Graul smcr_link_clear(lnk_new, false); 1106336ba09fSKarsten Graul out_reject: 1107336ba09fSKarsten Graul smc_llc_cli_add_link_reject(qentry); 1108336ba09fSKarsten Graul out: 1109ed990df2SKarsten Graul kfree(ini); 1110336ba09fSKarsten Graul kfree(qentry); 1111336ba09fSKarsten Graul return rc; 1112336ba09fSKarsten Graul } 1113336ba09fSKarsten Graul 1114b4ba4652SKarsten Graul static void smc_llc_send_request_add_link(struct smc_link *link) 1115b4ba4652SKarsten Graul { 1116b4ba4652SKarsten Graul struct smc_llc_msg_req_add_link_v2 *llc; 1117b4ba4652SKarsten Graul struct smc_wr_tx_pend_priv *pend; 1118b4ba4652SKarsten Graul struct smc_wr_v2_buf *wr_buf; 1119b4ba4652SKarsten Graul struct smc_gidlist gidlist; 1120b4ba4652SKarsten Graul int rc, len, i; 1121b4ba4652SKarsten Graul 1122b4ba4652SKarsten Graul if (!smc_wr_tx_link_hold(link)) 1123b4ba4652SKarsten Graul return; 1124b4ba4652SKarsten Graul if (link->lgr->type == SMC_LGR_SYMMETRIC || 1125b4ba4652SKarsten Graul link->lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1126b4ba4652SKarsten Graul goto put_out; 1127b4ba4652SKarsten Graul 1128b4ba4652SKarsten Graul smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid); 1129b4ba4652SKarsten Graul if (gidlist.len <= 1) 1130b4ba4652SKarsten Graul goto put_out; 1131b4ba4652SKarsten Graul 1132b4ba4652SKarsten Graul rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend); 1133b4ba4652SKarsten Graul if (rc) 1134b4ba4652SKarsten Graul goto put_out; 1135b4ba4652SKarsten Graul llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf; 1136b4ba4652SKarsten Graul memset(llc, 0, SMC_WR_TX_SIZE); 1137b4ba4652SKarsten Graul 1138b4ba4652SKarsten Graul llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK; 1139b4ba4652SKarsten Graul for (i = 0; i < gidlist.len; i++) 1140b4ba4652SKarsten Graul memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0])); 1141b4ba4652SKarsten Graul llc->gid_cnt = gidlist.len; 1142b4ba4652SKarsten Graul len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0])); 1143b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&llc->hd, link->lgr, len); 1144b4ba4652SKarsten Graul rc = smc_wr_tx_v2_send(link, pend, len); 1145b4ba4652SKarsten Graul if (!rc) 1146b4ba4652SKarsten Graul /* set REQ_ADD_LINK flow and wait for response from peer */ 1147b4ba4652SKarsten Graul link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK; 1148b4ba4652SKarsten Graul put_out: 1149b4ba4652SKarsten Graul smc_wr_tx_link_put(link); 1150b4ba4652SKarsten Graul } 1151b4ba4652SKarsten Graul 1152c48254faSKarsten Graul /* as an SMC client, invite server to start the add_link processing */ 1153c48254faSKarsten Graul static void smc_llc_cli_add_link_invite(struct smc_link *link, 1154c48254faSKarsten Graul struct smc_llc_qentry *qentry) 1155c48254faSKarsten Graul { 1156c48254faSKarsten Graul struct smc_link_group *lgr = smc_get_lgr(link); 1157ed990df2SKarsten Graul struct smc_init_info *ini = NULL; 1158c48254faSKarsten Graul 1159b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1160b4ba4652SKarsten Graul smc_llc_send_request_add_link(link); 1161b4ba4652SKarsten Graul goto out; 1162b4ba4652SKarsten Graul } 1163b4ba4652SKarsten Graul 1164c48254faSKarsten Graul if (lgr->type == SMC_LGR_SYMMETRIC || 1165c48254faSKarsten Graul lgr->type == SMC_LGR_ASYMMETRIC_PEER) 1166c48254faSKarsten Graul goto out; 1167c48254faSKarsten Graul 1168ed990df2SKarsten Graul ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1169ed990df2SKarsten Graul if (!ini) 1170c48254faSKarsten Graul goto out; 1171c48254faSKarsten Graul 1172ed990df2SKarsten Graul ini->vlan_id = lgr->vlan_id; 1173ed990df2SKarsten Graul smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1174ed990df2SKarsten Graul if (!ini->ib_dev) 1175ed990df2SKarsten Graul goto out; 1176ed990df2SKarsten Graul 1177ed990df2SKarsten Graul smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1], 1178ed990df2SKarsten Graul ini->ib_gid, NULL, SMC_LLC_REQ); 1179c48254faSKarsten Graul out: 1180ed990df2SKarsten Graul kfree(ini); 1181c48254faSKarsten Graul kfree(qentry); 1182c48254faSKarsten Graul } 1183c48254faSKarsten Graul 1184fffe83c8SKarsten Graul static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc) 1185fffe83c8SKarsten Graul { 1186fffe83c8SKarsten Graul int i; 1187fffe83c8SKarsten Graul 1188fffe83c8SKarsten Graul for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++) 1189fffe83c8SKarsten Graul if (llc->raw.data[i]) 1190fffe83c8SKarsten Graul return false; 1191fffe83c8SKarsten Graul return true; 1192fffe83c8SKarsten Graul } 1193fffe83c8SKarsten Graul 1194c48254faSKarsten Graul static bool smc_llc_is_local_add_link(union smc_llc_msg *llc) 1195c48254faSKarsten Graul { 1196b4ba4652SKarsten Graul if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK && 1197fffe83c8SKarsten Graul smc_llc_is_empty_llc_message(llc)) 1198c48254faSKarsten Graul return true; 1199c48254faSKarsten Graul return false; 1200c48254faSKarsten Graul } 1201c48254faSKarsten Graul 1202b1570a87SKarsten Graul static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) 1203b1570a87SKarsten Graul { 1204b1570a87SKarsten Graul struct smc_llc_qentry *qentry; 1205b1570a87SKarsten Graul 1206b1570a87SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 1207b1570a87SKarsten Graul 1208b5dd4d69SD. Wythe down_write(&lgr->llc_conf_mutex); 1209c48254faSKarsten Graul if (smc_llc_is_local_add_link(&qentry->msg)) 1210c48254faSKarsten Graul smc_llc_cli_add_link_invite(qentry->link, qentry); 1211c48254faSKarsten Graul else 1212b1570a87SKarsten Graul smc_llc_cli_add_link(qentry->link, qentry); 1213b5dd4d69SD. Wythe up_write(&lgr->llc_conf_mutex); 1214b1570a87SKarsten Graul } 1215b1570a87SKarsten Graul 12169c416878SKarsten Graul static int smc_llc_active_link_count(struct smc_link_group *lgr) 12179c416878SKarsten Graul { 12189c416878SKarsten Graul int i, link_count = 0; 12199c416878SKarsten Graul 12209c416878SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1221741a49a4SKarsten Graul if (!smc_link_active(&lgr->lnk[i])) 12229c416878SKarsten Graul continue; 12239c416878SKarsten Graul link_count++; 12249c416878SKarsten Graul } 12259c416878SKarsten Graul return link_count; 12269c416878SKarsten Graul } 12279c416878SKarsten Graul 1228c9a5d243SKarsten Graul /* find the asymmetric link when 3 links are established */ 1229c9a5d243SKarsten Graul static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) 1230c9a5d243SKarsten Graul { 1231c9a5d243SKarsten Graul int asym_idx = -ENOENT; 1232c9a5d243SKarsten Graul int i, j, k; 1233c9a5d243SKarsten Graul bool found; 1234c9a5d243SKarsten Graul 1235c9a5d243SKarsten Graul /* determine asymmetric link */ 1236c9a5d243SKarsten Graul found = false; 1237c9a5d243SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 1238c9a5d243SKarsten Graul for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { 1239c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[i]) || 1240c9a5d243SKarsten Graul !smc_link_usable(&lgr->lnk[j])) 1241c9a5d243SKarsten Graul continue; 1242c9a5d243SKarsten Graul if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, 1243c9a5d243SKarsten Graul SMC_GID_SIZE)) { 1244c9a5d243SKarsten Graul found = true; /* asym_lnk is i or j */ 1245c9a5d243SKarsten Graul break; 1246c9a5d243SKarsten Graul } 1247c9a5d243SKarsten Graul } 1248c9a5d243SKarsten Graul if (found) 1249c9a5d243SKarsten Graul break; 1250c9a5d243SKarsten Graul } 1251c9a5d243SKarsten Graul if (!found) 1252c9a5d243SKarsten Graul goto out; /* no asymmetric link */ 1253c9a5d243SKarsten Graul for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { 1254c9a5d243SKarsten Graul if (!smc_link_usable(&lgr->lnk[k])) 1255c9a5d243SKarsten Graul continue; 1256c9a5d243SKarsten Graul if (k != i && 1257c9a5d243SKarsten Graul !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, 1258c9a5d243SKarsten Graul SMC_GID_SIZE)) { 1259c9a5d243SKarsten Graul asym_idx = i; 1260c9a5d243SKarsten Graul break; 1261c9a5d243SKarsten Graul } 1262c9a5d243SKarsten Graul if (k != j && 1263c9a5d243SKarsten Graul !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, 1264c9a5d243SKarsten Graul SMC_GID_SIZE)) { 1265c9a5d243SKarsten Graul asym_idx = j; 1266c9a5d243SKarsten Graul break; 1267c9a5d243SKarsten Graul } 1268c9a5d243SKarsten Graul } 1269c9a5d243SKarsten Graul out: 1270c9a5d243SKarsten Graul return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; 1271c9a5d243SKarsten Graul } 1272c9a5d243SKarsten Graul 1273c9a5d243SKarsten Graul static void smc_llc_delete_asym_link(struct smc_link_group *lgr) 1274c9a5d243SKarsten Graul { 1275c9a5d243SKarsten Graul struct smc_link *lnk_new = NULL, *lnk_asym; 1276c9a5d243SKarsten Graul struct smc_llc_qentry *qentry; 1277c9a5d243SKarsten Graul int rc; 1278c9a5d243SKarsten Graul 1279c9a5d243SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 1280c9a5d243SKarsten Graul if (!lnk_asym) 1281c9a5d243SKarsten Graul return; /* no asymmetric link */ 1282c9a5d243SKarsten Graul if (!smc_link_downing(&lnk_asym->state)) 1283c9a5d243SKarsten Graul return; 1284c6f02ebeSKarsten Graul lnk_new = smc_switch_conns(lgr, lnk_asym, false); 1285c9a5d243SKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_asym); 1286c9a5d243SKarsten Graul if (!lnk_new) 1287c9a5d243SKarsten Graul goto out_free; 1288c9a5d243SKarsten Graul /* change flow type from ADD_LINK into DEL_LINK */ 1289c9a5d243SKarsten Graul lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; 1290c9a5d243SKarsten Graul rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, 1291c9a5d243SKarsten Graul true, SMC_LLC_DEL_NO_ASYM_NEEDED); 1292c9a5d243SKarsten Graul if (rc) { 1293c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 1294c9a5d243SKarsten Graul goto out_free; 1295c9a5d243SKarsten Graul } 1296c9a5d243SKarsten Graul qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, 1297c9a5d243SKarsten Graul SMC_LLC_DELETE_LINK); 1298c9a5d243SKarsten Graul if (!qentry) { 1299c9a5d243SKarsten Graul smcr_link_down_cond(lnk_new); 1300c9a5d243SKarsten Graul goto out_free; 1301c9a5d243SKarsten Graul } 1302c9a5d243SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1303c9a5d243SKarsten Graul out_free: 13040a99be43SKarsten Graul smcr_link_clear(lnk_asym, true); 1305c9a5d243SKarsten Graul } 1306c9a5d243SKarsten Graul 130757b49924SKarsten Graul static int smc_llc_srv_rkey_exchange(struct smc_link *link, 130857b49924SKarsten Graul struct smc_link *link_new) 130957b49924SKarsten Graul { 131057b49924SKarsten Graul struct smc_llc_msg_add_link_cont *addc_llc; 131157b49924SKarsten Graul struct smc_link_group *lgr = link->lgr; 131257b49924SKarsten Graul u8 max, num_rkeys_send, num_rkeys_recv; 131357b49924SKarsten Graul struct smc_llc_qentry *qentry = NULL; 131457b49924SKarsten Graul struct smc_buf_desc *buf_pos; 131557b49924SKarsten Graul int buf_lst; 131657b49924SKarsten Graul int rc = 0; 131757b49924SKarsten Graul int i; 131857b49924SKarsten Graul 1319aff7bfedSD. Wythe down_write(&lgr->rmbs_lock); 132057b49924SKarsten Graul num_rkeys_send = lgr->conns_num; 132157b49924SKarsten Graul buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); 132257b49924SKarsten Graul do { 132357b49924SKarsten Graul smc_llc_add_link_cont(link, link_new, &num_rkeys_send, 132457b49924SKarsten Graul &buf_lst, &buf_pos); 132557b49924SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, 132657b49924SKarsten Graul SMC_LLC_ADD_LINK_CONT); 132757b49924SKarsten Graul if (!qentry) { 132857b49924SKarsten Graul rc = -ETIMEDOUT; 132957b49924SKarsten Graul goto out; 133057b49924SKarsten Graul } 133157b49924SKarsten Graul addc_llc = &qentry->msg.add_link_cont; 133257b49924SKarsten Graul num_rkeys_recv = addc_llc->num_rkeys; 133357b49924SKarsten Graul max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); 133457b49924SKarsten Graul for (i = 0; i < max; i++) { 133557b49924SKarsten Graul smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, 133657b49924SKarsten Graul addc_llc->rt[i].rmb_key, 133757b49924SKarsten Graul addc_llc->rt[i].rmb_vaddr_new, 133857b49924SKarsten Graul addc_llc->rt[i].rmb_key_new); 133957b49924SKarsten Graul num_rkeys_recv--; 134057b49924SKarsten Graul } 134157b49924SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 134257b49924SKarsten Graul } while (num_rkeys_send || num_rkeys_recv); 134357b49924SKarsten Graul out: 1344aff7bfedSD. Wythe up_write(&lgr->rmbs_lock); 134557b49924SKarsten Graul return rc; 134657b49924SKarsten Graul } 134757b49924SKarsten Graul 13481551c95bSKarsten Graul static int smc_llc_srv_conf_link(struct smc_link *link, 13491551c95bSKarsten Graul struct smc_link *link_new, 13501551c95bSKarsten Graul enum smc_lgr_type lgr_new_t) 13511551c95bSKarsten Graul { 13521551c95bSKarsten Graul struct smc_link_group *lgr = link->lgr; 13531551c95bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 13541551c95bSKarsten Graul int rc; 13551551c95bSKarsten Graul 13561551c95bSKarsten Graul /* send CONFIRM LINK request over the RoCE fabric */ 13571551c95bSKarsten Graul rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); 13581551c95bSKarsten Graul if (rc) 13591551c95bSKarsten Graul return -ENOLINK; 13601551c95bSKarsten Graul /* receive CONFIRM LINK response over the RoCE fabric */ 1361a35fffbfSKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0); 1362a35fffbfSKarsten Graul if (!qentry || 1363b4ba4652SKarsten Graul qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) { 13641551c95bSKarsten Graul /* send DELETE LINK */ 13651551c95bSKarsten Graul smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, 13661551c95bSKarsten Graul false, SMC_LLC_DEL_LOST_PATH); 1367a35fffbfSKarsten Graul if (qentry) 1368a35fffbfSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 13691551c95bSKarsten Graul return -ENOLINK; 13701551c95bSKarsten Graul } 1371649758ffSKarsten Graul smc_llc_save_peer_uid(qentry); 13721551c95bSKarsten Graul smc_llc_link_active(link_new); 1373ad6c111bSKarsten Graul if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || 1374ad6c111bSKarsten Graul lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) 1375ad6c111bSKarsten Graul smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); 1376ad6c111bSKarsten Graul else 1377ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, lgr_new_t); 13781551c95bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 13791551c95bSKarsten Graul return 0; 13801551c95bSKarsten Graul } 13811551c95bSKarsten Graul 1382b4ba4652SKarsten Graul static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry) 1383b4ba4652SKarsten Graul { 1384b4ba4652SKarsten Graul qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; 1385b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr, 1386b4ba4652SKarsten Graul sizeof(qentry->msg)); 1387b4ba4652SKarsten Graul memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data)); 1388b4ba4652SKarsten Graul smc_llc_send_message(qentry->link, &qentry->msg); 1389b4ba4652SKarsten Graul } 1390b4ba4652SKarsten Graul 1391b4ba4652SKarsten Graul int smc_llc_srv_add_link(struct smc_link *link, 1392b4ba4652SKarsten Graul struct smc_llc_qentry *req_qentry) 13932d2209f2SKarsten Graul { 13942d2209f2SKarsten Graul enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; 13952d2209f2SKarsten Graul struct smc_link_group *lgr = link->lgr; 13962d2209f2SKarsten Graul struct smc_llc_msg_add_link *add_llc; 13972d2209f2SKarsten Graul struct smc_llc_qentry *qentry = NULL; 1398b4ba4652SKarsten Graul bool send_req_add_link_resp = false; 1399ed990df2SKarsten Graul struct smc_link *link_new = NULL; 1400b4ba4652SKarsten Graul struct smc_init_info *ini = NULL; 14012d2209f2SKarsten Graul int lnk_idx, rc = 0; 14022d2209f2SKarsten Graul 1403b4ba4652SKarsten Graul if (req_qentry && 1404b4ba4652SKarsten Graul req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK) 1405b4ba4652SKarsten Graul send_req_add_link_resp = true; 1406b4ba4652SKarsten Graul 1407ed990df2SKarsten Graul ini = kzalloc(sizeof(*ini), GFP_KERNEL); 1408b4ba4652SKarsten Graul if (!ini) { 1409b4ba4652SKarsten Graul rc = -ENOMEM; 1410b4ba4652SKarsten Graul goto out; 1411b4ba4652SKarsten Graul } 1412ed990df2SKarsten Graul 14132d2209f2SKarsten Graul /* ignore client add link recommendation, start new flow */ 1414ed990df2SKarsten Graul ini->vlan_id = lgr->vlan_id; 1415b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1416b4ba4652SKarsten Graul ini->check_smcrv2 = true; 1417b4ba4652SKarsten Graul ini->smcrv2.saddr = lgr->saddr; 1418b4ba4652SKarsten Graul if (send_req_add_link_resp) { 1419b4ba4652SKarsten Graul struct smc_llc_msg_req_add_link_v2 *req_add = 1420b4ba4652SKarsten Graul &req_qentry->msg.req_add_link; 1421b4ba4652SKarsten Graul 1422b4ba4652SKarsten Graul ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]); 1423b4ba4652SKarsten Graul } 1424b4ba4652SKarsten Graul } 1425ed990df2SKarsten Graul smc_pnet_find_alt_roce(lgr, ini, link->smcibdev); 1426b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) { 1427b4ba4652SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1428b4ba4652SKarsten Graul ini->smcrv2.ib_dev_v2 = link->smcibdev; 1429b4ba4652SKarsten Graul ini->smcrv2.ib_port_v2 = link->ibport; 1430b4ba4652SKarsten Graul } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) { 14312d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; 1432ed990df2SKarsten Graul ini->ib_dev = link->smcibdev; 1433ed990df2SKarsten Graul ini->ib_port = link->ibport; 14342d2209f2SKarsten Graul } 14352d2209f2SKarsten Graul lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); 1436ed990df2SKarsten Graul if (lnk_idx < 0) { 1437ed990df2SKarsten Graul rc = 0; 1438ed990df2SKarsten Graul goto out; 1439ed990df2SKarsten Graul } 14402d2209f2SKarsten Graul 1441ed990df2SKarsten Graul rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini); 14422d2209f2SKarsten Graul if (rc) 1443ed990df2SKarsten Graul goto out; 14442d2209f2SKarsten Graul link_new = &lgr->lnk[lnk_idx]; 1445b4ba4652SKarsten Graul 1446b4ba4652SKarsten Graul rc = smcr_buf_map_lgr(link_new); 1447b4ba4652SKarsten Graul if (rc) 1448b4ba4652SKarsten Graul goto out_err; 1449b4ba4652SKarsten Graul 14502d2209f2SKarsten Graul rc = smc_llc_send_add_link(link, 1451b4ba4652SKarsten Graul link_new->smcibdev->mac[link_new->ibport-1], 14522d2209f2SKarsten Graul link_new->gid, link_new, SMC_LLC_REQ); 14532d2209f2SKarsten Graul if (rc) 14542d2209f2SKarsten Graul goto out_err; 1455b4ba4652SKarsten Graul send_req_add_link_resp = false; 14562d2209f2SKarsten Graul /* receive ADD LINK response over the RoCE fabric */ 14572d2209f2SKarsten Graul qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); 14582d2209f2SKarsten Graul if (!qentry) { 14592d2209f2SKarsten Graul rc = -ETIMEDOUT; 14602d2209f2SKarsten Graul goto out_err; 14612d2209f2SKarsten Graul } 14622d2209f2SKarsten Graul add_llc = &qentry->msg.add_link; 14632d2209f2SKarsten Graul if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { 14642d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 14652d2209f2SKarsten Graul rc = -ENOLINK; 14662d2209f2SKarsten Graul goto out_err; 14672d2209f2SKarsten Graul } 14682d2209f2SKarsten Graul if (lgr->type == SMC_LGR_SINGLE && 14692d2209f2SKarsten Graul (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && 1470b4ba4652SKarsten Graul (lgr->smc_version == SMC_V2 || 1471b4ba4652SKarsten Graul !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) { 14722d2209f2SKarsten Graul lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; 14732d2209f2SKarsten Graul } 14742d2209f2SKarsten Graul smc_llc_save_add_link_info(link_new, add_llc); 14752d2209f2SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 14762d2209f2SKarsten Graul 14772d2209f2SKarsten Graul rc = smc_ib_ready_link(link_new); 14782d2209f2SKarsten Graul if (rc) 14792d2209f2SKarsten Graul goto out_err; 14802d2209f2SKarsten Graul rc = smcr_buf_reg_lgr(link_new); 14812d2209f2SKarsten Graul if (rc) 14822d2209f2SKarsten Graul goto out_err; 1483b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1484b4ba4652SKarsten Graul smc_llc_save_add_link_rkeys(link, link_new); 1485b4ba4652SKarsten Graul } else { 148657b49924SKarsten Graul rc = smc_llc_srv_rkey_exchange(link, link_new); 14872d2209f2SKarsten Graul if (rc) 14882d2209f2SKarsten Graul goto out_err; 1489b4ba4652SKarsten Graul } 14901551c95bSKarsten Graul rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); 14912d2209f2SKarsten Graul if (rc) 14922d2209f2SKarsten Graul goto out_err; 1493ed990df2SKarsten Graul kfree(ini); 14942d2209f2SKarsten Graul return 0; 14952d2209f2SKarsten Graul out_err: 1496ed990df2SKarsten Graul if (link_new) { 14978f3d65c1SKarsten Graul link_new->state = SMC_LNK_INACTIVE; 14980a99be43SKarsten Graul smcr_link_clear(link_new, false); 1499ed990df2SKarsten Graul } 1500ed990df2SKarsten Graul out: 1501ed990df2SKarsten Graul kfree(ini); 1502b4ba4652SKarsten Graul if (send_req_add_link_resp) 1503b4ba4652SKarsten Graul smc_llc_send_req_add_link_response(req_qentry); 15042d2209f2SKarsten Graul return rc; 15052d2209f2SKarsten Graul } 15062d2209f2SKarsten Graul 15072d2209f2SKarsten Graul static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) 15082d2209f2SKarsten Graul { 15092d2209f2SKarsten Graul struct smc_link *link = lgr->llc_flow_lcl.qentry->link; 1510b4ba4652SKarsten Graul struct smc_llc_qentry *qentry; 15112d2209f2SKarsten Graul int rc; 15122d2209f2SKarsten Graul 1513b4ba4652SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 15142d2209f2SKarsten Graul 1515b5dd4d69SD. Wythe down_write(&lgr->llc_conf_mutex); 1516b4ba4652SKarsten Graul rc = smc_llc_srv_add_link(link, qentry); 15172d2209f2SKarsten Graul if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { 15182d2209f2SKarsten Graul /* delete any asymmetric link */ 1519c9a5d243SKarsten Graul smc_llc_delete_asym_link(lgr); 15202d2209f2SKarsten Graul } 1521b5dd4d69SD. Wythe up_write(&lgr->llc_conf_mutex); 1522b4ba4652SKarsten Graul kfree(qentry); 15232d2209f2SKarsten Graul } 15242d2209f2SKarsten Graul 1525c48254faSKarsten Graul /* enqueue a local add_link req to trigger a new add_link flow */ 1526c48254faSKarsten Graul void smc_llc_add_link_local(struct smc_link *link) 15274dadd151SKarsten Graul { 152816cb3653SPujin Shi struct smc_llc_msg_add_link add_llc = {}; 15294dadd151SKarsten Graul 1530b4ba4652SKarsten Graul add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; 1531b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); 1532c48254faSKarsten Graul /* no dev and port needed */ 15334dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); 15344dadd151SKarsten Graul } 15354dadd151SKarsten Graul 1536b45e7f98SKarsten Graul /* worker to process an add link message */ 1537b45e7f98SKarsten Graul static void smc_llc_add_link_work(struct work_struct *work) 1538b45e7f98SKarsten Graul { 1539b45e7f98SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 1540b45e7f98SKarsten Graul llc_add_link_work); 1541b45e7f98SKarsten Graul 1542b45e7f98SKarsten Graul if (list_empty(&lgr->list)) { 1543b45e7f98SKarsten Graul /* link group is terminating */ 1544b45e7f98SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 1545b45e7f98SKarsten Graul goto out; 1546b45e7f98SKarsten Graul } 1547b45e7f98SKarsten Graul 1548b1570a87SKarsten Graul if (lgr->role == SMC_CLNT) 1549b1570a87SKarsten Graul smc_llc_process_cli_add_link(lgr); 15502d2209f2SKarsten Graul else 15512d2209f2SKarsten Graul smc_llc_process_srv_add_link(lgr); 1552b45e7f98SKarsten Graul out: 1553b4ba4652SKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK) 1554b45e7f98SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 1555b45e7f98SKarsten Graul } 1556b45e7f98SKarsten Graul 15574dadd151SKarsten Graul /* enqueue a local del_link msg to trigger a new del_link flow, 15584dadd151SKarsten Graul * called only for role SMC_SERV 15594dadd151SKarsten Graul */ 15604dadd151SKarsten Graul void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) 15614dadd151SKarsten Graul { 156216cb3653SPujin Shi struct smc_llc_msg_del_link del_llc = {}; 15634dadd151SKarsten Graul 1564b4ba4652SKarsten Graul del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1565b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc)); 15664dadd151SKarsten Graul del_llc.link_num = del_link_id; 15674dadd151SKarsten Graul del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); 15684dadd151SKarsten Graul del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 15694dadd151SKarsten Graul smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); 15704dadd151SKarsten Graul } 15714dadd151SKarsten Graul 15729c416878SKarsten Graul static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) 15739c416878SKarsten Graul { 15749c416878SKarsten Graul struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; 15759c416878SKarsten Graul struct smc_llc_msg_del_link *del_llc; 15769c416878SKarsten Graul struct smc_llc_qentry *qentry; 15779c416878SKarsten Graul int active_links; 15789c416878SKarsten Graul int lnk_idx; 15799c416878SKarsten Graul 15809c416878SKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 15819c416878SKarsten Graul lnk = qentry->link; 15829c416878SKarsten Graul del_llc = &qentry->msg.delete_link; 15839c416878SKarsten Graul 15849c416878SKarsten Graul if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 15859c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 15869c416878SKarsten Graul goto out; 15879c416878SKarsten Graul } 1588b5dd4d69SD. Wythe down_write(&lgr->llc_conf_mutex); 15899c416878SKarsten Graul /* delete single link */ 15909c416878SKarsten Graul for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { 15919c416878SKarsten Graul if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) 15929c416878SKarsten Graul continue; 15939c416878SKarsten Graul lnk_del = &lgr->lnk[lnk_idx]; 15949c416878SKarsten Graul break; 15959c416878SKarsten Graul } 15969c416878SKarsten Graul del_llc->hd.flags |= SMC_LLC_FLAG_RESP; 15979c416878SKarsten Graul if (!lnk_del) { 15989c416878SKarsten Graul /* link was not found */ 15999c416878SKarsten Graul del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); 16009c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); 16019c416878SKarsten Graul goto out_unlock; 16029c416878SKarsten Graul } 16039c416878SKarsten Graul lnk_asym = smc_llc_find_asym_link(lgr); 16049c416878SKarsten Graul 16059c416878SKarsten Graul del_llc->reason = 0; 16069c416878SKarsten Graul smc_llc_send_message(lnk, &qentry->msg); /* response */ 16079c416878SKarsten Graul 16088f3d65c1SKarsten Graul if (smc_link_downing(&lnk_del->state)) 16098f3d65c1SKarsten Graul smc_switch_conns(lgr, lnk_del, false); 16100a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 16119c416878SKarsten Graul 16129c416878SKarsten Graul active_links = smc_llc_active_link_count(lgr); 16139c416878SKarsten Graul if (lnk_del == lnk_asym) { 16149c416878SKarsten Graul /* expected deletion of asym link, don't change lgr state */ 16159c416878SKarsten Graul } else if (active_links == 1) { 1616ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 16179c416878SKarsten Graul } else if (!active_links) { 1618ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 16199c416878SKarsten Graul smc_lgr_terminate_sched(lgr); 16209c416878SKarsten Graul } 16219c416878SKarsten Graul out_unlock: 1622b5dd4d69SD. Wythe up_write(&lgr->llc_conf_mutex); 16239c416878SKarsten Graul out: 16249c416878SKarsten Graul kfree(qentry); 16259c416878SKarsten Graul } 16269c416878SKarsten Graul 1627f3811fd7SKarsten Graul /* try to send a DELETE LINK ALL request on any active link, 1628f3811fd7SKarsten Graul * waiting for send completion 1629f3811fd7SKarsten Graul */ 1630f3811fd7SKarsten Graul void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) 1631f3811fd7SKarsten Graul { 16327e94e46cSPujin Shi struct smc_llc_msg_del_link delllc = {}; 1633f3811fd7SKarsten Graul int i; 1634f3811fd7SKarsten Graul 1635b4ba4652SKarsten Graul delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK; 1636b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc)); 1637f3811fd7SKarsten Graul if (ord) 1638f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; 1639f3811fd7SKarsten Graul delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; 1640f3811fd7SKarsten Graul delllc.reason = htonl(rsn); 1641f3811fd7SKarsten Graul 1642f3811fd7SKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 164390cee52fSDust Li if (!smc_link_sendable(&lgr->lnk[i])) 1644f3811fd7SKarsten Graul continue; 1645f3811fd7SKarsten Graul if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) 1646f3811fd7SKarsten Graul break; 1647f3811fd7SKarsten Graul } 1648f3811fd7SKarsten Graul } 1649f3811fd7SKarsten Graul 165008ae27ddSKarsten Graul static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) 165108ae27ddSKarsten Graul { 165208ae27ddSKarsten Graul struct smc_llc_msg_del_link *del_llc; 165308ae27ddSKarsten Graul struct smc_link *lnk, *lnk_del; 165408ae27ddSKarsten Graul struct smc_llc_qentry *qentry; 165508ae27ddSKarsten Graul int active_links; 165608ae27ddSKarsten Graul int i; 165708ae27ddSKarsten Graul 1658b5dd4d69SD. Wythe down_write(&lgr->llc_conf_mutex); 165908ae27ddSKarsten Graul qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); 166008ae27ddSKarsten Graul lnk = qentry->link; 166108ae27ddSKarsten Graul del_llc = &qentry->msg.delete_link; 166208ae27ddSKarsten Graul 166308ae27ddSKarsten Graul if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { 166408ae27ddSKarsten Graul /* delete entire lgr */ 1665f3811fd7SKarsten Graul smc_llc_send_link_delete_all(lgr, true, ntohl( 1666f3811fd7SKarsten Graul qentry->msg.delete_link.reason)); 166708ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 166808ae27ddSKarsten Graul goto out; 166908ae27ddSKarsten Graul } 167008ae27ddSKarsten Graul /* delete single link */ 167108ae27ddSKarsten Graul lnk_del = NULL; 167208ae27ddSKarsten Graul for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { 167308ae27ddSKarsten Graul if (lgr->lnk[i].link_id == del_llc->link_num) { 167408ae27ddSKarsten Graul lnk_del = &lgr->lnk[i]; 167508ae27ddSKarsten Graul break; 167608ae27ddSKarsten Graul } 167708ae27ddSKarsten Graul } 167808ae27ddSKarsten Graul if (!lnk_del) 167908ae27ddSKarsten Graul goto out; /* asymmetric link already deleted */ 168008ae27ddSKarsten Graul 168108ae27ddSKarsten Graul if (smc_link_downing(&lnk_del->state)) { 1682b7eede75SKarsten Graul if (smc_switch_conns(lgr, lnk_del, false)) 168308ae27ddSKarsten Graul smc_wr_tx_wait_no_pending_sends(lnk_del); 168408ae27ddSKarsten Graul } 168508ae27ddSKarsten Graul if (!list_empty(&lgr->list)) { 168608ae27ddSKarsten Graul /* qentry is either a request from peer (send it back to 168708ae27ddSKarsten Graul * initiate the DELETE_LINK processing), or a locally 168808ae27ddSKarsten Graul * enqueued DELETE_LINK request (forward it) 168908ae27ddSKarsten Graul */ 169008ae27ddSKarsten Graul if (!smc_llc_send_message(lnk, &qentry->msg)) { 169108ae27ddSKarsten Graul struct smc_llc_qentry *qentry2; 169208ae27ddSKarsten Graul 169308ae27ddSKarsten Graul qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, 169408ae27ddSKarsten Graul SMC_LLC_DELETE_LINK); 1695ca7e3edcSYueHaibing if (qentry2) 169608ae27ddSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 169708ae27ddSKarsten Graul } 169808ae27ddSKarsten Graul } 16990a99be43SKarsten Graul smcr_link_clear(lnk_del, true); 170008ae27ddSKarsten Graul 170108ae27ddSKarsten Graul active_links = smc_llc_active_link_count(lgr); 170208ae27ddSKarsten Graul if (active_links == 1) { 1703ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); 170408ae27ddSKarsten Graul } else if (!active_links) { 1705ad6c111bSKarsten Graul smcr_lgr_set_type(lgr, SMC_LGR_NONE); 170608ae27ddSKarsten Graul smc_lgr_terminate_sched(lgr); 170708ae27ddSKarsten Graul } 170808ae27ddSKarsten Graul 170908ae27ddSKarsten Graul if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { 171008ae27ddSKarsten Graul /* trigger setup of asymm alt link */ 1711c48254faSKarsten Graul smc_llc_add_link_local(lnk); 171208ae27ddSKarsten Graul } 171308ae27ddSKarsten Graul out: 1714b5dd4d69SD. Wythe up_write(&lgr->llc_conf_mutex); 171508ae27ddSKarsten Graul kfree(qentry); 171608ae27ddSKarsten Graul } 171708ae27ddSKarsten Graul 17189ec6bf19SKarsten Graul static void smc_llc_delete_link_work(struct work_struct *work) 171952bedf37SKarsten Graul { 17209ec6bf19SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 17219ec6bf19SKarsten Graul llc_del_link_work); 172252bedf37SKarsten Graul 17239ec6bf19SKarsten Graul if (list_empty(&lgr->list)) { 17249ec6bf19SKarsten Graul /* link group is terminating */ 17259ec6bf19SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 17269ec6bf19SKarsten Graul goto out; 172752bedf37SKarsten Graul } 17289c416878SKarsten Graul 17299c416878SKarsten Graul if (lgr->role == SMC_CLNT) 17309c416878SKarsten Graul smc_llc_process_cli_delete_link(lgr); 173108ae27ddSKarsten Graul else 173208ae27ddSKarsten Graul smc_llc_process_srv_delete_link(lgr); 17339ec6bf19SKarsten Graul out: 17349ec6bf19SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); 173552bedf37SKarsten Graul } 173652bedf37SKarsten Graul 17373bc67e09SKarsten Graul /* process a confirm_rkey request from peer, remote flow */ 17383bc67e09SKarsten Graul static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) 17394ed75de5SKarsten Graul { 17403bc67e09SKarsten Graul struct smc_llc_msg_confirm_rkey *llc; 17413bc67e09SKarsten Graul struct smc_llc_qentry *qentry; 17423bc67e09SKarsten Graul struct smc_link *link; 17433bc67e09SKarsten Graul int num_entries; 17443bc67e09SKarsten Graul int rk_idx; 17453bc67e09SKarsten Graul int i; 17464ed75de5SKarsten Graul 17473bc67e09SKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 17483bc67e09SKarsten Graul llc = &qentry->msg.confirm_rkey; 17493bc67e09SKarsten Graul link = qentry->link; 17503bc67e09SKarsten Graul 17513bc67e09SKarsten Graul num_entries = llc->rtoken[0].num_rkeys; 1752b4ba4652SKarsten Graul if (num_entries > SMC_LLC_RKEYS_PER_MSG) 1753b4ba4652SKarsten Graul goto out_err; 17543bc67e09SKarsten Graul /* first rkey entry is for receiving link */ 17553bc67e09SKarsten Graul rk_idx = smc_rtoken_add(link, 17564ed75de5SKarsten Graul llc->rtoken[0].rmb_vaddr, 17574ed75de5SKarsten Graul llc->rtoken[0].rmb_key); 17583bc67e09SKarsten Graul if (rk_idx < 0) 17593bc67e09SKarsten Graul goto out_err; 17604ed75de5SKarsten Graul 17613bc67e09SKarsten Graul for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) 17623bc67e09SKarsten Graul smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, 17633bc67e09SKarsten Graul llc->rtoken[i].rmb_vaddr, 17643bc67e09SKarsten Graul llc->rtoken[i].rmb_key); 17653bc67e09SKarsten Graul /* max links is 3 so there is no need to support conf_rkey_cont msgs */ 17663bc67e09SKarsten Graul goto out; 17673bc67e09SKarsten Graul out_err: 17684ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 17693bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; 17703bc67e09SKarsten Graul out: 17713bc67e09SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 1772b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 17733bc67e09SKarsten Graul smc_llc_send_message(link, &qentry->msg); 17743bc67e09SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 17754ed75de5SKarsten Graul } 17764ed75de5SKarsten Graul 1777218b24feSKarsten Graul /* process a delete_rkey request from peer, remote flow */ 1778218b24feSKarsten Graul static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) 17794ed75de5SKarsten Graul { 1780218b24feSKarsten Graul struct smc_llc_msg_delete_rkey *llc; 1781218b24feSKarsten Graul struct smc_llc_qentry *qentry; 1782218b24feSKarsten Graul struct smc_link *link; 17834ed75de5SKarsten Graul u8 err_mask = 0; 17844ed75de5SKarsten Graul int i, max; 17854ed75de5SKarsten Graul 1786218b24feSKarsten Graul qentry = lgr->llc_flow_rmt.qentry; 1787218b24feSKarsten Graul llc = &qentry->msg.delete_rkey; 1788218b24feSKarsten Graul link = qentry->link; 1789218b24feSKarsten Graul 1790b4ba4652SKarsten Graul if (lgr->smc_version == SMC_V2) { 1791b4ba4652SKarsten Graul struct smc_llc_msg_delete_rkey_v2 *llcv2; 1792b4ba4652SKarsten Graul 1793b4ba4652SKarsten Graul memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc)); 1794b4ba4652SKarsten Graul llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2; 1795b4ba4652SKarsten Graul llcv2->num_inval_rkeys = 0; 1796b4ba4652SKarsten Graul 1797b4ba4652SKarsten Graul max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); 1798b4ba4652SKarsten Graul for (i = 0; i < max; i++) { 1799b4ba4652SKarsten Graul if (smc_rtoken_delete(link, llcv2->rkey[i])) 1800b4ba4652SKarsten Graul llcv2->num_inval_rkeys++; 1801b4ba4652SKarsten Graul } 1802b4ba4652SKarsten Graul memset(&llc->rkey[0], 0, sizeof(llc->rkey)); 1803b4ba4652SKarsten Graul memset(&llc->reserved2, 0, sizeof(llc->reserved2)); 1804b4ba4652SKarsten Graul smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc)); 1805b4ba4652SKarsten Graul if (llcv2->num_inval_rkeys) { 1806b4ba4652SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 1807b4ba4652SKarsten Graul llc->err_mask = llcv2->num_inval_rkeys; 1808b4ba4652SKarsten Graul } 1809b4ba4652SKarsten Graul goto finish; 1810b4ba4652SKarsten Graul } 1811b4ba4652SKarsten Graul 18124ed75de5SKarsten Graul max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); 18134ed75de5SKarsten Graul for (i = 0; i < max; i++) { 1814387707fdSKarsten Graul if (smc_rtoken_delete(link, llc->rkey[i])) 18154ed75de5SKarsten Graul err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); 18164ed75de5SKarsten Graul } 18174ed75de5SKarsten Graul if (err_mask) { 18184ed75de5SKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; 18194ed75de5SKarsten Graul llc->err_mask = err_mask; 18204ed75de5SKarsten Graul } 1821b4ba4652SKarsten Graul finish: 1822218b24feSKarsten Graul llc->hd.flags |= SMC_LLC_FLAG_RESP; 1823218b24feSKarsten Graul smc_llc_send_message(link, &qentry->msg); 1824218b24feSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); 1825218b24feSKarsten Graul } 18264ed75de5SKarsten Graul 18273e0c40afSKarsten Graul static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) 18283e0c40afSKarsten Graul { 1829de2fea7bSTony Lu pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: " 1830de2fea7bSTony Lu "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, 1831de2fea7bSTony Lu lgr->net->net_cookie, type); 18323e0c40afSKarsten Graul smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); 18333e0c40afSKarsten Graul smc_lgr_terminate_sched(lgr); 18343e0c40afSKarsten Graul } 18353e0c40afSKarsten Graul 18366c8968c4SKarsten Graul /* flush the llc event queue */ 183700a049cfSKarsten Graul static void smc_llc_event_flush(struct smc_link_group *lgr) 18389bf9abeaSUrsula Braun { 18396c8968c4SKarsten Graul struct smc_llc_qentry *qentry, *q; 18409bf9abeaSUrsula Braun 18416c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 18426c8968c4SKarsten Graul list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { 18436c8968c4SKarsten Graul list_del_init(&qentry->list); 18446c8968c4SKarsten Graul kfree(qentry); 18456c8968c4SKarsten Graul } 18466c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 18476c8968c4SKarsten Graul } 18486c8968c4SKarsten Graul 18496c8968c4SKarsten Graul static void smc_llc_event_handler(struct smc_llc_qentry *qentry) 18506c8968c4SKarsten Graul { 18516c8968c4SKarsten Graul union smc_llc_msg *llc = &qentry->msg; 18526c8968c4SKarsten Graul struct smc_link *link = qentry->link; 18530fb0b02bSKarsten Graul struct smc_link_group *lgr = link->lgr; 18546c8968c4SKarsten Graul 1855d854fcbfSKarsten Graul if (!smc_link_usable(link)) 18566c8968c4SKarsten Graul goto out; 1857313164daSKarsten Graul 1858b4ba4652SKarsten Graul switch (llc->raw.hdr.common.llc_type) { 1859313164daSKarsten Graul case SMC_LLC_TEST_LINK: 186056e8091cSKarsten Graul llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; 186156e8091cSKarsten Graul smc_llc_send_message(link, llc); 1862313164daSKarsten Graul break; 186352bedf37SKarsten Graul case SMC_LLC_ADD_LINK: 18640fb0b02bSKarsten Graul if (list_empty(&lgr->list)) 18650fb0b02bSKarsten Graul goto out; /* lgr is terminating */ 18660fb0b02bSKarsten Graul if (lgr->role == SMC_CLNT) { 1867c48254faSKarsten Graul if (smc_llc_is_local_add_link(llc)) { 1868c48254faSKarsten Graul if (lgr->llc_flow_lcl.type == 1869c48254faSKarsten Graul SMC_LLC_FLOW_ADD_LINK) 1870c48254faSKarsten Graul break; /* add_link in progress */ 1871c48254faSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_lcl, 1872c48254faSKarsten Graul qentry)) { 1873c48254faSKarsten Graul schedule_work(&lgr->llc_add_link_work); 1874c48254faSKarsten Graul } 1875c48254faSKarsten Graul return; 1876c48254faSKarsten Graul } 1877c48254faSKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 1878c48254faSKarsten Graul !lgr->llc_flow_lcl.qentry) { 18790fb0b02bSKarsten Graul /* a flow is waiting for this message */ 18800fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 18810fb0b02bSKarsten Graul qentry); 18826778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 1883b4ba4652SKarsten Graul return; 1884b4ba4652SKarsten Graul } 1885b4ba4652SKarsten Graul if (lgr->llc_flow_lcl.type == 1886b4ba4652SKarsten Graul SMC_LLC_FLOW_REQ_ADD_LINK) { 1887b4ba4652SKarsten Graul /* server started add_link processing */ 1888b4ba4652SKarsten Graul lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 1889b4ba4652SKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, 1890b4ba4652SKarsten Graul qentry); 1891b4ba4652SKarsten Graul schedule_work(&lgr->llc_add_link_work); 1892b4ba4652SKarsten Graul return; 1893b4ba4652SKarsten Graul } 1894b4ba4652SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1895b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 18960fb0b02bSKarsten Graul } 18970fb0b02bSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 18980fb0b02bSKarsten Graul /* as smc server, handle client suggestion */ 1899b45e7f98SKarsten Graul schedule_work(&lgr->llc_add_link_work); 19000fb0b02bSKarsten Graul } 19010fb0b02bSKarsten Graul return; 19020fb0b02bSKarsten Graul case SMC_LLC_CONFIRM_LINK: 190387f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 19040fb0b02bSKarsten Graul if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { 19050fb0b02bSKarsten Graul /* a flow is waiting for this message */ 19060fb0b02bSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 19076778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 19080fb0b02bSKarsten Graul return; 19090fb0b02bSKarsten Graul } 191052bedf37SKarsten Graul break; 191152bedf37SKarsten Graul case SMC_LLC_DELETE_LINK: 19129ec6bf19SKarsten Graul if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && 19139ec6bf19SKarsten Graul !lgr->llc_flow_lcl.qentry) { 19149ec6bf19SKarsten Graul /* DEL LINK REQ during ADD LINK SEQ */ 1915b9979c2eSKarsten Graul smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); 19166778a6beSKarsten Graul wake_up(&lgr->llc_msg_waiter); 1917b9979c2eSKarsten Graul } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 19189ec6bf19SKarsten Graul schedule_work(&lgr->llc_del_link_work); 19199ec6bf19SKarsten Graul } 19209ec6bf19SKarsten Graul return; 19214ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY: 19223bc67e09SKarsten Graul /* new request from remote, assign to remote flow */ 19233bc67e09SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 19243bc67e09SKarsten Graul /* process here, does not wait for more llc msgs */ 19253bc67e09SKarsten Graul smc_llc_rmt_conf_rkey(lgr); 19263bc67e09SKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 19273bc67e09SKarsten Graul } 19283bc67e09SKarsten Graul return; 19294ed75de5SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 193042d18accSKarsten Graul /* not used because max links is 3, and 3 rkeys fit into 193142d18accSKarsten Graul * one CONFIRM_RKEY message 193242d18accSKarsten Graul */ 19334ed75de5SKarsten Graul break; 19344ed75de5SKarsten Graul case SMC_LLC_DELETE_RKEY: 1935218b24feSKarsten Graul /* new request from remote, assign to remote flow */ 1936218b24feSKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { 1937218b24feSKarsten Graul /* process here, does not wait for more llc msgs */ 1938218b24feSKarsten Graul smc_llc_rmt_delete_rkey(lgr); 1939218b24feSKarsten Graul smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); 1940218b24feSKarsten Graul } 1941218b24feSKarsten Graul return; 1942b4ba4652SKarsten Graul case SMC_LLC_REQ_ADD_LINK: 1943b4ba4652SKarsten Graul /* handle response here, smc_llc_flow_stop() cannot be called 1944b4ba4652SKarsten Graul * in tasklet context 1945b4ba4652SKarsten Graul */ 1946b4ba4652SKarsten Graul if (lgr->role == SMC_CLNT && 1947b4ba4652SKarsten Graul lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK && 1948b4ba4652SKarsten Graul (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) { 1949b4ba4652SKarsten Graul smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl); 1950b4ba4652SKarsten Graul } else if (lgr->role == SMC_SERV) { 1951b4ba4652SKarsten Graul if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { 1952b4ba4652SKarsten Graul /* as smc server, handle client suggestion */ 1953b4ba4652SKarsten Graul lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; 1954b4ba4652SKarsten Graul schedule_work(&lgr->llc_add_link_work); 1955b4ba4652SKarsten Graul } 1956b4ba4652SKarsten Graul return; 1957b4ba4652SKarsten Graul } 1958b4ba4652SKarsten Graul break; 19593e0c40afSKarsten Graul default: 19603e0c40afSKarsten Graul smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); 19613e0c40afSKarsten Graul break; 1962313164daSKarsten Graul } 19636c8968c4SKarsten Graul out: 19646c8968c4SKarsten Graul kfree(qentry); 19656c8968c4SKarsten Graul } 19666c8968c4SKarsten Graul 19676c8968c4SKarsten Graul /* worker to process llc messages on the event queue */ 19686c8968c4SKarsten Graul static void smc_llc_event_work(struct work_struct *work) 19696c8968c4SKarsten Graul { 19706c8968c4SKarsten Graul struct smc_link_group *lgr = container_of(work, struct smc_link_group, 19716c8968c4SKarsten Graul llc_event_work); 19726c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 19736c8968c4SKarsten Graul 1974555da9afSKarsten Graul if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { 1975555da9afSKarsten Graul qentry = lgr->delayed_event; 1976555da9afSKarsten Graul lgr->delayed_event = NULL; 1977d535ca13SKarsten Graul if (smc_link_usable(qentry->link)) 1978d535ca13SKarsten Graul smc_llc_event_handler(qentry); 1979d535ca13SKarsten Graul else 1980555da9afSKarsten Graul kfree(qentry); 1981555da9afSKarsten Graul } 1982555da9afSKarsten Graul 19836c8968c4SKarsten Graul again: 19846c8968c4SKarsten Graul spin_lock_bh(&lgr->llc_event_q_lock); 19856c8968c4SKarsten Graul if (!list_empty(&lgr->llc_event_q)) { 19866c8968c4SKarsten Graul qentry = list_first_entry(&lgr->llc_event_q, 19876c8968c4SKarsten Graul struct smc_llc_qentry, list); 19886c8968c4SKarsten Graul list_del_init(&qentry->list); 19896c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 19906c8968c4SKarsten Graul smc_llc_event_handler(qentry); 19916c8968c4SKarsten Graul goto again; 19926c8968c4SKarsten Graul } 19936c8968c4SKarsten Graul spin_unlock_bh(&lgr->llc_event_q_lock); 19946c8968c4SKarsten Graul } 19956c8968c4SKarsten Graul 1996ef79d439SKarsten Graul /* process llc responses in tasklet context */ 1997a6688d91SKarsten Graul static void smc_llc_rx_response(struct smc_link *link, 1998a6688d91SKarsten Graul struct smc_llc_qentry *qentry) 1999ef79d439SKarsten Graul { 20002ff08678SKarsten Graul enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type; 20012ff08678SKarsten Graul struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl; 2002b4ba4652SKarsten Graul u8 llc_type = qentry->msg.raw.hdr.common.llc_type; 2003ef79d439SKarsten Graul 2004a6688d91SKarsten Graul switch (llc_type) { 2005ef79d439SKarsten Graul case SMC_LLC_TEST_LINK: 2006741a49a4SKarsten Graul if (smc_link_active(link)) 2007ef79d439SKarsten Graul complete(&link->llc_testlink_resp); 2008ef79d439SKarsten Graul break; 2009ef79d439SKarsten Graul case SMC_LLC_ADD_LINK: 201087f88cdaSKarsten Graul case SMC_LLC_ADD_LINK_CONT: 20112ff08678SKarsten Graul case SMC_LLC_CONFIRM_LINK: 20122ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry) 20132ff08678SKarsten Graul break; /* drop out-of-flow response */ 20142ff08678SKarsten Graul goto assign; 20152ff08678SKarsten Graul case SMC_LLC_DELETE_LINK: 20162ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry) 20172ff08678SKarsten Graul break; /* drop out-of-flow response */ 20182ff08678SKarsten Graul goto assign; 20193d88a21bSKarsten Graul case SMC_LLC_CONFIRM_RKEY: 20206d74c3a8SKarsten Graul case SMC_LLC_DELETE_RKEY: 20212ff08678SKarsten Graul if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry) 20222ff08678SKarsten Graul break; /* drop out-of-flow response */ 20232ff08678SKarsten Graul goto assign; 2024ef79d439SKarsten Graul case SMC_LLC_CONFIRM_RKEY_CONT: 202542d18accSKarsten Graul /* not used because max links is 3 */ 2026ef79d439SKarsten Graul break; 20273e0c40afSKarsten Graul default: 2028b4ba4652SKarsten Graul smc_llc_protocol_violation(link->lgr, 2029b4ba4652SKarsten Graul qentry->msg.raw.hdr.common.type); 20303e0c40afSKarsten Graul break; 2031ef79d439SKarsten Graul } 2032a6688d91SKarsten Graul kfree(qentry); 20332ff08678SKarsten Graul return; 20342ff08678SKarsten Graul assign: 20352ff08678SKarsten Graul /* assign responses to the local flow, we requested them */ 20362ff08678SKarsten Graul smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); 20372ff08678SKarsten Graul wake_up(&link->lgr->llc_msg_waiter); 2038ef79d439SKarsten Graul } 2039ef79d439SKarsten Graul 2040a6688d91SKarsten Graul static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) 20416c8968c4SKarsten Graul { 20426c8968c4SKarsten Graul struct smc_link_group *lgr = link->lgr; 20436c8968c4SKarsten Graul struct smc_llc_qentry *qentry; 20446c8968c4SKarsten Graul unsigned long flags; 20456c8968c4SKarsten Graul 20466c8968c4SKarsten Graul qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); 20476c8968c4SKarsten Graul if (!qentry) 20486c8968c4SKarsten Graul return; 20496c8968c4SKarsten Graul qentry->link = link; 20506c8968c4SKarsten Graul INIT_LIST_HEAD(&qentry->list); 20516c8968c4SKarsten Graul memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); 2052a6688d91SKarsten Graul 2053a6688d91SKarsten Graul /* process responses immediately */ 2054b4ba4652SKarsten Graul if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && 2055b4ba4652SKarsten Graul llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) { 2056a6688d91SKarsten Graul smc_llc_rx_response(link, qentry); 2057a6688d91SKarsten Graul return; 2058a6688d91SKarsten Graul } 2059a6688d91SKarsten Graul 2060a6688d91SKarsten Graul /* add requests to event queue */ 20616c8968c4SKarsten Graul spin_lock_irqsave(&lgr->llc_event_q_lock, flags); 20626c8968c4SKarsten Graul list_add_tail(&qentry->list, &lgr->llc_event_q); 20636c8968c4SKarsten Graul spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); 206422ef473dSKarsten Graul queue_work(system_highpri_wq, &lgr->llc_event_work); 20659bf9abeaSUrsula Braun } 20669bf9abeaSUrsula Braun 2067a6688d91SKarsten Graul /* copy received msg and add it to the event queue */ 2068a6688d91SKarsten Graul static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) 2069a6688d91SKarsten Graul { 2070a6688d91SKarsten Graul struct smc_link *link = (struct smc_link *)wc->qp->qp_context; 2071a6688d91SKarsten Graul union smc_llc_msg *llc = buf; 2072a6688d91SKarsten Graul 2073a6688d91SKarsten Graul if (wc->byte_len < sizeof(*llc)) 2074a6688d91SKarsten Graul return; /* short message */ 2075b4ba4652SKarsten Graul if (!llc->raw.hdr.common.llc_version) { 2076a6688d91SKarsten Graul if (llc->raw.hdr.length != sizeof(*llc)) 2077a6688d91SKarsten Graul return; /* invalid message */ 2078b4ba4652SKarsten Graul } else { 2079b4ba4652SKarsten Graul if (llc->raw.hdr.length_v2 < sizeof(*llc)) 2080b4ba4652SKarsten Graul return; /* invalid message */ 2081b4ba4652SKarsten Graul } 2082a6688d91SKarsten Graul 2083a6688d91SKarsten Graul smc_llc_enqueue(link, llc); 2084a6688d91SKarsten Graul } 2085a6688d91SKarsten Graul 208644aa81ceSKarsten Graul /***************************** worker, utils *********************************/ 2087877ae5beSKarsten Graul 2088877ae5beSKarsten Graul static void smc_llc_testlink_work(struct work_struct *work) 2089877ae5beSKarsten Graul { 2090877ae5beSKarsten Graul struct smc_link *link = container_of(to_delayed_work(work), 2091877ae5beSKarsten Graul struct smc_link, llc_testlink_wrk); 2092877ae5beSKarsten Graul unsigned long next_interval; 2093877ae5beSKarsten Graul unsigned long expire_time; 2094877ae5beSKarsten Graul u8 user_data[16] = { 0 }; 2095877ae5beSKarsten Graul int rc; 2096877ae5beSKarsten Graul 2097741a49a4SKarsten Graul if (!smc_link_active(link)) 2098877ae5beSKarsten Graul return; /* don't reschedule worker */ 2099877ae5beSKarsten Graul expire_time = link->wr_rx_tstamp + link->llc_testlink_time; 2100877ae5beSKarsten Graul if (time_is_after_jiffies(expire_time)) { 2101877ae5beSKarsten Graul next_interval = expire_time - jiffies; 2102877ae5beSKarsten Graul goto out; 2103877ae5beSKarsten Graul } 2104877ae5beSKarsten Graul reinit_completion(&link->llc_testlink_resp); 2105d97935faSKarsten Graul smc_llc_send_test_link(link, user_data); 2106877ae5beSKarsten Graul /* receive TEST LINK response over RoCE fabric */ 2107877ae5beSKarsten Graul rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, 2108877ae5beSKarsten Graul SMC_LLC_WAIT_TIME); 2109741a49a4SKarsten Graul if (!smc_link_active(link)) 21101020e1efSKarsten Graul return; /* link state changed */ 2111877ae5beSKarsten Graul if (rc <= 0) { 211287523930SKarsten Graul smcr_link_down_cond_sched(link); 2113877ae5beSKarsten Graul return; 2114877ae5beSKarsten Graul } 2115877ae5beSKarsten Graul next_interval = link->llc_testlink_time; 2116877ae5beSKarsten Graul out: 21171020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, next_interval); 2118877ae5beSKarsten Graul } 2119877ae5beSKarsten Graul 212000a049cfSKarsten Graul void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) 212100a049cfSKarsten Graul { 212200a049cfSKarsten Graul struct net *net = sock_net(smc->clcsock->sk); 212300a049cfSKarsten Graul 212400a049cfSKarsten Graul INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); 2125b45e7f98SKarsten Graul INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); 21269ec6bf19SKarsten Graul INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); 212700a049cfSKarsten Graul INIT_LIST_HEAD(&lgr->llc_event_q); 212800a049cfSKarsten Graul spin_lock_init(&lgr->llc_event_q_lock); 2129555da9afSKarsten Graul spin_lock_init(&lgr->llc_flow_lock); 21306778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_flow_waiter); 21316778a6beSKarsten Graul init_waitqueue_head(&lgr->llc_msg_waiter); 2132b5dd4d69SD. Wythe init_rwsem(&lgr->llc_conf_mutex); 213377eee325SWen Gu lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time); 213400a049cfSKarsten Graul } 213500a049cfSKarsten Graul 213600a049cfSKarsten Graul /* called after lgr was removed from lgr_list */ 213700a049cfSKarsten Graul void smc_llc_lgr_clear(struct smc_link_group *lgr) 213800a049cfSKarsten Graul { 213900a049cfSKarsten Graul smc_llc_event_flush(lgr); 21406778a6beSKarsten Graul wake_up_all(&lgr->llc_flow_waiter); 21416778a6beSKarsten Graul wake_up_all(&lgr->llc_msg_waiter); 214200a049cfSKarsten Graul cancel_work_sync(&lgr->llc_event_work); 2143b45e7f98SKarsten Graul cancel_work_sync(&lgr->llc_add_link_work); 21449ec6bf19SKarsten Graul cancel_work_sync(&lgr->llc_del_link_work); 2145555da9afSKarsten Graul if (lgr->delayed_event) { 2146555da9afSKarsten Graul kfree(lgr->delayed_event); 2147555da9afSKarsten Graul lgr->delayed_event = NULL; 2148555da9afSKarsten Graul } 214900a049cfSKarsten Graul } 215000a049cfSKarsten Graul 21512a4c57a9SKarsten Graul int smc_llc_link_init(struct smc_link *link) 2152877ae5beSKarsten Graul { 2153877ae5beSKarsten Graul init_completion(&link->llc_testlink_resp); 2154877ae5beSKarsten Graul INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); 21552a4c57a9SKarsten Graul return 0; 2156b32cf4abSKarsten Graul } 2157b32cf4abSKarsten Graul 215800a049cfSKarsten Graul void smc_llc_link_active(struct smc_link *link) 2159b32cf4abSKarsten Graul { 2160de2fea7bSTony Lu pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, " 21610a99be43SKarsten Graul "peerid %*phN, ibdev %s, ibport %d\n", 21620a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 2163de2fea7bSTony Lu link->lgr->net->net_cookie, 21640a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 21650a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 21660a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 2167877ae5beSKarsten Graul link->state = SMC_LNK_ACTIVE; 216800a049cfSKarsten Graul if (link->lgr->llc_testlink_time) { 2169c4a146c7STony Lu link->llc_testlink_time = link->lgr->llc_testlink_time; 21701020e1efSKarsten Graul schedule_delayed_work(&link->llc_testlink_wrk, 2171877ae5beSKarsten Graul link->llc_testlink_time); 2172877ae5beSKarsten Graul } 2173877ae5beSKarsten Graul } 2174877ae5beSKarsten Graul 2175877ae5beSKarsten Graul /* called in worker context */ 21760a99be43SKarsten Graul void smc_llc_link_clear(struct smc_link *link, bool log) 2177877ae5beSKarsten Graul { 21780a99be43SKarsten Graul if (log) 2179de2fea7bSTony Lu pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN" 21800a99be43SKarsten Graul ", peerid %*phN, ibdev %s, ibport %d\n", 21810a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->lgr->id, 2182de2fea7bSTony Lu link->lgr->net->net_cookie, 21830a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->link_uid, 21840a99be43SKarsten Graul SMC_LGR_ID_SIZE, &link->peer_link_uid, 21850a99be43SKarsten Graul link->smcibdev->ibdev->name, link->ibport); 21862140ac26SKarsten Graul complete(&link->llc_testlink_resp); 21872140ac26SKarsten Graul cancel_delayed_work_sync(&link->llc_testlink_wrk); 2188877ae5beSKarsten Graul } 2189877ae5beSKarsten Graul 21903d88a21bSKarsten Graul /* register a new rtoken at the remote peer (for all links) */ 21913d88a21bSKarsten Graul int smc_llc_do_confirm_rkey(struct smc_link *send_link, 219244aa81ceSKarsten Graul struct smc_buf_desc *rmb_desc) 219344aa81ceSKarsten Graul { 21943d88a21bSKarsten Graul struct smc_link_group *lgr = send_link->lgr; 21953d88a21bSKarsten Graul struct smc_llc_qentry *qentry = NULL; 21963d88a21bSKarsten Graul int rc = 0; 219744aa81ceSKarsten Graul 21983d88a21bSKarsten Graul rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); 21993d88a21bSKarsten Graul if (rc) 22003d88a21bSKarsten Graul goto out; 220144aa81ceSKarsten Graul /* receive CONFIRM RKEY response from server over RoCE fabric */ 22023d88a21bSKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 22033d88a21bSKarsten Graul SMC_LLC_CONFIRM_RKEY); 22043d88a21bSKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 22053d88a21bSKarsten Graul rc = -EFAULT; 22063d88a21bSKarsten Graul out: 22073d88a21bSKarsten Graul if (qentry) 22083d88a21bSKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 22093d88a21bSKarsten Graul return rc; 221044aa81ceSKarsten Graul } 221144aa81ceSKarsten Graul 221260e03c62SKarsten Graul /* unregister an rtoken at the remote peer */ 22136d74c3a8SKarsten Graul int smc_llc_do_delete_rkey(struct smc_link_group *lgr, 221460e03c62SKarsten Graul struct smc_buf_desc *rmb_desc) 221560e03c62SKarsten Graul { 22166d74c3a8SKarsten Graul struct smc_llc_qentry *qentry = NULL; 22176d74c3a8SKarsten Graul struct smc_link *send_link; 22180b29ec64SUrsula Braun int rc = 0; 221960e03c62SKarsten Graul 22206d74c3a8SKarsten Graul send_link = smc_llc_usable_link(lgr); 22216d74c3a8SKarsten Graul if (!send_link) 22226d74c3a8SKarsten Graul return -ENOLINK; 22236d74c3a8SKarsten Graul 22246d74c3a8SKarsten Graul /* protected by llc_flow control */ 22256d74c3a8SKarsten Graul rc = smc_llc_send_delete_rkey(send_link, rmb_desc); 222660e03c62SKarsten Graul if (rc) 222760e03c62SKarsten Graul goto out; 222860e03c62SKarsten Graul /* receive DELETE RKEY response from server over RoCE fabric */ 22296d74c3a8SKarsten Graul qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, 22306d74c3a8SKarsten Graul SMC_LLC_DELETE_RKEY); 22316d74c3a8SKarsten Graul if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) 223260e03c62SKarsten Graul rc = -EFAULT; 223360e03c62SKarsten Graul out: 22346d74c3a8SKarsten Graul if (qentry) 22356d74c3a8SKarsten Graul smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); 223660e03c62SKarsten Graul return rc; 223760e03c62SKarsten Graul } 223860e03c62SKarsten Graul 223945fa8da0SKarsten Graul void smc_llc_link_set_uid(struct smc_link *link) 224045fa8da0SKarsten Graul { 224145fa8da0SKarsten Graul __be32 link_uid; 224245fa8da0SKarsten Graul 224345fa8da0SKarsten Graul link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); 224445fa8da0SKarsten Graul memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); 224545fa8da0SKarsten Graul } 224645fa8da0SKarsten Graul 2247649758ffSKarsten Graul /* save peers link user id, used for debug purposes */ 2248649758ffSKarsten Graul void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) 2249649758ffSKarsten Graul { 2250649758ffSKarsten Graul memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, 2251649758ffSKarsten Graul SMC_LGR_ID_SIZE); 2252649758ffSKarsten Graul } 2253649758ffSKarsten Graul 225492334cfcSKarsten Graul /* evaluate confirm link request or response */ 225592334cfcSKarsten Graul int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, 225692334cfcSKarsten Graul enum smc_llc_reqresp type) 225792334cfcSKarsten Graul { 225845fa8da0SKarsten Graul if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ 225992334cfcSKarsten Graul qentry->link->link_id = qentry->msg.confirm_link.link_num; 226045fa8da0SKarsten Graul smc_llc_link_set_uid(qentry->link); 226145fa8da0SKarsten Graul } 226292334cfcSKarsten Graul if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) 226392334cfcSKarsten Graul return -ENOTSUPP; 226492334cfcSKarsten Graul return 0; 226592334cfcSKarsten Graul } 226692334cfcSKarsten Graul 22679bf9abeaSUrsula Braun /***************************** init, exit, misc ******************************/ 22689bf9abeaSUrsula Braun 22699bf9abeaSUrsula Braun static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { 22709bf9abeaSUrsula Braun { 22719bf9abeaSUrsula Braun .handler = smc_llc_rx_handler, 22729bf9abeaSUrsula Braun .type = SMC_LLC_CONFIRM_LINK 22739bf9abeaSUrsula Braun }, 22749bf9abeaSUrsula Braun { 2275313164daSKarsten Graul .handler = smc_llc_rx_handler, 2276313164daSKarsten Graul .type = SMC_LLC_TEST_LINK 2277313164daSKarsten Graul }, 2278313164daSKarsten Graul { 22794ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 228052bedf37SKarsten Graul .type = SMC_LLC_ADD_LINK 228152bedf37SKarsten Graul }, 228252bedf37SKarsten Graul { 228352bedf37SKarsten Graul .handler = smc_llc_rx_handler, 228487f88cdaSKarsten Graul .type = SMC_LLC_ADD_LINK_CONT 228587f88cdaSKarsten Graul }, 228687f88cdaSKarsten Graul { 228787f88cdaSKarsten Graul .handler = smc_llc_rx_handler, 228852bedf37SKarsten Graul .type = SMC_LLC_DELETE_LINK 228952bedf37SKarsten Graul }, 229052bedf37SKarsten Graul { 229152bedf37SKarsten Graul .handler = smc_llc_rx_handler, 22924ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY 22934ed75de5SKarsten Graul }, 22944ed75de5SKarsten Graul { 22954ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 22964ed75de5SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_CONT 22974ed75de5SKarsten Graul }, 22984ed75de5SKarsten Graul { 22994ed75de5SKarsten Graul .handler = smc_llc_rx_handler, 23004ed75de5SKarsten Graul .type = SMC_LLC_DELETE_RKEY 23014ed75de5SKarsten Graul }, 2302b4ba4652SKarsten Graul /* V2 types */ 2303b4ba4652SKarsten Graul { 2304b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2305b4ba4652SKarsten Graul .type = SMC_LLC_CONFIRM_LINK_V2 2306b4ba4652SKarsten Graul }, 2307b4ba4652SKarsten Graul { 2308b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2309b4ba4652SKarsten Graul .type = SMC_LLC_TEST_LINK_V2 2310b4ba4652SKarsten Graul }, 2311b4ba4652SKarsten Graul { 2312b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2313b4ba4652SKarsten Graul .type = SMC_LLC_ADD_LINK_V2 2314b4ba4652SKarsten Graul }, 2315b4ba4652SKarsten Graul { 2316b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2317b4ba4652SKarsten Graul .type = SMC_LLC_DELETE_LINK_V2 2318b4ba4652SKarsten Graul }, 2319b4ba4652SKarsten Graul { 2320b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2321b4ba4652SKarsten Graul .type = SMC_LLC_REQ_ADD_LINK_V2 2322b4ba4652SKarsten Graul }, 2323b4ba4652SKarsten Graul { 2324b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2325b4ba4652SKarsten Graul .type = SMC_LLC_CONFIRM_RKEY_V2 2326b4ba4652SKarsten Graul }, 2327b4ba4652SKarsten Graul { 2328b4ba4652SKarsten Graul .handler = smc_llc_rx_handler, 2329b4ba4652SKarsten Graul .type = SMC_LLC_DELETE_RKEY_V2 2330b4ba4652SKarsten Graul }, 23314ed75de5SKarsten Graul { 23329bf9abeaSUrsula Braun .handler = NULL, 23339bf9abeaSUrsula Braun } 23349bf9abeaSUrsula Braun }; 23359bf9abeaSUrsula Braun 23369bf9abeaSUrsula Braun int __init smc_llc_init(void) 23379bf9abeaSUrsula Braun { 23389bf9abeaSUrsula Braun struct smc_wr_rx_handler *handler; 23399bf9abeaSUrsula Braun int rc = 0; 23409bf9abeaSUrsula Braun 23419bf9abeaSUrsula Braun for (handler = smc_llc_rx_handlers; handler->handler; handler++) { 23429bf9abeaSUrsula Braun INIT_HLIST_NODE(&handler->list); 23439bf9abeaSUrsula Braun rc = smc_wr_rx_register_handler(handler); 23449bf9abeaSUrsula Braun if (rc) 23459bf9abeaSUrsula Braun break; 23469bf9abeaSUrsula Braun } 23479bf9abeaSUrsula Braun return rc; 23489bf9abeaSUrsula Braun } 2349