1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13 #include <net/tcp.h>
14 #include <rdma/ib_verbs.h>
15
16 #include "smc.h"
17 #include "smc_core.h"
18 #include "smc_clc.h"
19 #include "smc_llc.h"
20 #include "smc_pnet.h"
21
22 #define SMC_LLC_DATA_LEN 40
23
24 struct smc_llc_hdr {
25 struct smc_wr_rx_hdr common;
26 union {
27 struct {
28 u8 length; /* 44 */
29 #if defined(__BIG_ENDIAN_BITFIELD)
30 u8 reserved:4,
31 add_link_rej_rsn:4;
32 #elif defined(__LITTLE_ENDIAN_BITFIELD)
33 u8 add_link_rej_rsn:4,
34 reserved:4;
35 #endif
36 };
37 u16 length_v2; /* 44 - 8192*/
38 };
39 u8 flags;
40 } __packed; /* format defined in
41 * IBM Shared Memory Communications Version 2
42 * (https://www.ibm.com/support/pages/node/6326337)
43 */
44
45 #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
46
47 struct smc_llc_msg_confirm_link { /* type 0x01 */
48 struct smc_llc_hdr hd;
49 u8 sender_mac[ETH_ALEN];
50 u8 sender_gid[SMC_GID_SIZE];
51 u8 sender_qp_num[3];
52 u8 link_num;
53 u8 link_uid[SMC_LGR_ID_SIZE];
54 u8 max_links;
55 u8 max_conns;
56 u8 reserved[8];
57 };
58
59 #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
60 #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
61
62 struct smc_llc_msg_add_link { /* type 0x02 */
63 struct smc_llc_hdr hd;
64 u8 sender_mac[ETH_ALEN];
65 u8 reserved2[2];
66 u8 sender_gid[SMC_GID_SIZE];
67 u8 sender_qp_num[3];
68 u8 link_num;
69 #if defined(__BIG_ENDIAN_BITFIELD)
70 u8 reserved3 : 4,
71 qp_mtu : 4;
72 #elif defined(__LITTLE_ENDIAN_BITFIELD)
73 u8 qp_mtu : 4,
74 reserved3 : 4;
75 #endif
76 u8 initial_psn[3];
77 u8 reserved[8];
78 };
79
80 struct smc_llc_msg_add_link_cont_rt {
81 __be32 rmb_key;
82 __be32 rmb_key_new;
83 __be64 rmb_vaddr_new;
84 };
85
86 struct smc_llc_msg_add_link_v2_ext {
87 #if defined(__BIG_ENDIAN_BITFIELD)
88 u8 v2_direct : 1,
89 reserved : 7;
90 #elif defined(__LITTLE_ENDIAN_BITFIELD)
91 u8 reserved : 7,
92 v2_direct : 1;
93 #endif
94 u8 reserved2;
95 u8 client_target_gid[SMC_GID_SIZE];
96 u8 reserved3[8];
97 u16 num_rkeys;
98 struct smc_llc_msg_add_link_cont_rt rt[];
99 } __packed; /* format defined in
100 * IBM Shared Memory Communications Version 2
101 * (https://www.ibm.com/support/pages/node/6326337)
102 */
103
104 struct smc_llc_msg_req_add_link_v2 {
105 struct smc_llc_hdr hd;
106 u8 reserved[20];
107 u8 gid_cnt;
108 u8 reserved2[3];
109 u8 gid[][SMC_GID_SIZE];
110 };
111
112 #define SMC_LLC_RKEYS_PER_CONT_MSG 2
113
114 struct smc_llc_msg_add_link_cont { /* type 0x03 */
115 struct smc_llc_hdr hd;
116 u8 link_num;
117 u8 num_rkeys;
118 u8 reserved2[2];
119 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
120 u8 reserved[4];
121 } __packed; /* format defined in RFC7609 */
122
123 #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
124 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
125
126 struct smc_llc_msg_del_link { /* type 0x04 */
127 struct smc_llc_hdr hd;
128 u8 link_num;
129 __be32 reason;
130 u8 reserved[35];
131 } __packed; /* format defined in RFC7609 */
132
133 struct smc_llc_msg_test_link { /* type 0x07 */
134 struct smc_llc_hdr hd;
135 u8 user_data[16];
136 u8 reserved[24];
137 };
138
139 struct smc_rmb_rtoken {
140 union {
141 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
142 /* is actually the num of rtokens, first */
143 /* rtoken is always for the current link */
144 u8 link_id; /* link id of the rtoken */
145 };
146 __be32 rmb_key;
147 __be64 rmb_vaddr;
148 } __packed; /* format defined in RFC7609 */
149
150 #define SMC_LLC_RKEYS_PER_MSG 3
151 #define SMC_LLC_RKEYS_PER_MSG_V2 255
152
153 struct smc_llc_msg_confirm_rkey { /* type 0x06 */
154 struct smc_llc_hdr hd;
155 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
156 u8 reserved;
157 };
158
159 #define SMC_LLC_DEL_RKEY_MAX 8
160 #define SMC_LLC_DEL_RKEY_V2_INLINE 9
161 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
162 #define SMC_LLC_FLAG_RKEY_NEG 0x20
163
164 struct smc_llc_msg_delete_rkey { /* type 0x09 */
165 struct smc_llc_hdr hd;
166 u8 num_rkeys;
167 u8 err_mask;
168 u8 reserved[2];
169 __be32 rkey[8];
170 u8 reserved2[4];
171 };
172
173 struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */
174 struct smc_llc_hdr hd;
175 u8 num_rkeys;
176 u8 num_inval_rkeys;
177 u8 reserved[2];
178 __be32 rkey[];
179 };
180
181 /* the leading rkeys of a DELETE_RKEY_V2 fit into union smc_llc_msg */
182 struct smc_llc_msg_delete_rkey_v2_inline { /* type 0x29 */
183 struct smc_llc_hdr hd;
184 u8 num_rkeys;
185 u8 num_inval_rkeys;
186 u8 reserved[2];
187 __be32 rkey[SMC_LLC_DEL_RKEY_V2_INLINE];
188 };
189
190 union smc_llc_msg {
191 struct smc_llc_msg_confirm_link confirm_link;
192 struct smc_llc_msg_add_link add_link;
193 struct smc_llc_msg_req_add_link_v2 req_add_link;
194 struct smc_llc_msg_add_link_cont add_link_cont;
195 struct smc_llc_msg_del_link delete_link;
196
197 struct smc_llc_msg_confirm_rkey confirm_rkey;
198 struct smc_llc_msg_delete_rkey delete_rkey;
199 struct smc_llc_msg_delete_rkey_v2_inline delete_rkey_v2;
200
201 struct smc_llc_msg_test_link test_link;
202 struct {
203 struct smc_llc_hdr hdr;
204 u8 data[SMC_LLC_DATA_LEN];
205 } raw;
206 };
207
208 static_assert(SMC_LLC_DEL_RKEY_V2_INLINE ==
209 (sizeof(union smc_llc_msg) -
210 offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)) /
211 sizeof(__be32));
212 static_assert(offsetof(struct smc_llc_msg_delete_rkey_v2_inline, rkey) ==
213 offsetof(struct smc_llc_msg_delete_rkey_v2, rkey));
214
215 #define SMC_LLC_FLAG_RESP 0x80
216
217 struct smc_llc_qentry {
218 struct list_head list;
219 struct smc_link *link;
220 u16 body_len;
221 union smc_llc_msg msg;
222 u8 body[] __counted_by(body_len);
223 };
224
225 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc,
226 u32 byte_len);
227
smc_llc_flow_qentry_clr(struct smc_llc_flow * flow)228 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
229 {
230 struct smc_llc_qentry *qentry = flow->qentry;
231
232 flow->qentry = NULL;
233 return qentry;
234 }
235
smc_llc_flow_qentry_del(struct smc_llc_flow * flow)236 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
237 {
238 struct smc_llc_qentry *qentry;
239
240 if (flow->qentry) {
241 qentry = flow->qentry;
242 flow->qentry = NULL;
243 kfree(qentry);
244 }
245 }
246
smc_llc_flow_qentry_set(struct smc_llc_flow * flow,struct smc_llc_qentry * qentry)247 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
248 struct smc_llc_qentry *qentry)
249 {
250 flow->qentry = qentry;
251 }
252
smc_llc_flow_parallel(struct smc_link_group * lgr,u8 flow_type,struct smc_llc_qentry * qentry)253 static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
254 struct smc_llc_qentry *qentry)
255 {
256 u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
257
258 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
259 flow_type != msg_type && !lgr->delayed_event) {
260 lgr->delayed_event = qentry;
261 return;
262 }
263 /* drop parallel or already-in-progress llc requests */
264 if (flow_type != msg_type)
265 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
266 "LLC msg: msg %d flow %d role %d\n",
267 SMC_LGR_ID_SIZE, &lgr->id,
268 lgr->net->net_cookie,
269 qentry->msg.raw.hdr.common.type,
270 flow_type, lgr->role);
271 kfree(qentry);
272 }
273
274 /* try to start a new llc flow, initiated by an incoming llc msg */
smc_llc_flow_start(struct smc_llc_flow * flow,struct smc_llc_qentry * qentry)275 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
276 struct smc_llc_qentry *qentry)
277 {
278 struct smc_link_group *lgr = qentry->link->lgr;
279
280 spin_lock_bh(&lgr->llc_flow_lock);
281 if (flow->type) {
282 /* a flow is already active */
283 smc_llc_flow_parallel(lgr, flow->type, qentry);
284 spin_unlock_bh(&lgr->llc_flow_lock);
285 return false;
286 }
287 switch (qentry->msg.raw.hdr.common.llc_type) {
288 case SMC_LLC_ADD_LINK:
289 flow->type = SMC_LLC_FLOW_ADD_LINK;
290 break;
291 case SMC_LLC_DELETE_LINK:
292 flow->type = SMC_LLC_FLOW_DEL_LINK;
293 break;
294 case SMC_LLC_CONFIRM_RKEY:
295 case SMC_LLC_DELETE_RKEY:
296 flow->type = SMC_LLC_FLOW_RKEY;
297 break;
298 default:
299 flow->type = SMC_LLC_FLOW_NONE;
300 }
301 smc_llc_flow_qentry_set(flow, qentry);
302 spin_unlock_bh(&lgr->llc_flow_lock);
303 return true;
304 }
305
306 /* start a new local llc flow, wait till current flow finished */
smc_llc_flow_initiate(struct smc_link_group * lgr,enum smc_llc_flowtype type)307 int smc_llc_flow_initiate(struct smc_link_group *lgr,
308 enum smc_llc_flowtype type)
309 {
310 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
311 int rc;
312
313 /* all flows except confirm_rkey and delete_rkey are exclusive,
314 * confirm/delete rkey flows can run concurrently (local and remote)
315 */
316 if (type == SMC_LLC_FLOW_RKEY)
317 allowed_remote = SMC_LLC_FLOW_RKEY;
318 again:
319 if (list_empty(&lgr->list))
320 return -ENODEV;
321 spin_lock_bh(&lgr->llc_flow_lock);
322 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
323 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
324 lgr->llc_flow_rmt.type == allowed_remote)) {
325 lgr->llc_flow_lcl.type = type;
326 spin_unlock_bh(&lgr->llc_flow_lock);
327 return 0;
328 }
329 spin_unlock_bh(&lgr->llc_flow_lock);
330 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
331 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
332 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
333 lgr->llc_flow_rmt.type == allowed_remote))),
334 SMC_LLC_WAIT_TIME * 10);
335 if (!rc)
336 return -ETIMEDOUT;
337 goto again;
338 }
339
340 /* finish the current llc flow */
smc_llc_flow_stop(struct smc_link_group * lgr,struct smc_llc_flow * flow)341 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
342 {
343 spin_lock_bh(&lgr->llc_flow_lock);
344 smc_llc_flow_qentry_del(flow);
345 memset(flow, 0, sizeof(*flow));
346 flow->type = SMC_LLC_FLOW_NONE;
347 spin_unlock_bh(&lgr->llc_flow_lock);
348 if (!list_empty(&lgr->list) && lgr->delayed_event &&
349 flow == &lgr->llc_flow_lcl)
350 schedule_work(&lgr->llc_event_work);
351 else
352 wake_up(&lgr->llc_flow_waiter);
353 }
354
355 /* lnk is optional and used for early wakeup when link goes down, useful in
356 * cases where we wait for a response on the link after we sent a request
357 */
smc_llc_wait(struct smc_link_group * lgr,struct smc_link * lnk,int time_out,u8 exp_msg)358 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
359 struct smc_link *lnk,
360 int time_out, u8 exp_msg)
361 {
362 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
363 u8 rcv_msg;
364
365 wait_event_timeout(lgr->llc_msg_waiter,
366 (flow->qentry ||
367 (lnk && !smc_link_usable(lnk)) ||
368 list_empty(&lgr->list)),
369 time_out);
370 if (!flow->qentry ||
371 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
372 smc_llc_flow_qentry_del(flow);
373 goto out;
374 }
375 rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
376 if (exp_msg && rcv_msg != exp_msg) {
377 if (exp_msg == SMC_LLC_ADD_LINK &&
378 rcv_msg == SMC_LLC_DELETE_LINK) {
379 /* flow_start will delay the unexpected msg */
380 smc_llc_flow_start(&lgr->llc_flow_lcl,
381 smc_llc_flow_qentry_clr(flow));
382 return NULL;
383 }
384 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
385 "msg %d exp %d flow %d role %d flags %x\n",
386 SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
387 rcv_msg, exp_msg,
388 flow->type, lgr->role,
389 flow->qentry->msg.raw.hdr.flags);
390 smc_llc_flow_qentry_del(flow);
391 }
392 out:
393 return flow->qentry;
394 }
395
396 /********************************** send *************************************/
397
398 struct smc_llc_tx_pend {
399 };
400
401 /* handler for send/transmission completion of an LLC msg */
smc_llc_tx_handler(struct smc_wr_tx_pend_priv * pend,struct smc_link * link,enum ib_wc_status wc_status)402 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
403 struct smc_link *link,
404 enum ib_wc_status wc_status)
405 {
406 /* future work: handle wc_status error for recovery and failover */
407 }
408
409 /**
410 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
411 * @link: Pointer to SMC link used for sending LLC control message.
412 * @wr_buf: Out variable returning pointer to work request payload buffer.
413 * @pend: Out variable returning pointer to private pending WR tracking.
414 * It's the context the transmit complete handler will get.
415 *
416 * Reserves and pre-fills an entry for a pending work request send/tx.
417 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
418 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
419 *
420 * Return: 0 on success, otherwise an error value.
421 */
smc_llc_add_pending_send(struct smc_link * link,struct smc_wr_buf ** wr_buf,struct smc_wr_tx_pend_priv ** pend)422 static int smc_llc_add_pending_send(struct smc_link *link,
423 struct smc_wr_buf **wr_buf,
424 struct smc_wr_tx_pend_priv **pend)
425 {
426 int rc;
427
428 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
429 pend);
430 if (rc < 0)
431 return rc;
432 BUILD_BUG_ON_MSG(
433 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
434 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
435 BUILD_BUG_ON_MSG(
436 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
437 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
438 BUILD_BUG_ON_MSG(
439 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
440 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
441 return 0;
442 }
443
smc_llc_add_pending_send_v2(struct smc_link * link,struct smc_wr_v2_buf ** wr_buf,struct smc_wr_tx_pend_priv ** pend)444 static int smc_llc_add_pending_send_v2(struct smc_link *link,
445 struct smc_wr_v2_buf **wr_buf,
446 struct smc_wr_tx_pend_priv **pend)
447 {
448 int rc;
449
450 rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
451 if (rc < 0)
452 return rc;
453 return 0;
454 }
455
smc_llc_init_msg_hdr(struct smc_llc_hdr * hdr,struct smc_link_group * lgr,size_t len)456 static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
457 struct smc_link_group *lgr, size_t len)
458 {
459 if (lgr->smc_version == SMC_V2) {
460 hdr->common.llc_version = SMC_V2;
461 hdr->length_v2 = len;
462 } else {
463 hdr->common.llc_version = 0;
464 hdr->length = len;
465 }
466 }
467
468 /* high-level API to send LLC confirm link */
smc_llc_send_confirm_link(struct smc_link * link,enum smc_llc_reqresp reqresp)469 int smc_llc_send_confirm_link(struct smc_link *link,
470 enum smc_llc_reqresp reqresp)
471 {
472 struct smc_llc_msg_confirm_link *confllc;
473 struct smc_wr_tx_pend_priv *pend;
474 struct smc_wr_buf *wr_buf;
475 int rc;
476
477 if (!smc_wr_tx_link_hold(link))
478 return -ENOLINK;
479 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
480 if (rc)
481 goto put_out;
482 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
483 memset(confllc, 0, sizeof(*confllc));
484 confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
485 smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
486 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
487 if (reqresp == SMC_LLC_RESP)
488 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
489 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
490 ETH_ALEN);
491 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
492 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
493 confllc->link_num = link->link_id;
494 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
495 confllc->max_links = SMC_LINKS_ADD_LNK_MAX;
496 if (link->lgr->smc_version == SMC_V2 &&
497 link->lgr->peer_smc_release >= SMC_RELEASE_1) {
498 confllc->max_conns = link->lgr->max_conns;
499 confllc->max_links = link->lgr->max_links;
500 }
501 /* send llc message */
502 rc = smc_wr_tx_send(link, pend);
503 put_out:
504 smc_wr_tx_link_put(link);
505 return rc;
506 }
507
508 /* send LLC confirm rkey request */
smc_llc_send_confirm_rkey(struct smc_link * send_link,struct smc_buf_desc * rmb_desc)509 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
510 struct smc_buf_desc *rmb_desc)
511 {
512 struct smc_llc_msg_confirm_rkey *rkeyllc;
513 struct smc_wr_tx_pend_priv *pend;
514 struct smc_wr_buf *wr_buf;
515 struct smc_link *link;
516 int i, rc, rtok_ix;
517
518 if (!smc_wr_tx_link_hold(send_link))
519 return -ENOLINK;
520 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
521 if (rc)
522 goto put_out;
523 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
524 memset(rkeyllc, 0, sizeof(*rkeyllc));
525 rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
526 smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
527
528 rtok_ix = 1;
529 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
530 link = &send_link->lgr->lnk[i];
531 if (smc_link_active(link) && link != send_link) {
532 rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
533 rkeyllc->rtoken[rtok_ix].rmb_key =
534 htonl(rmb_desc->mr[link->link_idx]->rkey);
535 rkeyllc->rtoken[rtok_ix].rmb_vaddr = rmb_desc->is_vm ?
536 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
537 cpu_to_be64((u64)sg_dma_address
538 (rmb_desc->sgt[link->link_idx].sgl));
539 rtok_ix++;
540 }
541 }
542 /* rkey of send_link is in rtoken[0] */
543 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
544 rkeyllc->rtoken[0].rmb_key =
545 htonl(rmb_desc->mr[send_link->link_idx]->rkey);
546 rkeyllc->rtoken[0].rmb_vaddr = rmb_desc->is_vm ?
547 cpu_to_be64((uintptr_t)rmb_desc->cpu_addr) :
548 cpu_to_be64((u64)sg_dma_address
549 (rmb_desc->sgt[send_link->link_idx].sgl));
550 /* send llc message */
551 rc = smc_wr_tx_send(send_link, pend);
552 put_out:
553 smc_wr_tx_link_put(send_link);
554 return rc;
555 }
556
557 /* send LLC delete rkey request */
smc_llc_send_delete_rkey(struct smc_link * link,struct smc_buf_desc * rmb_desc)558 static int smc_llc_send_delete_rkey(struct smc_link *link,
559 struct smc_buf_desc *rmb_desc)
560 {
561 struct smc_llc_msg_delete_rkey *rkeyllc;
562 struct smc_wr_tx_pend_priv *pend;
563 struct smc_wr_buf *wr_buf;
564 int rc;
565
566 if (!smc_wr_tx_link_hold(link))
567 return -ENOLINK;
568 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
569 if (rc)
570 goto put_out;
571 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
572 memset(rkeyllc, 0, sizeof(*rkeyllc));
573 rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
574 smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
575 rkeyllc->num_rkeys = 1;
576 rkeyllc->rkey[0] = htonl(rmb_desc->mr[link->link_idx]->rkey);
577 /* send llc message */
578 rc = smc_wr_tx_send(link, pend);
579 put_out:
580 smc_wr_tx_link_put(link);
581 return rc;
582 }
583
584 /* return first buffer from any of the next buf lists */
_smc_llc_get_next_rmb(struct smc_link_group * lgr,int * buf_lst)585 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
586 int *buf_lst)
587 {
588 struct smc_buf_desc *buf_pos;
589
590 while (*buf_lst < SMC_RMBE_SIZES) {
591 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
592 struct smc_buf_desc, list);
593 if (buf_pos)
594 return buf_pos;
595 (*buf_lst)++;
596 }
597 return NULL;
598 }
599
600 /* return next rmb from buffer lists */
smc_llc_get_next_rmb(struct smc_link_group * lgr,int * buf_lst,struct smc_buf_desc * buf_pos)601 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
602 int *buf_lst,
603 struct smc_buf_desc *buf_pos)
604 {
605 struct smc_buf_desc *buf_next;
606
607 if (!buf_pos)
608 return _smc_llc_get_next_rmb(lgr, buf_lst);
609
610 if (list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
611 (*buf_lst)++;
612 return _smc_llc_get_next_rmb(lgr, buf_lst);
613 }
614 buf_next = list_next_entry(buf_pos, list);
615 return buf_next;
616 }
617
smc_llc_get_first_rmb(struct smc_link_group * lgr,int * buf_lst)618 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
619 int *buf_lst)
620 {
621 *buf_lst = 0;
622 return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
623 }
624
smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext * ext,struct smc_link * link,struct smc_link * link_new)625 static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
626 struct smc_link *link, struct smc_link *link_new)
627 {
628 struct smc_link_group *lgr = link->lgr;
629 struct smc_buf_desc *buf_pos;
630 int prim_lnk_idx, lnk_idx, i;
631 struct smc_buf_desc *rmb;
632 int len = sizeof(*ext);
633 int buf_lst;
634
635 ext->v2_direct = !lgr->uses_gateway;
636 memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
637
638 prim_lnk_idx = link->link_idx;
639 lnk_idx = link_new->link_idx;
640 down_write(&lgr->rmbs_lock);
641 ext->num_rkeys = lgr->conns_num;
642 if (!ext->num_rkeys)
643 goto out;
644 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
645 for (i = 0; i < ext->num_rkeys; i++) {
646 while (buf_pos && !(buf_pos)->used)
647 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
648 if (!buf_pos)
649 break;
650 rmb = buf_pos;
651 ext->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
652 ext->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
653 ext->rt[i].rmb_vaddr_new = rmb->is_vm ?
654 cpu_to_be64((uintptr_t)rmb->cpu_addr) :
655 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
656 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
657 }
658 len += i * sizeof(ext->rt[0]);
659 out:
660 up_write(&lgr->rmbs_lock);
661 return len;
662 }
663
664 /* send ADD LINK request or response */
smc_llc_send_add_link(struct smc_link * link,u8 mac[],u8 gid[],struct smc_link * link_new,enum smc_llc_reqresp reqresp)665 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
666 struct smc_link *link_new,
667 enum smc_llc_reqresp reqresp)
668 {
669 struct smc_llc_msg_add_link_v2_ext *ext = NULL;
670 struct smc_llc_msg_add_link *addllc;
671 struct smc_wr_tx_pend_priv *pend;
672 int len = sizeof(*addllc);
673 int rc;
674
675 if (!smc_wr_tx_link_hold(link))
676 return -ENOLINK;
677 if (link->lgr->smc_version == SMC_V2) {
678 struct smc_wr_v2_buf *wr_buf;
679
680 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
681 if (rc)
682 goto put_out;
683 addllc = (struct smc_llc_msg_add_link *)wr_buf;
684 ext = (struct smc_llc_msg_add_link_v2_ext *)
685 &wr_buf->raw[sizeof(*addllc)];
686 memset(ext, 0, SMC_WR_TX_SIZE);
687 } else {
688 struct smc_wr_buf *wr_buf;
689
690 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
691 if (rc)
692 goto put_out;
693 addllc = (struct smc_llc_msg_add_link *)wr_buf;
694 }
695
696 memset(addllc, 0, sizeof(*addllc));
697 addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
698 if (reqresp == SMC_LLC_RESP)
699 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
700 memcpy(addllc->sender_mac, mac, ETH_ALEN);
701 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
702 if (link_new) {
703 addllc->link_num = link_new->link_id;
704 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
705 hton24(addllc->initial_psn, link_new->psn_initial);
706 if (reqresp == SMC_LLC_REQ)
707 addllc->qp_mtu = link_new->path_mtu;
708 else
709 addllc->qp_mtu = min(link_new->path_mtu,
710 link_new->peer_mtu);
711 }
712 if (ext && link_new)
713 len += smc_llc_fill_ext_v2(ext, link, link_new);
714 smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
715 /* send llc message */
716 if (link->lgr->smc_version == SMC_V2)
717 rc = smc_wr_tx_v2_send(link, pend, len);
718 else
719 rc = smc_wr_tx_send(link, pend);
720 put_out:
721 smc_wr_tx_link_put(link);
722 return rc;
723 }
724
725 /* send DELETE LINK request or response */
smc_llc_send_delete_link(struct smc_link * link,u8 link_del_id,enum smc_llc_reqresp reqresp,bool orderly,u32 reason)726 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
727 enum smc_llc_reqresp reqresp, bool orderly,
728 u32 reason)
729 {
730 struct smc_llc_msg_del_link *delllc;
731 struct smc_wr_tx_pend_priv *pend;
732 struct smc_wr_buf *wr_buf;
733 int rc;
734
735 if (!smc_wr_tx_link_hold(link))
736 return -ENOLINK;
737 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
738 if (rc)
739 goto put_out;
740 delllc = (struct smc_llc_msg_del_link *)wr_buf;
741
742 memset(delllc, 0, sizeof(*delllc));
743 delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
744 smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
745 if (reqresp == SMC_LLC_RESP)
746 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
747 if (orderly)
748 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
749 if (link_del_id)
750 delllc->link_num = link_del_id;
751 else
752 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
753 delllc->reason = htonl(reason);
754 /* send llc message */
755 rc = smc_wr_tx_send(link, pend);
756 put_out:
757 smc_wr_tx_link_put(link);
758 return rc;
759 }
760
761 /* send LLC test link request */
smc_llc_send_test_link(struct smc_link * link,u8 user_data[16])762 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
763 {
764 struct smc_llc_msg_test_link *testllc;
765 struct smc_wr_tx_pend_priv *pend;
766 struct smc_wr_buf *wr_buf;
767 int rc;
768
769 if (!smc_wr_tx_link_hold(link))
770 return -ENOLINK;
771 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
772 if (rc)
773 goto put_out;
774 testllc = (struct smc_llc_msg_test_link *)wr_buf;
775 memset(testllc, 0, sizeof(*testllc));
776 testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
777 smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
778 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
779 /* send llc message */
780 rc = smc_wr_tx_send(link, pend);
781 put_out:
782 smc_wr_tx_link_put(link);
783 return rc;
784 }
785
786 /* schedule an llc send on link, may wait for buffers */
smc_llc_send_message(struct smc_link * link,void * llcbuf)787 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
788 {
789 struct smc_wr_tx_pend_priv *pend;
790 struct smc_wr_buf *wr_buf;
791 int rc;
792
793 if (!smc_wr_tx_link_hold(link))
794 return -ENOLINK;
795 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
796 if (rc)
797 goto put_out;
798 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
799 rc = smc_wr_tx_send(link, pend);
800 put_out:
801 smc_wr_tx_link_put(link);
802 return rc;
803 }
804
805 /* schedule an llc send on link, may wait for buffers,
806 * and wait for send completion notification.
807 * @return 0 on success
808 */
smc_llc_send_message_wait(struct smc_link * link,void * llcbuf)809 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
810 {
811 struct smc_wr_tx_pend_priv *pend;
812 struct smc_wr_buf *wr_buf;
813 int rc;
814
815 if (!smc_wr_tx_link_hold(link))
816 return -ENOLINK;
817 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
818 if (rc)
819 goto put_out;
820 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
821 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
822 put_out:
823 smc_wr_tx_link_put(link);
824 return rc;
825 }
826
827 /********************************* receive ***********************************/
828
smc_llc_alloc_alt_link(struct smc_link_group * lgr,enum smc_lgr_type lgr_new_t)829 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
830 enum smc_lgr_type lgr_new_t)
831 {
832 int i;
833
834 if (lgr->type == SMC_LGR_SYMMETRIC ||
835 (lgr->type != SMC_LGR_SINGLE &&
836 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
837 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
838 return -EMLINK;
839
840 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
841 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
842 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
843 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
844 return i;
845 } else {
846 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
847 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
848 return i;
849 }
850 return -EMLINK;
851 }
852
853 /* send one add_link_continue msg */
smc_llc_add_link_cont(struct smc_link * link,struct smc_link * link_new,u8 * num_rkeys_todo,int * buf_lst,struct smc_buf_desc ** buf_pos)854 static int smc_llc_add_link_cont(struct smc_link *link,
855 struct smc_link *link_new, u8 *num_rkeys_todo,
856 int *buf_lst, struct smc_buf_desc **buf_pos)
857 {
858 struct smc_llc_msg_add_link_cont *addc_llc;
859 struct smc_link_group *lgr = link->lgr;
860 int prim_lnk_idx, lnk_idx, i, rc;
861 struct smc_wr_tx_pend_priv *pend;
862 struct smc_wr_buf *wr_buf;
863 struct smc_buf_desc *rmb;
864 u8 n;
865
866 if (!smc_wr_tx_link_hold(link))
867 return -ENOLINK;
868 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
869 if (rc)
870 goto put_out;
871 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
872 memset(addc_llc, 0, sizeof(*addc_llc));
873
874 prim_lnk_idx = link->link_idx;
875 lnk_idx = link_new->link_idx;
876 addc_llc->link_num = link_new->link_id;
877 addc_llc->num_rkeys = *num_rkeys_todo;
878 n = *num_rkeys_todo;
879 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
880 while (*buf_pos && !(*buf_pos)->used)
881 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
882 if (!*buf_pos) {
883 addc_llc->num_rkeys = addc_llc->num_rkeys -
884 *num_rkeys_todo;
885 *num_rkeys_todo = 0;
886 break;
887 }
888 rmb = *buf_pos;
889
890 addc_llc->rt[i].rmb_key = htonl(rmb->mr[prim_lnk_idx]->rkey);
891 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr[lnk_idx]->rkey);
892 addc_llc->rt[i].rmb_vaddr_new = rmb->is_vm ?
893 cpu_to_be64((uintptr_t)rmb->cpu_addr) :
894 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
895
896 (*num_rkeys_todo)--;
897 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
898 }
899 addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
900 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
901 if (lgr->role == SMC_CLNT)
902 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
903 rc = smc_wr_tx_send(link, pend);
904 put_out:
905 smc_wr_tx_link_put(link);
906 return rc;
907 }
908
smc_llc_cli_rkey_exchange(struct smc_link * link,struct smc_link * link_new)909 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
910 struct smc_link *link_new)
911 {
912 struct smc_llc_msg_add_link_cont *addc_llc;
913 struct smc_link_group *lgr = link->lgr;
914 u8 max, num_rkeys_send, num_rkeys_recv;
915 struct smc_llc_qentry *qentry;
916 struct smc_buf_desc *buf_pos;
917 int buf_lst;
918 int rc = 0;
919 int i;
920
921 down_write(&lgr->rmbs_lock);
922 num_rkeys_send = lgr->conns_num;
923 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
924 do {
925 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
926 SMC_LLC_ADD_LINK_CONT);
927 if (!qentry) {
928 rc = -ETIMEDOUT;
929 break;
930 }
931 addc_llc = &qentry->msg.add_link_cont;
932 num_rkeys_recv = addc_llc->num_rkeys;
933 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
934 for (i = 0; i < max; i++) {
935 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
936 addc_llc->rt[i].rmb_key,
937 addc_llc->rt[i].rmb_vaddr_new,
938 addc_llc->rt[i].rmb_key_new);
939 num_rkeys_recv--;
940 }
941 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
942 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
943 &buf_lst, &buf_pos);
944 if (rc)
945 break;
946 } while (num_rkeys_send || num_rkeys_recv);
947
948 up_write(&lgr->rmbs_lock);
949 return rc;
950 }
951
952 /* prepare and send an add link reject response */
smc_llc_cli_add_link_reject(struct smc_llc_qentry * qentry)953 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
954 {
955 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
956 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
957 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
958 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
959 sizeof(qentry->msg));
960 return smc_llc_send_message(qentry->link, &qentry->msg);
961 }
962
smc_llc_cli_conf_link(struct smc_link * link,struct smc_init_info * ini,struct smc_link * link_new,enum smc_lgr_type lgr_new_t)963 static int smc_llc_cli_conf_link(struct smc_link *link,
964 struct smc_init_info *ini,
965 struct smc_link *link_new,
966 enum smc_lgr_type lgr_new_t)
967 {
968 struct smc_link_group *lgr = link->lgr;
969 struct smc_llc_qentry *qentry = NULL;
970 int rc = 0;
971
972 /* receive CONFIRM LINK request over RoCE fabric */
973 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
974 if (!qentry) {
975 rc = smc_llc_send_delete_link(link, link_new->link_id,
976 SMC_LLC_REQ, false,
977 SMC_LLC_DEL_LOST_PATH);
978 return -ENOLINK;
979 }
980 if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
981 /* received DELETE_LINK instead */
982 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
983 smc_llc_send_message(link, &qentry->msg);
984 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
985 return -ENOLINK;
986 }
987 smc_llc_save_peer_uid(qentry);
988 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
989
990 rc = smc_ib_modify_qp_rts(link_new);
991 if (rc) {
992 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
993 false, SMC_LLC_DEL_LOST_PATH);
994 return -ENOLINK;
995 }
996 smc_wr_remember_qp_attr(link_new);
997
998 rc = smcr_buf_reg_lgr(link_new);
999 if (rc) {
1000 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1001 false, SMC_LLC_DEL_LOST_PATH);
1002 return -ENOLINK;
1003 }
1004
1005 /* send CONFIRM LINK response over RoCE fabric */
1006 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
1007 if (rc) {
1008 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1009 false, SMC_LLC_DEL_LOST_PATH);
1010 return -ENOLINK;
1011 }
1012 smc_llc_link_active(link_new);
1013 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1014 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1015 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1016 else
1017 smcr_lgr_set_type(lgr, lgr_new_t);
1018 return 0;
1019 }
1020
smc_llc_save_add_link_rkeys(struct smc_link * link,struct smc_link * link_new,struct smc_llc_qentry * qentry)1021 static void smc_llc_save_add_link_rkeys(struct smc_link *link,
1022 struct smc_link *link_new,
1023 struct smc_llc_qentry *qentry)
1024 {
1025 const u32 rt_off = offsetof(struct smc_llc_msg_add_link_v2_ext, rt);
1026 struct smc_llc_msg_add_link_v2_ext *ext;
1027 struct smc_link_group *lgr = link->lgr;
1028 int max, i;
1029
1030 /* the rkey count itself is only there if enough bytes arrived */
1031 if (qentry->body_len < rt_off)
1032 return;
1033 ext = (struct smc_llc_msg_add_link_v2_ext *)qentry->body;
1034 max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1035 max = min_t(u32, max, (qentry->body_len - rt_off) /
1036 sizeof(ext->rt[0]));
1037 down_write(&lgr->rmbs_lock);
1038 for (i = 0; i < max; i++) {
1039 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1040 ext->rt[i].rmb_key,
1041 ext->rt[i].rmb_vaddr_new,
1042 ext->rt[i].rmb_key_new);
1043 }
1044 up_write(&lgr->rmbs_lock);
1045 }
1046
smc_llc_save_add_link_info(struct smc_link * link,struct smc_llc_msg_add_link * add_llc)1047 static void smc_llc_save_add_link_info(struct smc_link *link,
1048 struct smc_llc_msg_add_link *add_llc)
1049 {
1050 link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1051 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1052 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1053 link->peer_psn = ntoh24(add_llc->initial_psn);
1054 link->peer_mtu = add_llc->qp_mtu;
1055 }
1056
1057 /* as an SMC client, process an add link request */
smc_llc_cli_add_link(struct smc_link * link,struct smc_llc_qentry * qentry)1058 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1059 {
1060 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1061 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1062 struct smc_link_group *lgr = smc_get_lgr(link);
1063 struct smc_init_info *ini = NULL;
1064 struct smc_link *lnk_new = NULL;
1065 int lnk_idx, rc = 0;
1066
1067 if (!llc->qp_mtu)
1068 goto out_reject;
1069
1070 ini = kzalloc_obj(*ini);
1071 if (!ini) {
1072 rc = -ENOMEM;
1073 goto out_reject;
1074 }
1075
1076 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1077 rc = 0;
1078 goto out_reject;
1079 }
1080
1081 ini->vlan_id = lgr->vlan_id;
1082 if (lgr->smc_version == SMC_V2) {
1083 ini->check_smcrv2 = true;
1084 ini->smcrv2.saddr = lgr->saddr;
1085 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1086 }
1087 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1088 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1089 (lgr->smc_version == SMC_V2 ||
1090 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1091 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
1092 goto out_reject;
1093 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1094 }
1095 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1096 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1097 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1098 ini->smcrv2.ib_port_v2 = link->ibport;
1099 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1100 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1101 ini->ib_dev = link->smcibdev;
1102 ini->ib_port = link->ibport;
1103 }
1104 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1105 if (lnk_idx < 0)
1106 goto out_reject;
1107 lnk_new = &lgr->lnk[lnk_idx];
1108 rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
1109 if (rc)
1110 goto out_reject;
1111 smc_llc_save_add_link_info(lnk_new, llc);
1112 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */
1113 smc_llc_link_set_uid(lnk_new);
1114
1115 rc = smc_ib_ready_link(lnk_new);
1116 if (rc)
1117 goto out_clear_lnk;
1118
1119 rc = smcr_buf_map_lgr(lnk_new);
1120 if (rc)
1121 goto out_clear_lnk;
1122
1123 rc = smc_llc_send_add_link(link,
1124 lnk_new->smcibdev->mac[lnk_new->ibport - 1],
1125 lnk_new->gid, lnk_new, SMC_LLC_RESP);
1126 if (rc)
1127 goto out_clear_lnk;
1128 if (lgr->smc_version == SMC_V2) {
1129 smc_llc_save_add_link_rkeys(link, lnk_new, qentry);
1130 } else {
1131 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1132 if (rc) {
1133 rc = 0;
1134 goto out_clear_lnk;
1135 }
1136 }
1137 rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
1138 if (!rc)
1139 goto out;
1140 out_clear_lnk:
1141 lnk_new->state = SMC_LNK_INACTIVE;
1142 smcr_link_clear(lnk_new, false);
1143 out_reject:
1144 smc_llc_cli_add_link_reject(qentry);
1145 out:
1146 kfree(ini);
1147 kfree(qentry);
1148 return rc;
1149 }
1150
smc_llc_send_request_add_link(struct smc_link * link)1151 static void smc_llc_send_request_add_link(struct smc_link *link)
1152 {
1153 struct smc_llc_msg_req_add_link_v2 *llc;
1154 struct smc_wr_tx_pend_priv *pend;
1155 struct smc_wr_v2_buf *wr_buf;
1156 struct smc_gidlist gidlist;
1157 int rc, len, i;
1158
1159 if (!smc_wr_tx_link_hold(link))
1160 return;
1161 if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1162 link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1163 goto put_out;
1164
1165 smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1166 if (gidlist.len <= 1)
1167 goto put_out;
1168
1169 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1170 if (rc)
1171 goto put_out;
1172 llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1173 memset(llc, 0, SMC_WR_TX_SIZE);
1174
1175 llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1176 for (i = 0; i < gidlist.len; i++)
1177 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1178 llc->gid_cnt = gidlist.len;
1179 len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1180 smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1181 rc = smc_wr_tx_v2_send(link, pend, len);
1182 if (!rc)
1183 /* set REQ_ADD_LINK flow and wait for response from peer */
1184 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1185 put_out:
1186 smc_wr_tx_link_put(link);
1187 }
1188
1189 /* as an SMC client, invite server to start the add_link processing */
smc_llc_cli_add_link_invite(struct smc_link * link,struct smc_llc_qentry * qentry)1190 static void smc_llc_cli_add_link_invite(struct smc_link *link,
1191 struct smc_llc_qentry *qentry)
1192 {
1193 struct smc_link_group *lgr = smc_get_lgr(link);
1194 struct smc_init_info *ini = NULL;
1195
1196 if (lgr->smc_version == SMC_V2) {
1197 smc_llc_send_request_add_link(link);
1198 goto out;
1199 }
1200
1201 if (lgr->type == SMC_LGR_SYMMETRIC ||
1202 lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1203 goto out;
1204
1205 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1)
1206 goto out;
1207
1208 ini = kzalloc_obj(*ini);
1209 if (!ini)
1210 goto out;
1211
1212 ini->vlan_id = lgr->vlan_id;
1213 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1214 if (!ini->ib_dev)
1215 goto out;
1216
1217 smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1218 ini->ib_gid, NULL, SMC_LLC_REQ);
1219 out:
1220 kfree(ini);
1221 kfree(qentry);
1222 }
1223
smc_llc_is_empty_llc_message(union smc_llc_msg * llc)1224 static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1225 {
1226 int i;
1227
1228 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1229 if (llc->raw.data[i])
1230 return false;
1231 return true;
1232 }
1233
smc_llc_is_local_add_link(union smc_llc_msg * llc)1234 static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1235 {
1236 if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
1237 smc_llc_is_empty_llc_message(llc))
1238 return true;
1239 return false;
1240 }
1241
smc_llc_process_cli_add_link(struct smc_link_group * lgr)1242 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1243 {
1244 struct smc_llc_qentry *qentry;
1245
1246 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1247
1248 down_write(&lgr->llc_conf_mutex);
1249 if (smc_llc_is_local_add_link(&qentry->msg))
1250 smc_llc_cli_add_link_invite(qentry->link, qentry);
1251 else
1252 smc_llc_cli_add_link(qentry->link, qentry);
1253 up_write(&lgr->llc_conf_mutex);
1254 }
1255
smc_llc_active_link_count(struct smc_link_group * lgr)1256 static int smc_llc_active_link_count(struct smc_link_group *lgr)
1257 {
1258 int i, link_count = 0;
1259
1260 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1261 if (!smc_link_active(&lgr->lnk[i]))
1262 continue;
1263 link_count++;
1264 }
1265 return link_count;
1266 }
1267
1268 /* find the asymmetric link when 3 links are established */
smc_llc_find_asym_link(struct smc_link_group * lgr)1269 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1270 {
1271 int asym_idx = -ENOENT;
1272 int i, j, k;
1273 bool found;
1274
1275 /* determine asymmetric link */
1276 found = false;
1277 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1278 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1279 if (!smc_link_usable(&lgr->lnk[i]) ||
1280 !smc_link_usable(&lgr->lnk[j]))
1281 continue;
1282 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1283 SMC_GID_SIZE)) {
1284 found = true; /* asym_lnk is i or j */
1285 break;
1286 }
1287 }
1288 if (found)
1289 break;
1290 }
1291 if (!found)
1292 goto out; /* no asymmetric link */
1293 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1294 if (!smc_link_usable(&lgr->lnk[k]))
1295 continue;
1296 if (k != i &&
1297 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1298 SMC_GID_SIZE)) {
1299 asym_idx = i;
1300 break;
1301 }
1302 if (k != j &&
1303 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1304 SMC_GID_SIZE)) {
1305 asym_idx = j;
1306 break;
1307 }
1308 }
1309 out:
1310 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1311 }
1312
smc_llc_delete_asym_link(struct smc_link_group * lgr)1313 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1314 {
1315 struct smc_link *lnk_new = NULL, *lnk_asym;
1316 struct smc_llc_qentry *qentry;
1317 int rc;
1318
1319 lnk_asym = smc_llc_find_asym_link(lgr);
1320 if (!lnk_asym)
1321 return; /* no asymmetric link */
1322 if (!smc_link_downing(&lnk_asym->state))
1323 return;
1324 lnk_new = smc_switch_conns(lgr, lnk_asym, false);
1325 smc_wr_tx_wait_no_pending_sends(lnk_asym);
1326 if (!lnk_new)
1327 goto out_free;
1328 /* change flow type from ADD_LINK into DEL_LINK */
1329 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1330 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1331 true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1332 if (rc) {
1333 smcr_link_down_cond(lnk_new);
1334 goto out_free;
1335 }
1336 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1337 SMC_LLC_DELETE_LINK);
1338 if (!qentry) {
1339 smcr_link_down_cond(lnk_new);
1340 goto out_free;
1341 }
1342 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1343 out_free:
1344 smcr_link_clear(lnk_asym, true);
1345 }
1346
smc_llc_srv_rkey_exchange(struct smc_link * link,struct smc_link * link_new)1347 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1348 struct smc_link *link_new)
1349 {
1350 struct smc_llc_msg_add_link_cont *addc_llc;
1351 struct smc_link_group *lgr = link->lgr;
1352 u8 max, num_rkeys_send, num_rkeys_recv;
1353 struct smc_llc_qentry *qentry = NULL;
1354 struct smc_buf_desc *buf_pos;
1355 int buf_lst;
1356 int rc = 0;
1357 int i;
1358
1359 down_write(&lgr->rmbs_lock);
1360 num_rkeys_send = lgr->conns_num;
1361 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1362 do {
1363 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1364 &buf_lst, &buf_pos);
1365 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1366 SMC_LLC_ADD_LINK_CONT);
1367 if (!qentry) {
1368 rc = -ETIMEDOUT;
1369 goto out;
1370 }
1371 addc_llc = &qentry->msg.add_link_cont;
1372 num_rkeys_recv = addc_llc->num_rkeys;
1373 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1374 for (i = 0; i < max; i++) {
1375 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1376 addc_llc->rt[i].rmb_key,
1377 addc_llc->rt[i].rmb_vaddr_new,
1378 addc_llc->rt[i].rmb_key_new);
1379 num_rkeys_recv--;
1380 }
1381 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1382 } while (num_rkeys_send || num_rkeys_recv);
1383 out:
1384 up_write(&lgr->rmbs_lock);
1385 return rc;
1386 }
1387
smc_llc_srv_conf_link(struct smc_link * link,struct smc_link * link_new,enum smc_lgr_type lgr_new_t)1388 static int smc_llc_srv_conf_link(struct smc_link *link,
1389 struct smc_link *link_new,
1390 enum smc_lgr_type lgr_new_t)
1391 {
1392 struct smc_link_group *lgr = link->lgr;
1393 struct smc_llc_qentry *qentry = NULL;
1394 int rc;
1395
1396 /* send CONFIRM LINK request over the RoCE fabric */
1397 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1398 if (rc)
1399 return -ENOLINK;
1400 /* receive CONFIRM LINK response over the RoCE fabric */
1401 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1402 if (!qentry ||
1403 qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1404 /* send DELETE LINK */
1405 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1406 false, SMC_LLC_DEL_LOST_PATH);
1407 if (qentry)
1408 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1409 return -ENOLINK;
1410 }
1411 smc_llc_save_peer_uid(qentry);
1412 smc_llc_link_active(link_new);
1413 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1414 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1415 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1416 else
1417 smcr_lgr_set_type(lgr, lgr_new_t);
1418 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1419 return 0;
1420 }
1421
smc_llc_send_req_add_link_response(struct smc_llc_qentry * qentry)1422 static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1423 {
1424 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1425 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1426 sizeof(qentry->msg));
1427 memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1428 smc_llc_send_message(qentry->link, &qentry->msg);
1429 }
1430
smc_llc_srv_add_link(struct smc_link * link,struct smc_llc_qentry * req_qentry)1431 int smc_llc_srv_add_link(struct smc_link *link,
1432 struct smc_llc_qentry *req_qentry)
1433 {
1434 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1435 struct smc_link_group *lgr = link->lgr;
1436 struct smc_llc_msg_add_link *add_llc;
1437 struct smc_llc_qentry *qentry = NULL;
1438 bool send_req_add_link_resp = false;
1439 struct smc_link *link_new = NULL;
1440 struct smc_init_info *ini = NULL;
1441 int lnk_idx, rc = 0;
1442
1443 if (req_qentry &&
1444 req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1445 send_req_add_link_resp = true;
1446
1447 ini = kzalloc_obj(*ini);
1448 if (!ini) {
1449 rc = -ENOMEM;
1450 goto out;
1451 }
1452
1453 if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) {
1454 rc = 0;
1455 goto out;
1456 }
1457
1458 /* ignore client add link recommendation, start new flow */
1459 ini->vlan_id = lgr->vlan_id;
1460 if (lgr->smc_version == SMC_V2) {
1461 ini->check_smcrv2 = true;
1462 ini->smcrv2.saddr = lgr->saddr;
1463 if (send_req_add_link_resp) {
1464 struct smc_llc_msg_req_add_link_v2 *req_add =
1465 &req_qentry->msg.req_add_link;
1466
1467 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1468 }
1469 }
1470 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1471 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1472 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1473 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1474 ini->smcrv2.ib_port_v2 = link->ibport;
1475 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
1476 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1477 ini->ib_dev = link->smcibdev;
1478 ini->ib_port = link->ibport;
1479 }
1480 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1481 if (lnk_idx < 0) {
1482 rc = 0;
1483 goto out;
1484 }
1485
1486 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
1487 if (rc)
1488 goto out;
1489 link_new = &lgr->lnk[lnk_idx];
1490
1491 rc = smcr_buf_map_lgr(link_new);
1492 if (rc)
1493 goto out_err;
1494
1495 rc = smc_llc_send_add_link(link,
1496 link_new->smcibdev->mac[link_new->ibport-1],
1497 link_new->gid, link_new, SMC_LLC_REQ);
1498 if (rc)
1499 goto out_err;
1500 send_req_add_link_resp = false;
1501 /* receive ADD LINK response over the RoCE fabric */
1502 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1503 if (!qentry) {
1504 rc = -ETIMEDOUT;
1505 goto out_err;
1506 }
1507 add_llc = &qentry->msg.add_link;
1508 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1509 smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1510 rc = -ENOLINK;
1511 goto out_err;
1512 }
1513 if (lgr->type == SMC_LGR_SINGLE &&
1514 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1515 (lgr->smc_version == SMC_V2 ||
1516 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
1517 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1518 }
1519 smc_llc_save_add_link_info(link_new, add_llc);
1520 /* add_llc still points into qentry, so only detach it here */
1521 smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1522
1523 rc = smc_ib_ready_link(link_new);
1524 if (rc)
1525 goto out_err;
1526 rc = smcr_buf_reg_lgr(link_new);
1527 if (rc)
1528 goto out_err;
1529 if (lgr->smc_version == SMC_V2) {
1530 smc_llc_save_add_link_rkeys(link, link_new, qentry);
1531 } else {
1532 rc = smc_llc_srv_rkey_exchange(link, link_new);
1533 if (rc)
1534 goto out_err;
1535 }
1536 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1537 if (rc)
1538 goto out_err;
1539 goto out;
1540 out_err:
1541 if (link_new) {
1542 link_new->state = SMC_LNK_INACTIVE;
1543 smcr_link_clear(link_new, false);
1544 }
1545 out:
1546 kfree(qentry);
1547 kfree(ini);
1548 if (send_req_add_link_resp)
1549 smc_llc_send_req_add_link_response(req_qentry);
1550 return rc;
1551 }
1552
smc_llc_process_srv_add_link(struct smc_link_group * lgr)1553 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1554 {
1555 struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1556 struct smc_llc_qentry *qentry;
1557 int rc;
1558
1559 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1560
1561 down_write(&lgr->llc_conf_mutex);
1562 rc = smc_llc_srv_add_link(link, qentry);
1563 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1564 /* delete any asymmetric link */
1565 smc_llc_delete_asym_link(lgr);
1566 }
1567 up_write(&lgr->llc_conf_mutex);
1568 kfree(qentry);
1569 }
1570
1571 /* enqueue a local add_link req to trigger a new add_link flow */
smc_llc_add_link_local(struct smc_link * link)1572 void smc_llc_add_link_local(struct smc_link *link)
1573 {
1574 struct smc_llc_msg_add_link add_llc = {};
1575
1576 add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1577 smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
1578 /* no dev and port needed */
1579 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc,
1580 sizeof(union smc_llc_msg));
1581 }
1582
1583 /* worker to process an add link message */
smc_llc_add_link_work(struct work_struct * work)1584 static void smc_llc_add_link_work(struct work_struct *work)
1585 {
1586 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1587 llc_add_link_work);
1588
1589 if (list_empty(&lgr->list)) {
1590 /* link group is terminating */
1591 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1592 goto out;
1593 }
1594
1595 if (lgr->role == SMC_CLNT)
1596 smc_llc_process_cli_add_link(lgr);
1597 else
1598 smc_llc_process_srv_add_link(lgr);
1599 out:
1600 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1601 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1602 }
1603
1604 /* enqueue a local del_link msg to trigger a new del_link flow,
1605 * called only for role SMC_SERV
1606 */
smc_llc_srv_delete_link_local(struct smc_link * link,u8 del_link_id)1607 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1608 {
1609 struct smc_llc_msg_del_link del_llc = {};
1610
1611 del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1612 smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
1613 del_llc.link_num = del_link_id;
1614 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1615 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1616 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc,
1617 sizeof(union smc_llc_msg));
1618 }
1619
smc_llc_process_cli_delete_link(struct smc_link_group * lgr)1620 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1621 {
1622 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1623 struct smc_llc_msg_del_link *del_llc;
1624 struct smc_llc_qentry *qentry;
1625 int active_links;
1626 int lnk_idx;
1627
1628 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1629 lnk = qentry->link;
1630 del_llc = &qentry->msg.delete_link;
1631
1632 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1633 smc_lgr_terminate_sched(lgr);
1634 goto out;
1635 }
1636 down_write(&lgr->llc_conf_mutex);
1637 /* delete single link */
1638 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1639 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1640 continue;
1641 lnk_del = &lgr->lnk[lnk_idx];
1642 break;
1643 }
1644 del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1645 if (!lnk_del) {
1646 /* link was not found */
1647 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1648 smc_llc_send_message(lnk, &qentry->msg);
1649 goto out_unlock;
1650 }
1651 lnk_asym = smc_llc_find_asym_link(lgr);
1652
1653 del_llc->reason = 0;
1654 smc_llc_send_message(lnk, &qentry->msg); /* response */
1655
1656 if (smc_link_downing(&lnk_del->state))
1657 smc_switch_conns(lgr, lnk_del, false);
1658 smcr_link_clear(lnk_del, true);
1659
1660 active_links = smc_llc_active_link_count(lgr);
1661 if (lnk_del == lnk_asym) {
1662 /* expected deletion of asym link, don't change lgr state */
1663 } else if (active_links == 1) {
1664 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1665 } else if (!active_links) {
1666 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1667 smc_lgr_terminate_sched(lgr);
1668 }
1669 out_unlock:
1670 up_write(&lgr->llc_conf_mutex);
1671 out:
1672 kfree(qentry);
1673 }
1674
1675 /* try to send a DELETE LINK ALL request on any active link,
1676 * waiting for send completion
1677 */
smc_llc_send_link_delete_all(struct smc_link_group * lgr,bool ord,u32 rsn)1678 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1679 {
1680 struct smc_llc_msg_del_link delllc = {};
1681 int i;
1682
1683 delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1684 smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
1685 if (ord)
1686 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1687 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1688 delllc.reason = htonl(rsn);
1689
1690 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1691 if (!smc_link_sendable(&lgr->lnk[i]))
1692 continue;
1693 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1694 break;
1695 }
1696 }
1697
smc_llc_process_srv_delete_link(struct smc_link_group * lgr)1698 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1699 {
1700 struct smc_llc_msg_del_link *del_llc;
1701 struct smc_link *lnk, *lnk_del;
1702 struct smc_llc_qentry *qentry;
1703 int active_links;
1704 int i;
1705
1706 down_write(&lgr->llc_conf_mutex);
1707 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1708 lnk = qentry->link;
1709 del_llc = &qentry->msg.delete_link;
1710
1711 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1712 /* delete entire lgr */
1713 smc_llc_send_link_delete_all(lgr, true, ntohl(
1714 qentry->msg.delete_link.reason));
1715 smc_lgr_terminate_sched(lgr);
1716 goto out;
1717 }
1718 /* delete single link */
1719 lnk_del = NULL;
1720 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1721 if (lgr->lnk[i].link_id == del_llc->link_num) {
1722 lnk_del = &lgr->lnk[i];
1723 break;
1724 }
1725 }
1726 if (!lnk_del)
1727 goto out; /* asymmetric link already deleted */
1728
1729 if (smc_link_downing(&lnk_del->state)) {
1730 if (smc_switch_conns(lgr, lnk_del, false))
1731 smc_wr_tx_wait_no_pending_sends(lnk_del);
1732 }
1733 if (!list_empty(&lgr->list)) {
1734 /* qentry is either a request from peer (send it back to
1735 * initiate the DELETE_LINK processing), or a locally
1736 * enqueued DELETE_LINK request (forward it)
1737 */
1738 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1739 struct smc_llc_qentry *qentry2;
1740
1741 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1742 SMC_LLC_DELETE_LINK);
1743 if (qentry2)
1744 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1745 }
1746 }
1747 smcr_link_clear(lnk_del, true);
1748
1749 active_links = smc_llc_active_link_count(lgr);
1750 if (active_links == 1) {
1751 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1752 } else if (!active_links) {
1753 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1754 smc_lgr_terminate_sched(lgr);
1755 }
1756
1757 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1758 /* trigger setup of asymm alt link */
1759 smc_llc_add_link_local(lnk);
1760 }
1761 out:
1762 up_write(&lgr->llc_conf_mutex);
1763 kfree(qentry);
1764 }
1765
smc_llc_delete_link_work(struct work_struct * work)1766 static void smc_llc_delete_link_work(struct work_struct *work)
1767 {
1768 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1769 llc_del_link_work);
1770
1771 if (list_empty(&lgr->list)) {
1772 /* link group is terminating */
1773 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1774 goto out;
1775 }
1776
1777 if (lgr->role == SMC_CLNT)
1778 smc_llc_process_cli_delete_link(lgr);
1779 else
1780 smc_llc_process_srv_delete_link(lgr);
1781 out:
1782 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1783 }
1784
1785 /* process a confirm_rkey request from peer, remote flow */
smc_llc_rmt_conf_rkey(struct smc_link_group * lgr)1786 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1787 {
1788 struct smc_llc_msg_confirm_rkey *llc;
1789 struct smc_llc_qentry *qentry;
1790 struct smc_link *link;
1791 int num_entries;
1792 int rk_idx;
1793 int i;
1794
1795 qentry = lgr->llc_flow_rmt.qentry;
1796 llc = &qentry->msg.confirm_rkey;
1797 link = qentry->link;
1798
1799 num_entries = llc->rtoken[0].num_rkeys;
1800 if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1801 goto out_err;
1802 /* first rkey entry is for receiving link */
1803 rk_idx = smc_rtoken_add(link,
1804 llc->rtoken[0].rmb_vaddr,
1805 llc->rtoken[0].rmb_key);
1806 if (rk_idx < 0)
1807 goto out_err;
1808
1809 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1810 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1811 llc->rtoken[i].rmb_vaddr,
1812 llc->rtoken[i].rmb_key);
1813 /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1814 goto out;
1815 out_err:
1816 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1817 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1818 out:
1819 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1820 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1821 smc_llc_send_message(link, &qentry->msg);
1822 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1823 }
1824
1825 /* process a delete_rkey request from peer, remote flow */
smc_llc_rmt_delete_rkey(struct smc_link_group * lgr)1826 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1827 {
1828 struct smc_llc_msg_delete_rkey *llc;
1829 struct smc_llc_qentry *qentry;
1830 struct smc_link *link;
1831 u8 err_mask = 0;
1832 int i, max;
1833
1834 qentry = lgr->llc_flow_rmt.qentry;
1835 llc = &qentry->msg.delete_rkey;
1836 link = qentry->link;
1837
1838 if (lgr->smc_version == SMC_V2) {
1839 struct smc_llc_msg_delete_rkey_v2_inline *llcv2;
1840
1841 /* The leading SMC_LLC_DEL_RKEY_V2_INLINE rkeys are declared in
1842 * the message itself, any further ones were received into
1843 * qentry->body.
1844 */
1845 llcv2 = &qentry->msg.delete_rkey_v2;
1846 llcv2->num_inval_rkeys = 0;
1847
1848 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1849 max = min_t(u32, max, SMC_LLC_DEL_RKEY_V2_INLINE +
1850 qentry->body_len / sizeof(__be32));
1851 for (i = 0; i < max; i++) {
1852 __be32 rkey;
1853
1854 if (i < SMC_LLC_DEL_RKEY_V2_INLINE)
1855 rkey = llcv2->rkey[i];
1856 else
1857 memcpy(&rkey, qentry->body +
1858 (i - SMC_LLC_DEL_RKEY_V2_INLINE) *
1859 sizeof(rkey), sizeof(rkey));
1860 if (smc_rtoken_delete(link, rkey))
1861 llcv2->num_inval_rkeys++;
1862 }
1863 memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1864 memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1865 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1866 if (llcv2->num_inval_rkeys) {
1867 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1868 llc->err_mask = llcv2->num_inval_rkeys;
1869 }
1870 goto finish;
1871 }
1872
1873 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1874 for (i = 0; i < max; i++) {
1875 if (smc_rtoken_delete(link, llc->rkey[i]))
1876 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1877 }
1878 if (err_mask) {
1879 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1880 llc->err_mask = err_mask;
1881 }
1882 finish:
1883 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1884 smc_llc_send_message(link, &qentry->msg);
1885 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1886 }
1887
smc_llc_protocol_violation(struct smc_link_group * lgr,u8 type)1888 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1889 {
1890 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1891 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1892 lgr->net->net_cookie, type);
1893 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1894 smc_lgr_terminate_sched(lgr);
1895 }
1896
1897 /* flush the llc event queue */
smc_llc_event_flush(struct smc_link_group * lgr)1898 static void smc_llc_event_flush(struct smc_link_group *lgr)
1899 {
1900 struct smc_llc_qentry *qentry, *q;
1901
1902 spin_lock_bh(&lgr->llc_event_q_lock);
1903 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1904 list_del_init(&qentry->list);
1905 kfree(qentry);
1906 }
1907 spin_unlock_bh(&lgr->llc_event_q_lock);
1908 }
1909
smc_llc_event_handler(struct smc_llc_qentry * qentry)1910 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1911 {
1912 union smc_llc_msg *llc = &qentry->msg;
1913 struct smc_link *link = qentry->link;
1914 struct smc_link_group *lgr = link->lgr;
1915
1916 if (!smc_link_usable(link))
1917 goto out;
1918
1919 switch (llc->raw.hdr.common.llc_type) {
1920 case SMC_LLC_TEST_LINK:
1921 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1922 smc_llc_send_message(link, llc);
1923 break;
1924 case SMC_LLC_ADD_LINK:
1925 if (list_empty(&lgr->list))
1926 goto out; /* lgr is terminating */
1927 if (lgr->role == SMC_CLNT) {
1928 if (smc_llc_is_local_add_link(llc)) {
1929 if (lgr->llc_flow_lcl.type ==
1930 SMC_LLC_FLOW_ADD_LINK)
1931 break; /* add_link in progress */
1932 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1933 qentry)) {
1934 schedule_work(&lgr->llc_add_link_work);
1935 }
1936 return;
1937 }
1938 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1939 !lgr->llc_flow_lcl.qentry) {
1940 /* a flow is waiting for this message */
1941 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1942 qentry);
1943 wake_up(&lgr->llc_msg_waiter);
1944 return;
1945 }
1946 if (lgr->llc_flow_lcl.type ==
1947 SMC_LLC_FLOW_REQ_ADD_LINK) {
1948 /* server started add_link processing */
1949 /* free any qentry stashed in REQ_ADD_LINK state */
1950 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1951 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1952 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1953 qentry);
1954 schedule_work(&lgr->llc_add_link_work);
1955 return;
1956 }
1957 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1958 schedule_work(&lgr->llc_add_link_work);
1959 }
1960 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1961 /* as smc server, handle client suggestion */
1962 schedule_work(&lgr->llc_add_link_work);
1963 }
1964 return;
1965 case SMC_LLC_CONFIRM_LINK:
1966 case SMC_LLC_ADD_LINK_CONT:
1967 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE &&
1968 !lgr->llc_flow_lcl.qentry) {
1969 /* a flow is waiting for this message */
1970 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1971 wake_up(&lgr->llc_msg_waiter);
1972 return;
1973 }
1974 break;
1975 case SMC_LLC_DELETE_LINK:
1976 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1977 !lgr->llc_flow_lcl.qentry) {
1978 /* DEL LINK REQ during ADD LINK SEQ */
1979 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1980 wake_up(&lgr->llc_msg_waiter);
1981 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1982 schedule_work(&lgr->llc_del_link_work);
1983 }
1984 return;
1985 case SMC_LLC_CONFIRM_RKEY:
1986 /* new request from remote, assign to remote flow */
1987 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1988 /* process here, does not wait for more llc msgs */
1989 smc_llc_rmt_conf_rkey(lgr);
1990 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1991 }
1992 return;
1993 case SMC_LLC_CONFIRM_RKEY_CONT:
1994 /* not used because max links is 3, and 3 rkeys fit into
1995 * one CONFIRM_RKEY message
1996 */
1997 break;
1998 case SMC_LLC_DELETE_RKEY:
1999 /* new request from remote, assign to remote flow */
2000 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
2001 /* process here, does not wait for more llc msgs */
2002 smc_llc_rmt_delete_rkey(lgr);
2003 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
2004 }
2005 return;
2006 case SMC_LLC_REQ_ADD_LINK:
2007 /* handle response here, smc_llc_flow_stop() cannot be called
2008 * in tasklet context
2009 */
2010 if (lgr->role == SMC_CLNT &&
2011 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
2012 (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
2013 smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
2014 } else if (lgr->role == SMC_SERV) {
2015 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
2016 /* as smc server, handle client suggestion */
2017 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
2018 schedule_work(&lgr->llc_add_link_work);
2019 }
2020 return;
2021 }
2022 break;
2023 default:
2024 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
2025 break;
2026 }
2027 out:
2028 kfree(qentry);
2029 }
2030
2031 /* worker to process llc messages on the event queue */
smc_llc_event_work(struct work_struct * work)2032 static void smc_llc_event_work(struct work_struct *work)
2033 {
2034 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
2035 llc_event_work);
2036 struct smc_llc_qentry *qentry;
2037
2038 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
2039 qentry = lgr->delayed_event;
2040 lgr->delayed_event = NULL;
2041 if (smc_link_usable(qentry->link))
2042 smc_llc_event_handler(qentry);
2043 else
2044 kfree(qentry);
2045 }
2046
2047 again:
2048 spin_lock_bh(&lgr->llc_event_q_lock);
2049 if (!list_empty(&lgr->llc_event_q)) {
2050 qentry = list_first_entry(&lgr->llc_event_q,
2051 struct smc_llc_qentry, list);
2052 list_del_init(&qentry->list);
2053 spin_unlock_bh(&lgr->llc_event_q_lock);
2054 smc_llc_event_handler(qentry);
2055 goto again;
2056 }
2057 spin_unlock_bh(&lgr->llc_event_q_lock);
2058 }
2059
2060 /* process llc responses in tasklet context */
smc_llc_rx_response(struct smc_link * link,struct smc_llc_qentry * qentry)2061 static void smc_llc_rx_response(struct smc_link *link,
2062 struct smc_llc_qentry *qentry)
2063 {
2064 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
2065 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
2066 u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
2067
2068 switch (llc_type) {
2069 case SMC_LLC_TEST_LINK:
2070 if (smc_link_active(link))
2071 complete(&link->llc_testlink_resp);
2072 break;
2073 case SMC_LLC_ADD_LINK:
2074 case SMC_LLC_ADD_LINK_CONT:
2075 case SMC_LLC_CONFIRM_LINK:
2076 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2077 break; /* drop out-of-flow response */
2078 goto assign;
2079 case SMC_LLC_DELETE_LINK:
2080 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2081 break; /* drop out-of-flow response */
2082 goto assign;
2083 case SMC_LLC_CONFIRM_RKEY:
2084 case SMC_LLC_DELETE_RKEY:
2085 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2086 break; /* drop out-of-flow response */
2087 goto assign;
2088 case SMC_LLC_CONFIRM_RKEY_CONT:
2089 /* not used because max links is 3 */
2090 break;
2091 default:
2092 smc_llc_protocol_violation(link->lgr,
2093 qentry->msg.raw.hdr.common.type);
2094 break;
2095 }
2096 kfree(qentry);
2097 return;
2098 assign:
2099 /* assign responses to the local flow, we requested them */
2100 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2101 wake_up(&link->lgr->llc_msg_waiter);
2102 }
2103
2104 /* the longest tail either reader of qentry->body can use */
smc_llc_max_body_len(union smc_llc_msg * llc)2105 static u32 smc_llc_max_body_len(union smc_llc_msg *llc)
2106 {
2107 switch (llc->raw.hdr.common.llc_type) {
2108 case SMC_LLC_ADD_LINK:
2109 return offsetof(struct smc_llc_msg_add_link_v2_ext, rt) +
2110 SMC_LLC_RKEYS_PER_MSG_V2 *
2111 sizeof(struct smc_llc_msg_add_link_cont_rt);
2112 case SMC_LLC_DELETE_RKEY:
2113 return (SMC_LLC_RKEYS_PER_MSG_V2 -
2114 SMC_LLC_DEL_RKEY_V2_INLINE) * sizeof(__be32);
2115 default:
2116 return 0;
2117 }
2118 }
2119
smc_llc_enqueue(struct smc_link * link,union smc_llc_msg * llc,u32 byte_len)2120 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc,
2121 u32 byte_len)
2122 {
2123 struct smc_link_group *lgr = link->lgr;
2124 struct smc_llc_qentry *qentry;
2125 unsigned long flags;
2126 u16 body_len = 0;
2127
2128 /* V2 messages can be longer than the inline union smc_llc_msg. Carry
2129 * the remainder in the qentry itself, so that its lifetime and its
2130 * length match the message the peer actually sent.
2131 */
2132 if (lgr->smc_version == SMC_V2 && byte_len > SMC_WR_TX_SIZE)
2133 body_len = min_t(u32, byte_len, SMC_WR_BUF_V2_SIZE) -
2134 SMC_WR_TX_SIZE;
2135 body_len = min_t(u32, body_len, smc_llc_max_body_len(llc));
2136
2137 qentry = kmalloc_flex(*qentry, body, body_len, GFP_ATOMIC);
2138 if (!qentry)
2139 return;
2140 qentry->body_len = body_len;
2141 qentry->link = link;
2142 INIT_LIST_HEAD(&qentry->list);
2143 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
2144 if (body_len) {
2145 u8 *src = smc_link_shared_v2_rxbuf(link) ?
2146 (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc;
2147
2148 memcpy(qentry->body, src + SMC_WR_TX_SIZE, body_len);
2149 }
2150
2151 /* process responses immediately */
2152 if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2153 llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
2154 smc_llc_rx_response(link, qentry);
2155 return;
2156 }
2157
2158 /* add requests to event queue */
2159 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2160 list_add_tail(&qentry->list, &lgr->llc_event_q);
2161 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
2162 queue_work(system_highpri_wq, &lgr->llc_event_work);
2163 }
2164
2165 /* copy received msg and add it to the event queue */
smc_llc_rx_handler(struct ib_wc * wc,void * buf)2166 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2167 {
2168 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2169 union smc_llc_msg *llc = buf;
2170
2171 if (wc->byte_len < sizeof(*llc))
2172 return; /* short message */
2173 if (!llc->raw.hdr.common.llc_version) {
2174 if (llc->raw.hdr.length != sizeof(*llc))
2175 return; /* invalid message */
2176 } else {
2177 if (llc->raw.hdr.length_v2 < sizeof(*llc))
2178 return; /* invalid message */
2179 }
2180
2181 smc_llc_enqueue(link, llc, wc->byte_len);
2182 }
2183
2184 /***************************** worker, utils *********************************/
2185
smc_llc_testlink_work(struct work_struct * work)2186 static void smc_llc_testlink_work(struct work_struct *work)
2187 {
2188 struct smc_link *link = container_of(to_delayed_work(work),
2189 struct smc_link, llc_testlink_wrk);
2190 unsigned long next_interval;
2191 unsigned long expire_time;
2192 u8 user_data[16] = { 0 };
2193 int rc;
2194
2195 if (!smc_link_active(link))
2196 return; /* don't reschedule worker */
2197 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2198 if (time_is_after_jiffies(expire_time)) {
2199 next_interval = expire_time - jiffies;
2200 goto out;
2201 }
2202 reinit_completion(&link->llc_testlink_resp);
2203 smc_llc_send_test_link(link, user_data);
2204 /* receive TEST LINK response over RoCE fabric */
2205 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2206 SMC_LLC_WAIT_TIME);
2207 if (!smc_link_active(link))
2208 return; /* link state changed */
2209 if (rc <= 0) {
2210 smcr_link_down_cond_sched(link);
2211 return;
2212 }
2213 next_interval = link->llc_testlink_time;
2214 out:
2215 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
2216 }
2217
smc_llc_lgr_init(struct smc_link_group * lgr,struct smc_sock * smc)2218 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2219 {
2220 struct net *net = sock_net(smc->clcsock->sk);
2221
2222 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
2223 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
2224 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
2225 INIT_LIST_HEAD(&lgr->llc_event_q);
2226 spin_lock_init(&lgr->llc_event_q_lock);
2227 spin_lock_init(&lgr->llc_flow_lock);
2228 init_waitqueue_head(&lgr->llc_flow_waiter);
2229 init_waitqueue_head(&lgr->llc_msg_waiter);
2230 init_rwsem(&lgr->llc_conf_mutex);
2231 lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
2232 lgr->max_send_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_send_wr));
2233 lgr->max_recv_wr = (u16)(READ_ONCE(net->smc.sysctl_smcr_max_recv_wr));
2234 }
2235
2236 /* called after lgr was removed from lgr_list */
smc_llc_lgr_clear(struct smc_link_group * lgr)2237 void smc_llc_lgr_clear(struct smc_link_group *lgr)
2238 {
2239 smc_llc_event_flush(lgr);
2240 wake_up_all(&lgr->llc_flow_waiter);
2241 wake_up_all(&lgr->llc_msg_waiter);
2242 cancel_work_sync(&lgr->llc_event_work);
2243 cancel_work_sync(&lgr->llc_add_link_work);
2244 cancel_work_sync(&lgr->llc_del_link_work);
2245 if (lgr->delayed_event) {
2246 kfree(lgr->delayed_event);
2247 lgr->delayed_event = NULL;
2248 }
2249 }
2250
smc_llc_link_init(struct smc_link * link)2251 int smc_llc_link_init(struct smc_link *link)
2252 {
2253 init_completion(&link->llc_testlink_resp);
2254 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2255 return 0;
2256 }
2257
smc_llc_link_active(struct smc_link * link)2258 void smc_llc_link_active(struct smc_link *link)
2259 {
2260 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
2261 "peerid %*phN, ibdev %s, ibport %d\n",
2262 SMC_LGR_ID_SIZE, &link->lgr->id,
2263 link->lgr->net->net_cookie,
2264 SMC_LGR_ID_SIZE, &link->link_uid,
2265 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2266 link->smcibdev->ibdev->name, link->ibport);
2267 link->state = SMC_LNK_ACTIVE;
2268 if (link->lgr->llc_testlink_time) {
2269 link->llc_testlink_time = link->lgr->llc_testlink_time;
2270 schedule_delayed_work(&link->llc_testlink_wrk,
2271 link->llc_testlink_time);
2272 }
2273 }
2274
2275 /* called in worker context */
smc_llc_link_clear(struct smc_link * link,bool log)2276 void smc_llc_link_clear(struct smc_link *link, bool log)
2277 {
2278 if (log)
2279 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
2280 ", peerid %*phN, ibdev %s, ibport %d\n",
2281 SMC_LGR_ID_SIZE, &link->lgr->id,
2282 link->lgr->net->net_cookie,
2283 SMC_LGR_ID_SIZE, &link->link_uid,
2284 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2285 link->smcibdev->ibdev->name, link->ibport);
2286 complete(&link->llc_testlink_resp);
2287 cancel_delayed_work_sync(&link->llc_testlink_wrk);
2288 }
2289
2290 /* register a new rtoken at the remote peer (for all links) */
smc_llc_do_confirm_rkey(struct smc_link * send_link,struct smc_buf_desc * rmb_desc)2291 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
2292 struct smc_buf_desc *rmb_desc)
2293 {
2294 struct smc_link_group *lgr = send_link->lgr;
2295 struct smc_llc_qentry *qentry = NULL;
2296 int rc = 0;
2297
2298 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2299 if (rc)
2300 goto out;
2301 /* receive CONFIRM RKEY response from server over RoCE fabric */
2302 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2303 SMC_LLC_CONFIRM_RKEY);
2304 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2305 rc = -EFAULT;
2306 out:
2307 if (qentry)
2308 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2309 return rc;
2310 }
2311
2312 /* unregister an rtoken at the remote peer */
smc_llc_do_delete_rkey(struct smc_link_group * lgr,struct smc_buf_desc * rmb_desc)2313 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
2314 struct smc_buf_desc *rmb_desc)
2315 {
2316 struct smc_llc_qentry *qentry = NULL;
2317 struct smc_link *send_link;
2318 int rc = 0;
2319
2320 send_link = smc_llc_usable_link(lgr);
2321 if (!send_link)
2322 return -ENOLINK;
2323
2324 /* protected by llc_flow control */
2325 rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
2326 if (rc)
2327 goto out;
2328 /* receive DELETE RKEY response from server over RoCE fabric */
2329 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2330 SMC_LLC_DELETE_RKEY);
2331 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2332 rc = -EFAULT;
2333 out:
2334 if (qentry)
2335 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
2336 return rc;
2337 }
2338
smc_llc_link_set_uid(struct smc_link * link)2339 void smc_llc_link_set_uid(struct smc_link *link)
2340 {
2341 __be32 link_uid;
2342
2343 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2344 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2345 }
2346
2347 /* save peers link user id, used for debug purposes */
smc_llc_save_peer_uid(struct smc_llc_qentry * qentry)2348 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2349 {
2350 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2351 SMC_LGR_ID_SIZE);
2352 }
2353
2354 /* evaluate confirm link request or response */
smc_llc_eval_conf_link(struct smc_llc_qentry * qentry,enum smc_llc_reqresp type)2355 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2356 enum smc_llc_reqresp type)
2357 {
2358 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */
2359 qentry->link->link_id = qentry->msg.confirm_link.link_num;
2360 smc_llc_link_set_uid(qentry->link);
2361 }
2362 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2363 return -ENOTSUPP;
2364 return 0;
2365 }
2366
2367 /***************************** init, exit, misc ******************************/
2368
2369 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2370 {
2371 .handler = smc_llc_rx_handler,
2372 .type = SMC_LLC_CONFIRM_LINK
2373 },
2374 {
2375 .handler = smc_llc_rx_handler,
2376 .type = SMC_LLC_TEST_LINK
2377 },
2378 {
2379 .handler = smc_llc_rx_handler,
2380 .type = SMC_LLC_ADD_LINK
2381 },
2382 {
2383 .handler = smc_llc_rx_handler,
2384 .type = SMC_LLC_ADD_LINK_CONT
2385 },
2386 {
2387 .handler = smc_llc_rx_handler,
2388 .type = SMC_LLC_DELETE_LINK
2389 },
2390 {
2391 .handler = smc_llc_rx_handler,
2392 .type = SMC_LLC_CONFIRM_RKEY
2393 },
2394 {
2395 .handler = smc_llc_rx_handler,
2396 .type = SMC_LLC_CONFIRM_RKEY_CONT
2397 },
2398 {
2399 .handler = smc_llc_rx_handler,
2400 .type = SMC_LLC_DELETE_RKEY
2401 },
2402 /* V2 types */
2403 {
2404 .handler = smc_llc_rx_handler,
2405 .type = SMC_LLC_CONFIRM_LINK_V2
2406 },
2407 {
2408 .handler = smc_llc_rx_handler,
2409 .type = SMC_LLC_TEST_LINK_V2
2410 },
2411 {
2412 .handler = smc_llc_rx_handler,
2413 .type = SMC_LLC_ADD_LINK_V2
2414 },
2415 {
2416 .handler = smc_llc_rx_handler,
2417 .type = SMC_LLC_DELETE_LINK_V2
2418 },
2419 {
2420 .handler = smc_llc_rx_handler,
2421 .type = SMC_LLC_REQ_ADD_LINK_V2
2422 },
2423 {
2424 .handler = smc_llc_rx_handler,
2425 .type = SMC_LLC_CONFIRM_RKEY_V2
2426 },
2427 {
2428 .handler = smc_llc_rx_handler,
2429 .type = SMC_LLC_DELETE_RKEY_V2
2430 },
2431 {
2432 .handler = NULL,
2433 }
2434 };
2435
smc_llc_init(void)2436 int __init smc_llc_init(void)
2437 {
2438 struct smc_wr_rx_handler *handler;
2439 int rc = 0;
2440
2441 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2442 INIT_HLIST_NODE(&handler->list);
2443 rc = smc_wr_rx_register_handler(handler);
2444 if (rc)
2445 break;
2446 }
2447 return rc;
2448 }
2449