1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2018 Chelsio Communications, Inc.
4 *
5 * Written by: Atul Gupta (atul.gupta@chelsio.com)
6 */
7
8 #include <linux/module.h>
9 #include <linux/list.h>
10 #include <linux/workqueue.h>
11 #include <linux/skbuff.h>
12 #include <linux/timer.h>
13 #include <linux/notifier.h>
14 #include <linux/inetdevice.h>
15 #include <linux/ip.h>
16 #include <linux/tcp.h>
17 #include <linux/sched/signal.h>
18 #include <linux/kallsyms.h>
19 #include <linux/kprobes.h>
20 #include <linux/if_vlan.h>
21 #include <linux/ipv6.h>
22 #include <net/ipv6.h>
23 #include <net/transp_v6.h>
24 #include <net/ip6_route.h>
25 #include <net/inet_common.h>
26 #include <net/tcp.h>
27 #include <net/dst.h>
28 #include <net/tls.h>
29 #include <net/addrconf.h>
30 #include <net/secure_seq.h>
31
32 #include "chtls.h"
33 #include "chtls_cm.h"
34 #include "clip_tbl.h"
35 #include "t4_tcb.h"
36
37 /*
38 * State transitions and actions for close. Note that if we are in SYN_SENT
39 * we remain in that state as we cannot control a connection while it's in
40 * SYN_SENT; such connections are allowed to establish and are then aborted.
41 */
42 static unsigned char new_state[16] = {
43 /* current state: new state: action: */
44 /* (Invalid) */ TCP_CLOSE,
45 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
46 /* TCP_SYN_SENT */ TCP_SYN_SENT,
47 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
48 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
49 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
50 /* TCP_TIME_WAIT */ TCP_CLOSE,
51 /* TCP_CLOSE */ TCP_CLOSE,
52 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
53 /* TCP_LAST_ACK */ TCP_LAST_ACK,
54 /* TCP_LISTEN */ TCP_CLOSE,
55 /* TCP_CLOSING */ TCP_CLOSING,
56 };
57
chtls_sock_create(struct chtls_dev * cdev)58 static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
59 {
60 struct chtls_sock *csk = kzalloc_obj(*csk, GFP_ATOMIC);
61
62 if (!csk)
63 return NULL;
64
65 csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
66 if (!csk->txdata_skb_cache) {
67 kfree(csk);
68 return NULL;
69 }
70
71 kref_init(&csk->kref);
72 csk->cdev = cdev;
73 skb_queue_head_init(&csk->txq);
74 csk->wr_skb_head = NULL;
75 csk->wr_skb_tail = NULL;
76 csk->mss = MAX_MSS;
77 csk->tlshws.ofld = 1;
78 csk->tlshws.txkey = -1;
79 csk->tlshws.rxkey = -1;
80 csk->tlshws.mfs = TLS_MFS;
81 skb_queue_head_init(&csk->tlshws.sk_recv_queue);
82 return csk;
83 }
84
chtls_sock_release(struct kref * ref)85 static void chtls_sock_release(struct kref *ref)
86 {
87 struct chtls_sock *csk =
88 container_of(ref, struct chtls_sock, kref);
89
90 kfree(csk);
91 }
92
chtls_find_netdev(struct chtls_dev * cdev,struct sock * sk)93 static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
94 struct sock *sk)
95 {
96 struct adapter *adap = pci_get_drvdata(cdev->pdev);
97 struct net_device *ndev = cdev->ports[0];
98 #if IS_ENABLED(CONFIG_IPV6)
99 struct net_device *temp;
100 int addr_type;
101 #endif
102 int i;
103
104 switch (sk->sk_family) {
105 case PF_INET:
106 if (likely(!inet_sk(sk)->inet_rcv_saddr))
107 return ndev;
108 ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
109 break;
110 #if IS_ENABLED(CONFIG_IPV6)
111 case PF_INET6:
112 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
113 if (likely(addr_type == IPV6_ADDR_ANY))
114 return ndev;
115
116 for_each_netdev_rcu(&init_net, temp) {
117 if (ipv6_chk_addr(&init_net, (struct in6_addr *)
118 &sk->sk_v6_rcv_saddr, temp, 1)) {
119 ndev = temp;
120 break;
121 }
122 }
123 break;
124 #endif
125 default:
126 return NULL;
127 }
128
129 if (!ndev)
130 return NULL;
131
132 if (is_vlan_dev(ndev))
133 ndev = vlan_dev_real_dev(ndev);
134
135 for_each_port(adap, i)
136 if (cdev->ports[i] == ndev)
137 return ndev;
138 return NULL;
139 }
140
assign_rxopt(struct sock * sk,unsigned int opt)141 static void assign_rxopt(struct sock *sk, unsigned int opt)
142 {
143 const struct chtls_dev *cdev;
144 struct chtls_sock *csk;
145 struct tcp_sock *tp;
146
147 csk = rcu_dereference_sk_user_data(sk);
148 tp = tcp_sk(sk);
149
150 cdev = csk->cdev;
151 tp->tcp_header_len = sizeof(struct tcphdr);
152 tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
153 tp->mss_cache = tp->rx_opt.mss_clamp;
154 tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
155 tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt);
156 tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
157 SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
158 if (!tp->rx_opt.wscale_ok)
159 tp->rx_opt.rcv_wscale = 0;
160 if (tp->rx_opt.tstamp_ok) {
161 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
162 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
163 } else if (csk->opt2 & TSTAMPS_EN_F) {
164 csk->opt2 &= ~TSTAMPS_EN_F;
165 csk->mtu_idx = TCPOPT_MSS_G(opt);
166 }
167 }
168
chtls_purge_receive_queue(struct sock * sk)169 static void chtls_purge_receive_queue(struct sock *sk)
170 {
171 struct sk_buff *skb;
172
173 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
174 skb_dstref_steal(skb);
175 kfree_skb(skb);
176 }
177 }
178
chtls_purge_write_queue(struct sock * sk)179 static void chtls_purge_write_queue(struct sock *sk)
180 {
181 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
182 struct sk_buff *skb;
183
184 while ((skb = __skb_dequeue(&csk->txq))) {
185 sk->sk_wmem_queued -= skb->truesize;
186 __kfree_skb(skb);
187 }
188 }
189
chtls_purge_recv_queue(struct sock * sk)190 static void chtls_purge_recv_queue(struct sock *sk)
191 {
192 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
193 struct chtls_hws *tlsk = &csk->tlshws;
194 struct sk_buff *skb;
195
196 while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
197 skb_dstref_steal(skb);
198 kfree_skb(skb);
199 }
200 }
201
abort_arp_failure(void * handle,struct sk_buff * skb)202 static void abort_arp_failure(void *handle, struct sk_buff *skb)
203 {
204 struct cpl_abort_req *req = cplhdr(skb);
205 struct chtls_dev *cdev;
206
207 cdev = (struct chtls_dev *)handle;
208 req->cmd = CPL_ABORT_NO_RST;
209 cxgb4_ofld_send(cdev->lldi->ports[0], skb);
210 }
211
alloc_ctrl_skb(struct sk_buff * skb,int len)212 static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
213 {
214 if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
215 __skb_trim(skb, 0);
216 refcount_inc(&skb->users);
217 } else {
218 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
219 }
220 return skb;
221 }
222
chtls_send_abort(struct sock * sk,int mode,struct sk_buff * skb)223 static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
224 {
225 struct cpl_abort_req *req;
226 struct chtls_sock *csk;
227 struct tcp_sock *tp;
228
229 csk = rcu_dereference_sk_user_data(sk);
230 tp = tcp_sk(sk);
231
232 if (!skb)
233 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
234
235 req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
236 INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
237 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
238 req->rsvd0 = htonl(tp->snd_nxt);
239 req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
240 req->cmd = mode;
241 t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
242 send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
243 }
244
chtls_send_reset(struct sock * sk,int mode,struct sk_buff * skb)245 static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
246 {
247 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
248
249 if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
250 !csk->cdev)) {
251 if (sk->sk_state == TCP_SYN_RECV)
252 csk_set_flag(csk, CSK_RST_ABORTED);
253 goto out;
254 }
255
256 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
257 struct tcp_sock *tp = tcp_sk(sk);
258
259 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
260 WARN_ONCE(1, "send tx flowc error");
261 csk_set_flag(csk, CSK_TX_DATA_SENT);
262 }
263
264 csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
265 chtls_purge_write_queue(sk);
266
267 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
268 if (sk->sk_state != TCP_SYN_RECV)
269 chtls_send_abort(sk, mode, skb);
270 else
271 chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W,
272 TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0,
273 TCB_FIELD_COOKIE_TFLAG, 1);
274
275 return;
276 out:
277 kfree_skb(skb);
278 }
279
release_tcp_port(struct sock * sk)280 static void release_tcp_port(struct sock *sk)
281 {
282 if (inet_csk(sk)->icsk_bind_hash)
283 inet_put_port(sk);
284 }
285
tcp_uncork(struct sock * sk)286 static void tcp_uncork(struct sock *sk)
287 {
288 struct tcp_sock *tp = tcp_sk(sk);
289
290 if (tp->nonagle & TCP_NAGLE_CORK) {
291 tp->nonagle &= ~TCP_NAGLE_CORK;
292 chtls_tcp_push(sk, 0);
293 }
294 }
295
chtls_close_conn(struct sock * sk)296 static void chtls_close_conn(struct sock *sk)
297 {
298 struct cpl_close_con_req *req;
299 struct chtls_sock *csk;
300 struct sk_buff *skb;
301 unsigned int tid;
302 unsigned int len;
303
304 len = roundup(sizeof(struct cpl_close_con_req), 16);
305 csk = rcu_dereference_sk_user_data(sk);
306 tid = csk->tid;
307
308 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
309 req = (struct cpl_close_con_req *)__skb_put(skb, len);
310 memset(req, 0, len);
311 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
312 FW_WR_IMMDLEN_V(sizeof(*req) -
313 sizeof(req->wr)));
314 req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
315 FW_WR_FLOWID_V(tid));
316
317 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
318
319 tcp_uncork(sk);
320 skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
321 if (sk->sk_state != TCP_SYN_SENT)
322 chtls_push_frames(csk, 1);
323 }
324
325 /*
326 * Perform a state transition during close and return the actions indicated
327 * for the transition. Do not make this function inline, the main reason
328 * it exists at all is to avoid multiple inlining of tcp_set_state.
329 */
make_close_transition(struct sock * sk)330 static int make_close_transition(struct sock *sk)
331 {
332 int next = (int)new_state[sk->sk_state];
333
334 tcp_set_state(sk, next & TCP_STATE_MASK);
335 return next & TCP_ACTION_FIN;
336 }
337
chtls_close(struct sock * sk,long timeout)338 void chtls_close(struct sock *sk, long timeout)
339 {
340 int data_lost, prev_state;
341 struct chtls_sock *csk;
342
343 csk = rcu_dereference_sk_user_data(sk);
344
345 lock_sock(sk);
346 sk->sk_shutdown |= SHUTDOWN_MASK;
347
348 data_lost = skb_queue_len(&sk->sk_receive_queue);
349 data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
350 chtls_purge_recv_queue(sk);
351 chtls_purge_receive_queue(sk);
352
353 if (sk->sk_state == TCP_CLOSE) {
354 goto wait;
355 } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
356 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
357 release_tcp_port(sk);
358 goto unlock;
359 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
360 sk->sk_prot->disconnect(sk, 0);
361 } else if (make_close_transition(sk)) {
362 chtls_close_conn(sk);
363 }
364 wait:
365 if (timeout)
366 sk_stream_wait_close(sk, timeout);
367
368 unlock:
369 prev_state = sk->sk_state;
370 sock_hold(sk);
371 sock_orphan(sk);
372
373 release_sock(sk);
374
375 local_bh_disable();
376 bh_lock_sock(sk);
377
378 if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
379 goto out;
380
381 if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
382 !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
383 struct sk_buff *skb;
384
385 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
386 if (skb)
387 chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
388 }
389
390 if (sk->sk_state == TCP_CLOSE)
391 inet_csk_destroy_sock(sk);
392
393 out:
394 bh_unlock_sock(sk);
395 local_bh_enable();
396 sock_put(sk);
397 }
398
399 /*
400 * Wait until a socket enters on of the given states.
401 */
wait_for_states(struct sock * sk,unsigned int states)402 static int wait_for_states(struct sock *sk, unsigned int states)
403 {
404 DECLARE_WAITQUEUE(wait, current);
405 struct socket_wq _sk_wq;
406 long current_timeo;
407 int err = 0;
408
409 current_timeo = 200;
410
411 /*
412 * We want this to work even when there's no associated struct socket.
413 * In that case we provide a temporary wait_queue_head_t.
414 */
415 if (!sk->sk_wq) {
416 init_waitqueue_head(&_sk_wq.wait);
417 _sk_wq.fasync_list = NULL;
418 init_rcu_head_on_stack(&_sk_wq.rcu);
419 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
420 }
421
422 add_wait_queue(sk_sleep(sk), &wait);
423 while (!sk_in_state(sk, states)) {
424 if (!current_timeo) {
425 err = -EBUSY;
426 break;
427 }
428 if (signal_pending(current)) {
429 err = sock_intr_errno(current_timeo);
430 break;
431 }
432 set_current_state(TASK_UNINTERRUPTIBLE);
433 release_sock(sk);
434 if (!sk_in_state(sk, states))
435 current_timeo = schedule_timeout(current_timeo);
436 __set_current_state(TASK_RUNNING);
437 lock_sock(sk);
438 }
439 remove_wait_queue(sk_sleep(sk), &wait);
440
441 if (rcu_dereference(sk->sk_wq) == &_sk_wq)
442 sk->sk_wq = NULL;
443 return err;
444 }
445
chtls_disconnect(struct sock * sk,int flags)446 int chtls_disconnect(struct sock *sk, int flags)
447 {
448 struct tcp_sock *tp;
449 int err;
450
451 tp = tcp_sk(sk);
452 chtls_purge_recv_queue(sk);
453 chtls_purge_receive_queue(sk);
454 chtls_purge_write_queue(sk);
455
456 if (sk->sk_state != TCP_CLOSE) {
457 sk->sk_err = ECONNRESET;
458 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
459 err = wait_for_states(sk, TCPF_CLOSE);
460 if (err)
461 return err;
462 }
463 chtls_purge_recv_queue(sk);
464 chtls_purge_receive_queue(sk);
465 tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
466 return tcp_disconnect(sk, flags);
467 }
468
469 #define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
470 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
chtls_shutdown(struct sock * sk,int how)471 void chtls_shutdown(struct sock *sk, int how)
472 {
473 if ((how & SEND_SHUTDOWN) &&
474 sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
475 make_close_transition(sk))
476 chtls_close_conn(sk);
477 }
478
chtls_destroy_sock(struct sock * sk)479 void chtls_destroy_sock(struct sock *sk)
480 {
481 struct chtls_sock *csk;
482
483 csk = rcu_dereference_sk_user_data(sk);
484 chtls_purge_recv_queue(sk);
485 csk->ulp_mode = ULP_MODE_NONE;
486 chtls_purge_write_queue(sk);
487 free_tls_keyid(sk);
488 kref_put(&csk->kref, chtls_sock_release);
489 if (sk->sk_family == AF_INET)
490 sk->sk_prot = &tcp_prot;
491 #if IS_ENABLED(CONFIG_IPV6)
492 else
493 sk->sk_prot = &tcpv6_prot;
494 #endif
495 sk->sk_prot->destroy(sk);
496 }
497
reset_listen_child(struct sock * child)498 static void reset_listen_child(struct sock *child)
499 {
500 struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
501 struct sk_buff *skb;
502
503 skb = alloc_ctrl_skb(csk->txdata_skb_cache,
504 sizeof(struct cpl_abort_req));
505
506 chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
507 sock_orphan(child);
508 tcp_orphan_count_inc();
509 if (child->sk_state == TCP_CLOSE)
510 inet_csk_destroy_sock(child);
511 }
512
chtls_disconnect_acceptq(struct sock * listen_sk)513 static void chtls_disconnect_acceptq(struct sock *listen_sk)
514 {
515 struct request_sock **pprev;
516
517 pprev = ACCEPT_QUEUE(listen_sk);
518 while (*pprev) {
519 struct request_sock *req = *pprev;
520
521 if (req->rsk_ops == &chtls_rsk_ops ||
522 req->rsk_ops == &chtls_rsk_opsv6) {
523 struct sock *child = req->sk;
524
525 *pprev = req->dl_next;
526 sk_acceptq_removed(listen_sk);
527 reqsk_put(req);
528 sock_hold(child);
529 local_bh_disable();
530 bh_lock_sock(child);
531 release_tcp_port(child);
532 reset_listen_child(child);
533 bh_unlock_sock(child);
534 local_bh_enable();
535 sock_put(child);
536 } else {
537 pprev = &req->dl_next;
538 }
539 }
540 }
541
listen_hashfn(const struct sock * sk)542 static int listen_hashfn(const struct sock *sk)
543 {
544 return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
545 }
546
listen_hash_add(struct chtls_dev * cdev,struct sock * sk,unsigned int stid)547 static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
548 struct sock *sk,
549 unsigned int stid)
550 {
551 struct listen_info *p = kmalloc_obj(*p);
552
553 if (p) {
554 int key = listen_hashfn(sk);
555
556 p->sk = sk;
557 p->stid = stid;
558 spin_lock(&cdev->listen_lock);
559 p->next = cdev->listen_hash_tab[key];
560 cdev->listen_hash_tab[key] = p;
561 spin_unlock(&cdev->listen_lock);
562 }
563 return p;
564 }
565
listen_hash_find(struct chtls_dev * cdev,struct sock * sk)566 static int listen_hash_find(struct chtls_dev *cdev,
567 struct sock *sk)
568 {
569 struct listen_info *p;
570 int stid = -1;
571 int key;
572
573 key = listen_hashfn(sk);
574
575 spin_lock(&cdev->listen_lock);
576 for (p = cdev->listen_hash_tab[key]; p; p = p->next)
577 if (p->sk == sk) {
578 stid = p->stid;
579 break;
580 }
581 spin_unlock(&cdev->listen_lock);
582 return stid;
583 }
584
listen_hash_del(struct chtls_dev * cdev,struct sock * sk)585 static int listen_hash_del(struct chtls_dev *cdev,
586 struct sock *sk)
587 {
588 struct listen_info *p, **prev;
589 int stid = -1;
590 int key;
591
592 key = listen_hashfn(sk);
593 prev = &cdev->listen_hash_tab[key];
594
595 spin_lock(&cdev->listen_lock);
596 for (p = *prev; p; prev = &p->next, p = p->next)
597 if (p->sk == sk) {
598 stid = p->stid;
599 *prev = p->next;
600 kfree(p);
601 break;
602 }
603 spin_unlock(&cdev->listen_lock);
604 return stid;
605 }
606
cleanup_syn_rcv_conn(struct sock * child,struct sock * parent)607 static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
608 {
609 struct request_sock *req;
610 struct chtls_sock *csk;
611
612 csk = rcu_dereference_sk_user_data(child);
613 req = csk->passive_reap_next;
614
615 reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
616 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
617 chtls_reqsk_free(req);
618 csk->passive_reap_next = NULL;
619 }
620
chtls_reset_synq(struct listen_ctx * listen_ctx)621 static void chtls_reset_synq(struct listen_ctx *listen_ctx)
622 {
623 struct sock *listen_sk = listen_ctx->lsk;
624
625 while (!skb_queue_empty(&listen_ctx->synq)) {
626 struct chtls_sock *csk =
627 container_of((struct synq *)skb_peek
628 (&listen_ctx->synq), struct chtls_sock, synq);
629 struct sock *child = csk->sk;
630
631 cleanup_syn_rcv_conn(child, listen_sk);
632 sock_hold(child);
633 local_bh_disable();
634 bh_lock_sock(child);
635 release_tcp_port(child);
636 reset_listen_child(child);
637 bh_unlock_sock(child);
638 local_bh_enable();
639 sock_put(child);
640 }
641 }
642
chtls_listen_start(struct chtls_dev * cdev,struct sock * sk)643 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
644 {
645 struct net_device *ndev;
646 #if IS_ENABLED(CONFIG_IPV6)
647 bool clip_valid = false;
648 #endif
649 struct listen_ctx *ctx;
650 struct adapter *adap;
651 struct port_info *pi;
652 int ret = 0;
653 int stid;
654
655 rcu_read_lock();
656 ndev = chtls_find_netdev(cdev, sk);
657 rcu_read_unlock();
658 if (!ndev)
659 return -EBADF;
660
661 pi = netdev_priv(ndev);
662 adap = pi->adapter;
663 if (!(adap->flags & CXGB4_FULL_INIT_DONE))
664 return -EBADF;
665
666 if (listen_hash_find(cdev, sk) >= 0) /* already have it */
667 return -EADDRINUSE;
668
669 ctx = kmalloc_obj(*ctx);
670 if (!ctx)
671 return -ENOMEM;
672
673 __module_get(THIS_MODULE);
674 ctx->lsk = sk;
675 ctx->cdev = cdev;
676 ctx->state = T4_LISTEN_START_PENDING;
677 skb_queue_head_init(&ctx->synq);
678
679 stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
680 if (stid < 0)
681 goto free_ctx;
682
683 sock_hold(sk);
684 if (!listen_hash_add(cdev, sk, stid))
685 goto free_stid;
686
687 if (sk->sk_family == PF_INET) {
688 ret = cxgb4_create_server(ndev, stid,
689 inet_sk(sk)->inet_rcv_saddr,
690 inet_sk(sk)->inet_sport, 0,
691 cdev->lldi->rxq_ids[0]);
692 #if IS_ENABLED(CONFIG_IPV6)
693 } else {
694 int addr_type;
695
696 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
697 if (addr_type != IPV6_ADDR_ANY) {
698 ret = cxgb4_clip_get(ndev, (const u32 *)
699 &sk->sk_v6_rcv_saddr, 1);
700 if (ret)
701 goto del_hash;
702 clip_valid = true;
703 }
704 ret = cxgb4_create_server6(ndev, stid,
705 &sk->sk_v6_rcv_saddr,
706 inet_sk(sk)->inet_sport,
707 cdev->lldi->rxq_ids[0]);
708 #endif
709 }
710 if (ret > 0)
711 ret = net_xmit_errno(ret);
712 if (ret)
713 goto del_hash;
714 return 0;
715 del_hash:
716 #if IS_ENABLED(CONFIG_IPV6)
717 if (clip_valid)
718 cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
719 #endif
720 listen_hash_del(cdev, sk);
721 free_stid:
722 cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
723 sock_put(sk);
724 free_ctx:
725 kfree(ctx);
726 module_put(THIS_MODULE);
727 return -EBADF;
728 }
729
chtls_listen_stop(struct chtls_dev * cdev,struct sock * sk)730 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
731 {
732 struct listen_ctx *listen_ctx;
733 int stid;
734
735 stid = listen_hash_del(cdev, sk);
736 if (stid < 0)
737 return;
738
739 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
740 chtls_reset_synq(listen_ctx);
741
742 cxgb4_remove_server(cdev->lldi->ports[0], stid,
743 cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
744
745 #if IS_ENABLED(CONFIG_IPV6)
746 if (sk->sk_family == PF_INET6) {
747 struct net_device *ndev = chtls_find_netdev(cdev, sk);
748 int addr_type = 0;
749
750 addr_type = ipv6_addr_type((const struct in6_addr *)
751 &sk->sk_v6_rcv_saddr);
752 if (addr_type != IPV6_ADDR_ANY)
753 cxgb4_clip_release(ndev, (const u32 *)
754 &sk->sk_v6_rcv_saddr, 1);
755 }
756 #endif
757 chtls_disconnect_acceptq(sk);
758 }
759
chtls_pass_open_rpl(struct chtls_dev * cdev,struct sk_buff * skb)760 static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
761 {
762 struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
763 unsigned int stid = GET_TID(rpl);
764 struct listen_ctx *listen_ctx;
765
766 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
767 if (!listen_ctx)
768 return CPL_RET_BUF_DONE;
769
770 if (listen_ctx->state == T4_LISTEN_START_PENDING) {
771 listen_ctx->state = T4_LISTEN_STARTED;
772 return CPL_RET_BUF_DONE;
773 }
774
775 if (rpl->status != CPL_ERR_NONE) {
776 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
777 rpl->status, stid);
778 } else {
779 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
780 sock_put(listen_ctx->lsk);
781 kfree(listen_ctx);
782 module_put(THIS_MODULE);
783 }
784 return CPL_RET_BUF_DONE;
785 }
786
chtls_close_listsrv_rpl(struct chtls_dev * cdev,struct sk_buff * skb)787 static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
788 {
789 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
790 struct listen_ctx *listen_ctx;
791 unsigned int stid;
792 void *data;
793
794 stid = GET_TID(rpl);
795 data = lookup_stid(cdev->tids, stid);
796 listen_ctx = (struct listen_ctx *)data;
797
798 if (rpl->status != CPL_ERR_NONE) {
799 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
800 rpl->status, stid);
801 } else {
802 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
803 sock_put(listen_ctx->lsk);
804 kfree(listen_ctx);
805 module_put(THIS_MODULE);
806 }
807 return CPL_RET_BUF_DONE;
808 }
809
chtls_purge_wr_queue(struct sock * sk)810 static void chtls_purge_wr_queue(struct sock *sk)
811 {
812 struct sk_buff *skb;
813
814 while ((skb = dequeue_wr(sk)) != NULL)
815 kfree_skb(skb);
816 }
817
chtls_release_resources(struct sock * sk)818 static void chtls_release_resources(struct sock *sk)
819 {
820 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
821 struct chtls_dev *cdev = csk->cdev;
822 unsigned int tid = csk->tid;
823 struct tid_info *tids;
824
825 if (!cdev)
826 return;
827
828 tids = cdev->tids;
829 kfree_skb(csk->txdata_skb_cache);
830 csk->txdata_skb_cache = NULL;
831
832 if (csk->wr_credits != csk->wr_max_credits) {
833 chtls_purge_wr_queue(sk);
834 chtls_reset_wr_list(csk);
835 }
836
837 if (csk->l2t_entry) {
838 cxgb4_l2t_release(csk->l2t_entry);
839 csk->l2t_entry = NULL;
840 }
841
842 if (sk->sk_state != TCP_SYN_SENT) {
843 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
844 sock_put(sk);
845 }
846 }
847
chtls_conn_done(struct sock * sk)848 static void chtls_conn_done(struct sock *sk)
849 {
850 if (sock_flag(sk, SOCK_DEAD))
851 chtls_purge_receive_queue(sk);
852 sk_wakeup_sleepers(sk, 0);
853 tcp_done(sk);
854 }
855
do_abort_syn_rcv(struct sock * child,struct sock * parent)856 static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
857 {
858 /*
859 * If the server is still open we clean up the child connection,
860 * otherwise the server already did the clean up as it was purging
861 * its SYN queue and the skb was just sitting in its backlog.
862 */
863 if (likely(parent->sk_state == TCP_LISTEN)) {
864 cleanup_syn_rcv_conn(child, parent);
865 /* Without the below call to sock_orphan,
866 * we leak the socket resource with syn_flood test
867 * as inet_csk_destroy_sock will not be called
868 * in tcp_done since SOCK_DEAD flag is not set.
869 * Kernel handles this differently where new socket is
870 * created only after 3 way handshake is done.
871 */
872 sock_orphan(child);
873 tcp_orphan_count_inc();
874 chtls_release_resources(child);
875 chtls_conn_done(child);
876 } else {
877 if (csk_flag(child, CSK_RST_ABORTED)) {
878 chtls_release_resources(child);
879 chtls_conn_done(child);
880 }
881 }
882 }
883
pass_open_abort(struct sock * child,struct sock * parent,struct sk_buff * skb)884 static void pass_open_abort(struct sock *child, struct sock *parent,
885 struct sk_buff *skb)
886 {
887 do_abort_syn_rcv(child, parent);
888 kfree_skb(skb);
889 }
890
bl_pass_open_abort(struct sock * lsk,struct sk_buff * skb)891 static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
892 {
893 pass_open_abort(skb->sk, lsk, skb);
894 }
895
chtls_pass_open_arp_failure(struct sock * sk,struct sk_buff * skb)896 static void chtls_pass_open_arp_failure(struct sock *sk,
897 struct sk_buff *skb)
898 {
899 const struct request_sock *oreq;
900 struct chtls_sock *csk;
901 struct chtls_dev *cdev;
902 struct sock *parent;
903 void *data;
904
905 csk = rcu_dereference_sk_user_data(sk);
906 cdev = csk->cdev;
907
908 /*
909 * If the connection is being aborted due to the parent listening
910 * socket going away there's nothing to do, the ABORT_REQ will close
911 * the connection.
912 */
913 if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
914 kfree_skb(skb);
915 return;
916 }
917
918 oreq = csk->passive_reap_next;
919 data = lookup_stid(cdev->tids, oreq->ts_recent);
920 parent = ((struct listen_ctx *)data)->lsk;
921
922 bh_lock_sock(parent);
923 if (!sock_owned_by_user(parent)) {
924 pass_open_abort(sk, parent, skb);
925 } else {
926 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
927 __sk_add_backlog(parent, skb);
928 }
929 bh_unlock_sock(parent);
930 }
931
chtls_accept_rpl_arp_failure(void * handle,struct sk_buff * skb)932 static void chtls_accept_rpl_arp_failure(void *handle,
933 struct sk_buff *skb)
934 {
935 struct sock *sk = (struct sock *)handle;
936
937 sock_hold(sk);
938 process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
939 sock_put(sk);
940 }
941
chtls_select_mss(const struct chtls_sock * csk,unsigned int pmtu,struct cpl_pass_accept_req * req)942 static unsigned int chtls_select_mss(const struct chtls_sock *csk,
943 unsigned int pmtu,
944 struct cpl_pass_accept_req *req)
945 {
946 struct chtls_dev *cdev;
947 struct dst_entry *dst;
948 unsigned int tcpoptsz;
949 unsigned int iphdrsz;
950 unsigned int mtu_idx;
951 struct tcp_sock *tp;
952 unsigned int mss;
953 struct sock *sk;
954 u16 user_mss;
955
956 mss = ntohs(req->tcpopt.mss);
957 sk = csk->sk;
958 dst = __sk_dst_get(sk);
959 cdev = csk->cdev;
960 tp = tcp_sk(sk);
961 tcpoptsz = 0;
962
963 #if IS_ENABLED(CONFIG_IPV6)
964 if (sk->sk_family == AF_INET6)
965 iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
966 else
967 #endif
968 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
969 if (req->tcpopt.tstamp)
970 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
971
972 tp->advmss = dst_metric_advmss(dst);
973 user_mss = USER_MSS(tp);
974 if (user_mss && tp->advmss > user_mss)
975 tp->advmss = user_mss;
976 if (tp->advmss > pmtu - iphdrsz)
977 tp->advmss = pmtu - iphdrsz;
978 if (mss && tp->advmss > mss)
979 tp->advmss = mss;
980
981 tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
982 iphdrsz + tcpoptsz,
983 tp->advmss - tcpoptsz,
984 8, &mtu_idx);
985 tp->advmss -= iphdrsz;
986
987 inet_csk(sk)->icsk_pmtu_cookie = pmtu;
988 return mtu_idx;
989 }
990
select_rcv_wscale(int space,int wscale_ok,int win_clamp)991 static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
992 {
993 int wscale = 0;
994
995 if (space > MAX_RCV_WND)
996 space = MAX_RCV_WND;
997 if (win_clamp && win_clamp < space)
998 space = win_clamp;
999
1000 if (wscale_ok) {
1001 while (wscale < 14 && (65535 << wscale) < space)
1002 wscale++;
1003 }
1004 return wscale;
1005 }
1006
chtls_pass_accept_rpl(struct sk_buff * skb,struct cpl_pass_accept_req * req,unsigned int tid)1007 static void chtls_pass_accept_rpl(struct sk_buff *skb,
1008 struct cpl_pass_accept_req *req,
1009 unsigned int tid)
1010
1011 {
1012 struct cpl_t5_pass_accept_rpl *rpl5;
1013 struct cxgb4_lld_info *lldi;
1014 const struct tcphdr *tcph;
1015 const struct tcp_sock *tp;
1016 struct chtls_sock *csk;
1017 unsigned int len;
1018 struct sock *sk;
1019 u32 opt2, hlen;
1020 u64 opt0;
1021
1022 sk = skb->sk;
1023 tp = tcp_sk(sk);
1024 csk = sk->sk_user_data;
1025 csk->tid = tid;
1026 lldi = csk->cdev->lldi;
1027 len = roundup(sizeof(*rpl5), 16);
1028
1029 rpl5 = __skb_put_zero(skb, len);
1030 INIT_TP_WR(rpl5, tid);
1031
1032 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1033 csk->tid));
1034 csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
1035 req);
1036 opt0 = TCAM_BYPASS_F |
1037 WND_SCALE_V(RCV_WSCALE(tp)) |
1038 MSS_IDX_V(csk->mtu_idx) |
1039 L2T_IDX_V(csk->l2t_entry->idx) |
1040 NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
1041 TX_CHAN_V(csk->tx_chan) |
1042 SMAC_SEL_V(csk->smac_idx) |
1043 DSCP_V(csk->tos >> 2) |
1044 ULP_MODE_V(ULP_MODE_TLS) |
1045 RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
1046
1047 opt2 = RX_CHANNEL_V(0) |
1048 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1049
1050 if (!is_t5(lldi->adapter_type))
1051 opt2 |= RX_FC_DISABLE_F;
1052 if (req->tcpopt.tstamp)
1053 opt2 |= TSTAMPS_EN_F;
1054 if (req->tcpopt.sack)
1055 opt2 |= SACK_EN_F;
1056 hlen = ntohl(req->hdr_len);
1057
1058 tcph = (struct tcphdr *)((u8 *)(req + 1) +
1059 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1060 if (tcph->ece && tcph->cwr)
1061 opt2 |= CCTRL_ECN_V(1);
1062 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1063 opt2 |= T5_ISS_F;
1064 opt2 |= T5_OPT_2_VALID_F;
1065 opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
1066 rpl5->opt0 = cpu_to_be64(opt0);
1067 rpl5->opt2 = cpu_to_be32(opt2);
1068 rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);
1069 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
1070 t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
1071 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
1072 }
1073
inet_inherit_port(struct sock * lsk,struct sock * newsk)1074 static void inet_inherit_port(struct sock *lsk, struct sock *newsk)
1075 {
1076 local_bh_disable();
1077 __inet_inherit_port(lsk, newsk);
1078 local_bh_enable();
1079 }
1080
chtls_backlog_rcv(struct sock * sk,struct sk_buff * skb)1081 static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1082 {
1083 if (skb->protocol) {
1084 kfree_skb(skb);
1085 return 0;
1086 }
1087 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
1088 return 0;
1089 }
1090
chtls_set_tcp_window(struct chtls_sock * csk)1091 static void chtls_set_tcp_window(struct chtls_sock *csk)
1092 {
1093 struct net_device *ndev = csk->egress_dev;
1094 struct port_info *pi = netdev_priv(ndev);
1095 unsigned int linkspeed;
1096 u8 scale;
1097
1098 linkspeed = pi->link_cfg.speed;
1099 scale = linkspeed / SPEED_10000;
1100 #define CHTLS_10G_RCVWIN (256 * 1024)
1101 csk->rcv_win = CHTLS_10G_RCVWIN;
1102 if (scale)
1103 csk->rcv_win *= scale;
1104 #define CHTLS_10G_SNDWIN (256 * 1024)
1105 csk->snd_win = CHTLS_10G_SNDWIN;
1106 if (scale)
1107 csk->snd_win *= scale;
1108 }
1109
chtls_recv_sock(struct sock * lsk,struct request_sock * oreq,void * network_hdr,const struct cpl_pass_accept_req * req,struct chtls_dev * cdev)1110 static struct sock *chtls_recv_sock(struct sock *lsk,
1111 struct request_sock *oreq,
1112 void *network_hdr,
1113 const struct cpl_pass_accept_req *req,
1114 struct chtls_dev *cdev)
1115 {
1116 struct adapter *adap = pci_get_drvdata(cdev->pdev);
1117 struct neighbour *n = NULL;
1118 struct inet_sock *newinet;
1119 const struct iphdr *iph;
1120 struct tls_context *ctx;
1121 struct net_device *ndev;
1122 struct chtls_sock *csk;
1123 struct dst_entry *dst;
1124 struct tcp_sock *tp;
1125 struct sock *newsk;
1126 bool found = false;
1127 u16 port_id;
1128 int rxq_idx;
1129 int step, i;
1130
1131 iph = (const struct iphdr *)network_hdr;
1132 newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1133 if (!newsk)
1134 goto free_oreq;
1135
1136 if (lsk->sk_family == AF_INET) {
1137 dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1138 if (!dst)
1139 goto free_sk;
1140
1141 n = dst_neigh_lookup(dst, &iph->saddr);
1142 #if IS_ENABLED(CONFIG_IPV6)
1143 } else {
1144 const struct ipv6hdr *ip6h;
1145 struct flowi6 fl6;
1146
1147 ip6h = (const struct ipv6hdr *)network_hdr;
1148 memset(&fl6, 0, sizeof(fl6));
1149 fl6.flowi6_proto = IPPROTO_TCP;
1150 fl6.saddr = ip6h->daddr;
1151 fl6.daddr = ip6h->saddr;
1152 fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
1153 fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
1154 security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6));
1155 dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
1156 if (IS_ERR(dst))
1157 goto free_sk;
1158 n = dst_neigh_lookup(dst, &ip6h->saddr);
1159 #endif
1160 }
1161 if (!n || !n->dev)
1162 goto free_dst;
1163
1164 ndev = n->dev;
1165 if (is_vlan_dev(ndev))
1166 ndev = vlan_dev_real_dev(ndev);
1167
1168 for_each_port(adap, i)
1169 if (cdev->ports[i] == ndev)
1170 found = true;
1171
1172 if (!found)
1173 goto free_dst;
1174
1175 port_id = cxgb4_port_idx(ndev);
1176
1177 csk = chtls_sock_create(cdev);
1178 if (!csk)
1179 goto free_dst;
1180
1181 csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1182 if (!csk->l2t_entry)
1183 goto free_csk;
1184
1185 newsk->sk_user_data = csk;
1186 newsk->sk_backlog_rcv = chtls_backlog_rcv;
1187
1188 tp = tcp_sk(newsk);
1189 newinet = inet_sk(newsk);
1190
1191 if (iph->version == 0x4) {
1192 newinet->inet_daddr = iph->saddr;
1193 newinet->inet_rcv_saddr = iph->daddr;
1194 newinet->inet_saddr = iph->daddr;
1195 #if IS_ENABLED(CONFIG_IPV6)
1196 } else {
1197 struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
1198 struct inet_request_sock *treq = inet_rsk(oreq);
1199 struct ipv6_pinfo *newnp = inet6_sk(newsk);
1200 struct ipv6_pinfo *np = inet6_sk(lsk);
1201
1202 newinet->pinet6 = &newtcp6sk->inet6;
1203 newinet->ipv6_fl_list = NULL;
1204 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1205 newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
1206 newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
1207 inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
1208 newnp->pktoptions = NULL;
1209 newsk->sk_bound_dev_if = treq->ir_iif;
1210 newinet->inet_opt = NULL;
1211 newinet->inet_daddr = LOOPBACK4_IPV6;
1212 newinet->inet_saddr = LOOPBACK4_IPV6;
1213 #endif
1214 }
1215
1216 oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1217 sk_setup_caps(newsk, dst);
1218 ctx = tls_get_ctx(lsk);
1219 newsk->sk_destruct = ctx->sk_destruct;
1220 newsk->sk_prot_creator = lsk->sk_prot_creator;
1221 csk->sk = newsk;
1222 csk->passive_reap_next = oreq;
1223 csk->tx_chan = cxgb4_port_chan(ndev);
1224 csk->port_id = port_id;
1225 csk->egress_dev = ndev;
1226 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1227 chtls_set_tcp_window(csk);
1228 tp->rcv_wnd = csk->rcv_win;
1229 csk->sndbuf = csk->snd_win;
1230 csk->ulp_mode = ULP_MODE_TLS;
1231 step = cdev->lldi->nrxq / cdev->lldi->nchan;
1232 rxq_idx = port_id * step;
1233 rxq_idx += cdev->round_robin_cnt++ % step;
1234 csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx];
1235 csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1236 port_id * step;
1237 csk->sndbuf = newsk->sk_sndbuf;
1238 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
1239 RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
1240 READ_ONCE(sock_net(newsk)->
1241 ipv4.sysctl_tcp_window_scaling),
1242 tp->window_clamp);
1243 neigh_release(n);
1244 inet_inherit_port(lsk, newsk);
1245 csk_set_flag(csk, CSK_CONN_INLINE);
1246 bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1247
1248 return newsk;
1249 free_csk:
1250 chtls_sock_release(&csk->kref);
1251 free_dst:
1252 if (n)
1253 neigh_release(n);
1254 dst_release(dst);
1255 free_sk:
1256 inet_csk_prepare_forced_close(newsk);
1257 tcp_done(newsk);
1258 free_oreq:
1259 chtls_reqsk_free(oreq);
1260 return NULL;
1261 }
1262
1263 /*
1264 * Populate a TID_RELEASE WR. The skb must be already propely sized.
1265 */
mk_tid_release(struct sk_buff * skb,unsigned int chan,unsigned int tid)1266 static void mk_tid_release(struct sk_buff *skb,
1267 unsigned int chan, unsigned int tid)
1268 {
1269 struct cpl_tid_release *req;
1270 unsigned int len;
1271
1272 len = roundup(sizeof(struct cpl_tid_release), 16);
1273 req = (struct cpl_tid_release *)__skb_put(skb, len);
1274 memset(req, 0, len);
1275 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1276 INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1277 }
1278
chtls_get_module(struct sock * sk)1279 static int chtls_get_module(struct sock *sk)
1280 {
1281 struct inet_connection_sock *icsk = inet_csk(sk);
1282
1283 if (!try_module_get(icsk->icsk_ulp_ops->owner))
1284 return -1;
1285
1286 return 0;
1287 }
1288
chtls_pass_accept_request(struct sock * sk,struct sk_buff * skb)1289 static void chtls_pass_accept_request(struct sock *sk,
1290 struct sk_buff *skb)
1291 {
1292 struct cpl_t5_pass_accept_rpl *rpl;
1293 struct cpl_pass_accept_req *req;
1294 struct listen_ctx *listen_ctx;
1295 struct vlan_ethhdr *vlan_eh;
1296 struct request_sock *oreq;
1297 struct sk_buff *reply_skb;
1298 struct chtls_sock *csk;
1299 struct chtls_dev *cdev;
1300 struct ipv6hdr *ip6h;
1301 struct tcphdr *tcph;
1302 struct sock *newsk;
1303 struct ethhdr *eh;
1304 struct iphdr *iph;
1305 void *network_hdr;
1306 unsigned int stid;
1307 unsigned int len;
1308 unsigned int tid;
1309 bool th_ecn, ect;
1310 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1311 u16 eth_hdr_len;
1312 bool ecn_ok;
1313
1314 req = cplhdr(skb) + RSS_HDR;
1315 tid = GET_TID(req);
1316 cdev = BLOG_SKB_CB(skb)->cdev;
1317 newsk = lookup_tid(cdev->tids, tid);
1318 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1319 if (newsk) {
1320 pr_info("tid (%d) already in use\n", tid);
1321 return;
1322 }
1323
1324 len = roundup(sizeof(*rpl), 16);
1325 reply_skb = alloc_skb(len, GFP_ATOMIC);
1326 if (!reply_skb) {
1327 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1328 kfree_skb(skb);
1329 return;
1330 }
1331
1332 if (sk->sk_state != TCP_LISTEN)
1333 goto reject;
1334
1335 if (inet_csk_reqsk_queue_is_full(sk))
1336 goto reject;
1337
1338 if (sk_acceptq_is_full(sk))
1339 goto reject;
1340
1341
1342 eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1343 if (eth_hdr_len == ETH_HLEN) {
1344 eh = (struct ethhdr *)(req + 1);
1345 iph = (struct iphdr *)(eh + 1);
1346 ip6h = (struct ipv6hdr *)(eh + 1);
1347 network_hdr = (void *)(eh + 1);
1348 } else {
1349 vlan_eh = (struct vlan_ethhdr *)(req + 1);
1350 iph = (struct iphdr *)(vlan_eh + 1);
1351 ip6h = (struct ipv6hdr *)(vlan_eh + 1);
1352 network_hdr = (void *)(vlan_eh + 1);
1353 }
1354
1355 if (iph->version == 0x4) {
1356 tcph = (struct tcphdr *)(iph + 1);
1357 skb_set_network_header(skb, (void *)iph - (void *)req);
1358 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1359 } else {
1360 tcph = (struct tcphdr *)(ip6h + 1);
1361 skb_set_network_header(skb, (void *)ip6h - (void *)req);
1362 oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
1363 }
1364
1365 if (!oreq)
1366 goto reject;
1367
1368 oreq->rsk_rcv_wnd = 0;
1369 oreq->rsk_window_clamp = 0;
1370 oreq->syncookie = 0;
1371 oreq->mss = 0;
1372 oreq->ts_recent = 0;
1373
1374 tcp_rsk(oreq)->tfo_listener = false;
1375 tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1376 chtls_set_req_port(oreq, tcph->source, tcph->dest);
1377 if (iph->version == 0x4) {
1378 chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1379 ip_dsfield = ipv4_get_dsfield(iph);
1380 #if IS_ENABLED(CONFIG_IPV6)
1381 } else {
1382 inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
1383 inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
1384 ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
1385 #endif
1386 }
1387 if (req->tcpopt.wsf <= 14 &&
1388 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {
1389 inet_rsk(oreq)->wscale_ok = 1;
1390 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1391 }
1392 inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
1393 th_ecn = tcph->ece && tcph->cwr;
1394 if (th_ecn) {
1395 ect = !INET_ECN_is_not_ect(ip_dsfield);
1396 ecn_ok = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn);
1397 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1398 inet_rsk(oreq)->ecn_ok = 1;
1399 }
1400
1401 newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1402 if (!newsk)
1403 goto reject;
1404
1405 if (chtls_get_module(newsk))
1406 goto reject;
1407 inet_csk_reqsk_queue_added(sk);
1408 reply_skb->sk = newsk;
1409 chtls_install_cpl_ops(newsk);
1410 cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1411 csk = rcu_dereference_sk_user_data(newsk);
1412 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1413 csk->listen_ctx = listen_ctx;
1414 __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1415 chtls_pass_accept_rpl(reply_skb, req, tid);
1416 kfree_skb(skb);
1417 return;
1418
1419 reject:
1420 mk_tid_release(reply_skb, 0, tid);
1421 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1422 kfree_skb(skb);
1423 }
1424
1425 /*
1426 * Handle a CPL_PASS_ACCEPT_REQ message.
1427 */
chtls_pass_accept_req(struct chtls_dev * cdev,struct sk_buff * skb)1428 static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1429 {
1430 struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1431 struct listen_ctx *ctx;
1432 unsigned int stid;
1433 unsigned int tid;
1434 struct sock *lsk;
1435 void *data;
1436
1437 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1438 tid = GET_TID(req);
1439
1440 data = lookup_stid(cdev->tids, stid);
1441 if (!data)
1442 return 1;
1443
1444 ctx = (struct listen_ctx *)data;
1445 lsk = ctx->lsk;
1446
1447 if (unlikely(tid_out_of_range(cdev->tids, tid))) {
1448 pr_info("passive open TID %u too large\n", tid);
1449 return 1;
1450 }
1451
1452 BLOG_SKB_CB(skb)->cdev = cdev;
1453 process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1454 return 0;
1455 }
1456
1457 /*
1458 * Completes some final bits of initialization for just established connections
1459 * and changes their state to TCP_ESTABLISHED.
1460 *
1461 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1462 */
make_established(struct sock * sk,u32 snd_isn,unsigned int opt)1463 static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1464 {
1465 struct tcp_sock *tp = tcp_sk(sk);
1466
1467 tp->pushed_seq = snd_isn;
1468 tp->write_seq = snd_isn;
1469 tp->snd_nxt = snd_isn;
1470 tp->snd_una = snd_isn;
1471 atomic_set(&inet_sk(sk)->inet_id, get_random_u16());
1472 assign_rxopt(sk, opt);
1473
1474 if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1475 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1476
1477 smp_mb();
1478 tcp_set_state(sk, TCP_ESTABLISHED);
1479 }
1480
chtls_abort_conn(struct sock * sk,struct sk_buff * skb)1481 static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1482 {
1483 struct sk_buff *abort_skb;
1484
1485 abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1486 if (abort_skb)
1487 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1488 }
1489
1490 static struct sock *reap_list;
1491 static DEFINE_SPINLOCK(reap_list_lock);
1492
1493 /*
1494 * Process the reap list.
1495 */
DECLARE_TASK_FUNC(process_reap_list,task_param)1496 DECLARE_TASK_FUNC(process_reap_list, task_param)
1497 {
1498 spin_lock_bh(&reap_list_lock);
1499 while (reap_list) {
1500 struct sock *sk = reap_list;
1501 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1502
1503 reap_list = csk->passive_reap_next;
1504 csk->passive_reap_next = NULL;
1505 spin_unlock(&reap_list_lock);
1506 sock_hold(sk);
1507
1508 bh_lock_sock(sk);
1509 chtls_abort_conn(sk, NULL);
1510 sock_orphan(sk);
1511 if (sk->sk_state == TCP_CLOSE)
1512 inet_csk_destroy_sock(sk);
1513 bh_unlock_sock(sk);
1514 sock_put(sk);
1515 spin_lock(&reap_list_lock);
1516 }
1517 spin_unlock_bh(&reap_list_lock);
1518 }
1519
1520 static DECLARE_WORK(reap_task, process_reap_list);
1521
add_to_reap_list(struct sock * sk)1522 static void add_to_reap_list(struct sock *sk)
1523 {
1524 struct chtls_sock *csk = sk->sk_user_data;
1525
1526 local_bh_disable();
1527 release_tcp_port(sk); /* release the port immediately */
1528
1529 spin_lock(&reap_list_lock);
1530 csk->passive_reap_next = reap_list;
1531 reap_list = sk;
1532 if (!csk->passive_reap_next)
1533 schedule_work(&reap_task);
1534 spin_unlock(&reap_list_lock);
1535 local_bh_enable();
1536 }
1537
add_pass_open_to_parent(struct sock * child,struct sock * lsk,struct chtls_dev * cdev)1538 static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1539 struct chtls_dev *cdev)
1540 {
1541 struct request_sock *oreq;
1542 struct chtls_sock *csk;
1543
1544 if (lsk->sk_state != TCP_LISTEN)
1545 return;
1546
1547 csk = child->sk_user_data;
1548 oreq = csk->passive_reap_next;
1549 csk->passive_reap_next = NULL;
1550
1551 reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1552 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1553
1554 if (sk_acceptq_is_full(lsk)) {
1555 chtls_reqsk_free(oreq);
1556 add_to_reap_list(child);
1557 } else {
1558 refcount_set(&oreq->rsk_refcnt, 1);
1559 inet_csk_reqsk_queue_add(lsk, oreq, child);
1560 lsk->sk_data_ready(lsk);
1561 }
1562 }
1563
bl_add_pass_open_to_parent(struct sock * lsk,struct sk_buff * skb)1564 static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1565 {
1566 struct sock *child = skb->sk;
1567
1568 skb->sk = NULL;
1569 add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1570 kfree_skb(skb);
1571 }
1572
chtls_pass_establish(struct chtls_dev * cdev,struct sk_buff * skb)1573 static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1574 {
1575 struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1576 struct chtls_sock *csk;
1577 struct sock *lsk, *sk;
1578 unsigned int hwtid;
1579
1580 hwtid = GET_TID(req);
1581 sk = lookup_tid(cdev->tids, hwtid);
1582 if (!sk)
1583 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1584
1585 bh_lock_sock(sk);
1586 if (unlikely(sock_owned_by_user(sk))) {
1587 kfree_skb(skb);
1588 } else {
1589 unsigned int stid;
1590 void *data;
1591
1592 csk = sk->sk_user_data;
1593 csk->wr_max_credits = 64;
1594 csk->wr_credits = 64;
1595 csk->wr_unacked = 0;
1596 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1597 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1598 sk->sk_state_change(sk);
1599 if (unlikely(sk->sk_socket))
1600 sk_wake_async(sk, 0, POLL_OUT);
1601
1602 data = lookup_stid(cdev->tids, stid);
1603 if (!data) {
1604 /* listening server close */
1605 kfree_skb(skb);
1606 goto unlock;
1607 }
1608 lsk = ((struct listen_ctx *)data)->lsk;
1609
1610 bh_lock_sock(lsk);
1611 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1612 /* removed from synq */
1613 bh_unlock_sock(lsk);
1614 kfree_skb(skb);
1615 goto unlock;
1616 }
1617
1618 if (likely(!sock_owned_by_user(lsk))) {
1619 kfree_skb(skb);
1620 add_pass_open_to_parent(sk, lsk, cdev);
1621 } else {
1622 skb->sk = sk;
1623 BLOG_SKB_CB(skb)->cdev = cdev;
1624 BLOG_SKB_CB(skb)->backlog_rcv =
1625 bl_add_pass_open_to_parent;
1626 __sk_add_backlog(lsk, skb);
1627 }
1628 bh_unlock_sock(lsk);
1629 }
1630 unlock:
1631 bh_unlock_sock(sk);
1632 return 0;
1633 }
1634
1635 /*
1636 * Handle receipt of an urgent pointer.
1637 */
handle_urg_ptr(struct sock * sk,u32 urg_seq)1638 static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1639 {
1640 struct tcp_sock *tp = tcp_sk(sk);
1641
1642 urg_seq--;
1643 if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1644 return; /* duplicate pointer */
1645
1646 sk_send_sigurg(sk);
1647 if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1648 !sock_flag(sk, SOCK_URGINLINE) &&
1649 tp->copied_seq != tp->rcv_nxt) {
1650 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1651
1652 tp->copied_seq++;
1653 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1654 chtls_free_skb(sk, skb);
1655 }
1656
1657 tp->urg_data = TCP_URG_NOTYET;
1658 tp->urg_seq = urg_seq;
1659 }
1660
check_sk_callbacks(struct chtls_sock * csk)1661 static void check_sk_callbacks(struct chtls_sock *csk)
1662 {
1663 struct sock *sk = csk->sk;
1664
1665 if (unlikely(sk->sk_user_data &&
1666 !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1667 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1668 }
1669
1670 /*
1671 * Handles Rx data that arrives in a state where the socket isn't accepting
1672 * new data.
1673 */
handle_excess_rx(struct sock * sk,struct sk_buff * skb)1674 static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1675 {
1676 if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1677 chtls_abort_conn(sk, skb);
1678
1679 kfree_skb(skb);
1680 }
1681
chtls_recv_data(struct sock * sk,struct sk_buff * skb)1682 static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1683 {
1684 struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1685 struct chtls_sock *csk;
1686 struct tcp_sock *tp;
1687
1688 csk = rcu_dereference_sk_user_data(sk);
1689 tp = tcp_sk(sk);
1690
1691 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1692 handle_excess_rx(sk, skb);
1693 return;
1694 }
1695
1696 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1697 ULP_SKB_CB(skb)->psh = hdr->psh;
1698 skb_ulp_mode(skb) = ULP_MODE_NONE;
1699
1700 skb_reset_transport_header(skb);
1701 __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1702 if (!skb->data_len)
1703 __skb_trim(skb, ntohs(hdr->len));
1704
1705 if (unlikely(hdr->urg))
1706 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1707 if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1708 tp->urg_seq - tp->rcv_nxt < skb->len))
1709 tp->urg_data = TCP_URG_VALID |
1710 skb->data[tp->urg_seq - tp->rcv_nxt];
1711
1712 if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1713 csk->delack_mode = hdr->dack_mode;
1714 csk->delack_seq = tp->rcv_nxt;
1715 }
1716
1717 tcp_hdr(skb)->fin = 0;
1718 tp->rcv_nxt += skb->len;
1719
1720 __skb_queue_tail(&sk->sk_receive_queue, skb);
1721
1722 if (!sock_flag(sk, SOCK_DEAD)) {
1723 check_sk_callbacks(csk);
1724 sk->sk_data_ready(sk);
1725 }
1726 }
1727
chtls_rx_data(struct chtls_dev * cdev,struct sk_buff * skb)1728 static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1729 {
1730 struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1731 unsigned int hwtid = GET_TID(req);
1732 struct sock *sk;
1733
1734 sk = lookup_tid(cdev->tids, hwtid);
1735 if (unlikely(!sk)) {
1736 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1737 return -EINVAL;
1738 }
1739 skb_dstref_steal(skb);
1740 process_cpl_msg(chtls_recv_data, sk, skb);
1741 return 0;
1742 }
1743
chtls_recv_pdu(struct sock * sk,struct sk_buff * skb)1744 static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1745 {
1746 struct cpl_tls_data *hdr = cplhdr(skb);
1747 struct chtls_sock *csk;
1748 struct chtls_hws *tlsk;
1749 struct tcp_sock *tp;
1750
1751 csk = rcu_dereference_sk_user_data(sk);
1752 tlsk = &csk->tlshws;
1753 tp = tcp_sk(sk);
1754
1755 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1756 handle_excess_rx(sk, skb);
1757 return;
1758 }
1759
1760 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1761 ULP_SKB_CB(skb)->flags = 0;
1762 skb_ulp_mode(skb) = ULP_MODE_TLS;
1763
1764 skb_reset_transport_header(skb);
1765 __skb_pull(skb, sizeof(*hdr));
1766 if (!skb->data_len)
1767 __skb_trim(skb,
1768 CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1769
1770 if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1771 tp->rcv_nxt < skb->len))
1772 tp->urg_data = TCP_URG_VALID |
1773 skb->data[tp->urg_seq - tp->rcv_nxt];
1774
1775 tcp_hdr(skb)->fin = 0;
1776 tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1777 __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1778 }
1779
chtls_rx_pdu(struct chtls_dev * cdev,struct sk_buff * skb)1780 static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1781 {
1782 struct cpl_tls_data *req = cplhdr(skb);
1783 unsigned int hwtid = GET_TID(req);
1784 struct sock *sk;
1785
1786 sk = lookup_tid(cdev->tids, hwtid);
1787 if (unlikely(!sk)) {
1788 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1789 return -EINVAL;
1790 }
1791 skb_dstref_steal(skb);
1792 process_cpl_msg(chtls_recv_pdu, sk, skb);
1793 return 0;
1794 }
1795
chtls_set_hdrlen(struct sk_buff * skb,unsigned int nlen)1796 static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1797 {
1798 struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1799
1800 skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1801 tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1802 }
1803
chtls_rx_hdr(struct sock * sk,struct sk_buff * skb)1804 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1805 {
1806 struct tlsrx_cmp_hdr *tls_hdr_pkt;
1807 struct cpl_rx_tls_cmp *cmp_cpl;
1808 struct sk_buff *skb_rec;
1809 struct chtls_sock *csk;
1810 struct chtls_hws *tlsk;
1811 struct tcp_sock *tp;
1812
1813 cmp_cpl = cplhdr(skb);
1814 csk = rcu_dereference_sk_user_data(sk);
1815 tlsk = &csk->tlshws;
1816 tp = tcp_sk(sk);
1817
1818 ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1819 ULP_SKB_CB(skb)->flags = 0;
1820
1821 skb_reset_transport_header(skb);
1822 __skb_pull(skb, sizeof(*cmp_cpl));
1823 tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1824 if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1825 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1826 if (!skb->data_len)
1827 __skb_trim(skb, TLS_HEADER_LENGTH);
1828
1829 tp->rcv_nxt +=
1830 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1831
1832 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
1833 skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1834 if (!skb_rec) {
1835 __skb_queue_tail(&sk->sk_receive_queue, skb);
1836 } else {
1837 chtls_set_hdrlen(skb, tlsk->pldlen);
1838 tlsk->pldlen = 0;
1839 __skb_queue_tail(&sk->sk_receive_queue, skb);
1840 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1841 }
1842
1843 if (!sock_flag(sk, SOCK_DEAD)) {
1844 check_sk_callbacks(csk);
1845 sk->sk_data_ready(sk);
1846 }
1847 }
1848
chtls_rx_cmp(struct chtls_dev * cdev,struct sk_buff * skb)1849 static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1850 {
1851 struct cpl_rx_tls_cmp *req = cplhdr(skb);
1852 unsigned int hwtid = GET_TID(req);
1853 struct sock *sk;
1854
1855 sk = lookup_tid(cdev->tids, hwtid);
1856 if (unlikely(!sk)) {
1857 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1858 return -EINVAL;
1859 }
1860 skb_dstref_steal(skb);
1861 process_cpl_msg(chtls_rx_hdr, sk, skb);
1862
1863 return 0;
1864 }
1865
chtls_timewait(struct sock * sk)1866 static void chtls_timewait(struct sock *sk)
1867 {
1868 struct tcp_sock *tp = tcp_sk(sk);
1869
1870 tp->rcv_nxt++;
1871 tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
1872 tp->srtt_us = 0;
1873 tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1874 }
1875
chtls_peer_close(struct sock * sk,struct sk_buff * skb)1876 static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1877 {
1878 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1879
1880 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1881 goto out;
1882
1883 sk->sk_shutdown |= RCV_SHUTDOWN;
1884 sock_set_flag(sk, SOCK_DONE);
1885
1886 switch (sk->sk_state) {
1887 case TCP_SYN_RECV:
1888 case TCP_ESTABLISHED:
1889 tcp_set_state(sk, TCP_CLOSE_WAIT);
1890 break;
1891 case TCP_FIN_WAIT1:
1892 tcp_set_state(sk, TCP_CLOSING);
1893 break;
1894 case TCP_FIN_WAIT2:
1895 chtls_release_resources(sk);
1896 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1897 chtls_conn_done(sk);
1898 else
1899 chtls_timewait(sk);
1900 break;
1901 default:
1902 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1903 }
1904
1905 if (!sock_flag(sk, SOCK_DEAD)) {
1906 sk->sk_state_change(sk);
1907 /* Do not send POLL_HUP for half duplex close. */
1908
1909 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1910 sk->sk_state == TCP_CLOSE)
1911 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1912 else
1913 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1914 }
1915 out:
1916 kfree_skb(skb);
1917 }
1918
chtls_close_con_rpl(struct sock * sk,struct sk_buff * skb)1919 static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1920 {
1921 struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1922 struct chtls_sock *csk;
1923 struct tcp_sock *tp;
1924
1925 csk = rcu_dereference_sk_user_data(sk);
1926
1927 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1928 goto out;
1929
1930 tp = tcp_sk(sk);
1931
1932 tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
1933
1934 switch (sk->sk_state) {
1935 case TCP_CLOSING:
1936 chtls_release_resources(sk);
1937 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1938 chtls_conn_done(sk);
1939 else
1940 chtls_timewait(sk);
1941 break;
1942 case TCP_LAST_ACK:
1943 chtls_release_resources(sk);
1944 chtls_conn_done(sk);
1945 break;
1946 case TCP_FIN_WAIT1:
1947 tcp_set_state(sk, TCP_FIN_WAIT2);
1948 sk->sk_shutdown |= SEND_SHUTDOWN;
1949
1950 if (!sock_flag(sk, SOCK_DEAD))
1951 sk->sk_state_change(sk);
1952 else if (tcp_sk(sk)->linger2 < 0 &&
1953 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1954 chtls_abort_conn(sk, skb);
1955 else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT))
1956 chtls_set_quiesce_ctrl(sk, 0);
1957 break;
1958 default:
1959 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1960 }
1961 out:
1962 kfree_skb(skb);
1963 }
1964
get_cpl_skb(struct sk_buff * skb,size_t len,gfp_t gfp)1965 static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1966 size_t len, gfp_t gfp)
1967 {
1968 if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1969 WARN_ONCE(skb->len < len, "skb alloc error");
1970 __skb_trim(skb, len);
1971 skb_get(skb);
1972 } else {
1973 skb = alloc_skb(len, gfp);
1974 if (skb)
1975 __skb_put(skb, len);
1976 }
1977 return skb;
1978 }
1979
set_abort_rpl_wr(struct sk_buff * skb,unsigned int tid,int cmd)1980 static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1981 int cmd)
1982 {
1983 struct cpl_abort_rpl *rpl = cplhdr(skb);
1984
1985 INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1986 rpl->cmd = cmd;
1987 }
1988
send_defer_abort_rpl(struct chtls_dev * cdev,struct sk_buff * skb)1989 static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1990 {
1991 struct cpl_abort_req_rss *req = cplhdr(skb);
1992 struct sk_buff *reply_skb;
1993
1994 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1995 GFP_KERNEL | __GFP_NOFAIL);
1996 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1997 set_abort_rpl_wr(reply_skb, GET_TID(req),
1998 (req->status & CPL_ABORT_NO_RST));
1999 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
2000 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2001 kfree_skb(skb);
2002 }
2003
2004 /*
2005 * Add an skb to the deferred skb queue for processing from process context.
2006 */
t4_defer_reply(struct sk_buff * skb,struct chtls_dev * cdev,defer_handler_t handler)2007 static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
2008 defer_handler_t handler)
2009 {
2010 DEFERRED_SKB_CB(skb)->handler = handler;
2011 spin_lock_bh(&cdev->deferq.lock);
2012 __skb_queue_tail(&cdev->deferq, skb);
2013 if (skb_queue_len(&cdev->deferq) == 1)
2014 schedule_work(&cdev->deferq_task);
2015 spin_unlock_bh(&cdev->deferq.lock);
2016 }
2017
chtls_send_abort_rpl(struct sock * sk,struct sk_buff * skb,struct chtls_dev * cdev,int status,int queue)2018 static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
2019 struct chtls_dev *cdev,
2020 int status, int queue)
2021 {
2022 struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2023 struct sk_buff *reply_skb;
2024 struct chtls_sock *csk;
2025 unsigned int tid;
2026
2027 csk = rcu_dereference_sk_user_data(sk);
2028 tid = GET_TID(req);
2029
2030 reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
2031 if (!reply_skb) {
2032 req->status = (queue << 1) | status;
2033 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
2034 return;
2035 }
2036
2037 set_abort_rpl_wr(reply_skb, tid, status);
2038 kfree_skb(skb);
2039 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
2040 if (csk_conn_inline(csk)) {
2041 struct l2t_entry *e = csk->l2t_entry;
2042
2043 if (e && sk->sk_state != TCP_SYN_RECV) {
2044 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2045 return;
2046 }
2047 }
2048 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2049 }
2050
2051 /*
2052 * This is run from a listener's backlog to abort a child connection in
2053 * SYN_RCV state (i.e., one on the listener's SYN queue).
2054 */
bl_abort_syn_rcv(struct sock * lsk,struct sk_buff * skb)2055 static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
2056 {
2057 struct chtls_sock *csk;
2058 struct sock *child;
2059 int queue;
2060
2061 child = skb->sk;
2062 csk = rcu_dereference_sk_user_data(child);
2063 queue = csk->txq_idx;
2064
2065 skb->sk = NULL;
2066 chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
2067 CPL_ABORT_NO_RST, queue);
2068 do_abort_syn_rcv(child, lsk);
2069 }
2070
abort_syn_rcv(struct sock * sk,struct sk_buff * skb)2071 static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
2072 {
2073 const struct request_sock *oreq;
2074 struct listen_ctx *listen_ctx;
2075 struct chtls_sock *csk;
2076 struct chtls_dev *cdev;
2077 struct sock *psk;
2078 void *ctx;
2079
2080 csk = sk->sk_user_data;
2081 oreq = csk->passive_reap_next;
2082 cdev = csk->cdev;
2083
2084 if (!oreq)
2085 return -1;
2086
2087 ctx = lookup_stid(cdev->tids, oreq->ts_recent);
2088 if (!ctx)
2089 return -1;
2090
2091 listen_ctx = (struct listen_ctx *)ctx;
2092 psk = listen_ctx->lsk;
2093
2094 bh_lock_sock(psk);
2095 if (!sock_owned_by_user(psk)) {
2096 int queue = csk->txq_idx;
2097
2098 chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
2099 do_abort_syn_rcv(sk, psk);
2100 } else {
2101 skb->sk = sk;
2102 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
2103 __sk_add_backlog(psk, skb);
2104 }
2105 bh_unlock_sock(psk);
2106 return 0;
2107 }
2108
chtls_abort_req_rss(struct sock * sk,struct sk_buff * skb)2109 static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
2110 {
2111 const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2112 struct chtls_sock *csk = sk->sk_user_data;
2113 int rst_status = CPL_ABORT_NO_RST;
2114 int queue = csk->txq_idx;
2115
2116 if (is_neg_adv(req->status)) {
2117 kfree_skb(skb);
2118 return;
2119 }
2120
2121 csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
2122
2123 if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
2124 !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
2125 struct tcp_sock *tp = tcp_sk(sk);
2126
2127 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
2128 WARN_ONCE(1, "send_tx_flowc error");
2129 csk_set_flag(csk, CSK_TX_DATA_SENT);
2130 }
2131
2132 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
2133
2134 if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2135 sk->sk_err = ETIMEDOUT;
2136
2137 if (!sock_flag(sk, SOCK_DEAD))
2138 sk_error_report(sk);
2139
2140 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
2141 return;
2142
2143 }
2144
2145 chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
2146 rst_status, queue);
2147 chtls_release_resources(sk);
2148 chtls_conn_done(sk);
2149 }
2150
chtls_abort_rpl_rss(struct sock * sk,struct sk_buff * skb)2151 static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2152 {
2153 struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2154 struct chtls_sock *csk;
2155 struct chtls_dev *cdev;
2156
2157 csk = rcu_dereference_sk_user_data(sk);
2158 cdev = csk->cdev;
2159
2160 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2161 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2162 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2163 if (sk->sk_state == TCP_SYN_SENT) {
2164 cxgb4_remove_tid(cdev->tids,
2165 csk->port_id,
2166 GET_TID(rpl),
2167 sk->sk_family);
2168 sock_put(sk);
2169 }
2170 chtls_release_resources(sk);
2171 chtls_conn_done(sk);
2172 }
2173 }
2174 kfree_skb(skb);
2175 }
2176
chtls_conn_cpl(struct chtls_dev * cdev,struct sk_buff * skb)2177 static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2178 {
2179 struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2180 void (*fn)(struct sock *sk, struct sk_buff *skb);
2181 unsigned int hwtid = GET_TID(req);
2182 struct chtls_sock *csk;
2183 struct sock *sk;
2184 u8 opcode;
2185
2186 opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2187
2188 sk = lookup_tid(cdev->tids, hwtid);
2189 if (!sk)
2190 goto rel_skb;
2191
2192 csk = sk->sk_user_data;
2193
2194 switch (opcode) {
2195 case CPL_PEER_CLOSE:
2196 fn = chtls_peer_close;
2197 break;
2198 case CPL_CLOSE_CON_RPL:
2199 fn = chtls_close_con_rpl;
2200 break;
2201 case CPL_ABORT_REQ_RSS:
2202 /*
2203 * Save the offload device in the skb, we may process this
2204 * message after the socket has closed.
2205 */
2206 BLOG_SKB_CB(skb)->cdev = csk->cdev;
2207 fn = chtls_abort_req_rss;
2208 break;
2209 case CPL_ABORT_RPL_RSS:
2210 fn = chtls_abort_rpl_rss;
2211 break;
2212 default:
2213 goto rel_skb;
2214 }
2215
2216 process_cpl_msg(fn, sk, skb);
2217 return 0;
2218
2219 rel_skb:
2220 kfree_skb(skb);
2221 return 0;
2222 }
2223
chtls_rx_ack(struct sock * sk,struct sk_buff * skb)2224 static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2225 {
2226 struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2227 struct chtls_sock *csk = sk->sk_user_data;
2228 struct tcp_sock *tp = tcp_sk(sk);
2229 u32 credits = hdr->credits;
2230 u32 snd_una;
2231
2232 snd_una = ntohl(hdr->snd_una);
2233 csk->wr_credits += credits;
2234
2235 if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2236 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2237
2238 while (credits) {
2239 struct sk_buff *pskb = csk->wr_skb_head;
2240 u32 csum;
2241
2242 if (unlikely(!pskb)) {
2243 if (csk->wr_nondata)
2244 csk->wr_nondata -= credits;
2245 break;
2246 }
2247 csum = (__force u32)pskb->csum;
2248 if (unlikely(credits < csum)) {
2249 pskb->csum = (__force __wsum)(csum - credits);
2250 break;
2251 }
2252 dequeue_wr(sk);
2253 credits -= csum;
2254 kfree_skb(pskb);
2255 }
2256 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2257 if (unlikely(before(snd_una, tp->snd_una))) {
2258 kfree_skb(skb);
2259 return;
2260 }
2261
2262 if (tp->snd_una != snd_una) {
2263 tp->snd_una = snd_una;
2264 tp->rcv_tstamp = tcp_jiffies32;
2265 if (tp->snd_una == tp->snd_nxt &&
2266 !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2267 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2268 }
2269 }
2270
2271 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2272 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2273
2274 csk->wr_credits -= fclen16;
2275 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2276 csk_reset_flag(csk, CSK_TX_FAILOVER);
2277 }
2278 if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2279 sk->sk_write_space(sk);
2280
2281 kfree_skb(skb);
2282 }
2283
chtls_wr_ack(struct chtls_dev * cdev,struct sk_buff * skb)2284 static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2285 {
2286 struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2287 unsigned int hwtid = GET_TID(rpl);
2288 struct sock *sk;
2289
2290 sk = lookup_tid(cdev->tids, hwtid);
2291 if (unlikely(!sk)) {
2292 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2293 return -EINVAL;
2294 }
2295 process_cpl_msg(chtls_rx_ack, sk, skb);
2296
2297 return 0;
2298 }
2299
chtls_set_tcb_rpl(struct chtls_dev * cdev,struct sk_buff * skb)2300 static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
2301 {
2302 struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR;
2303 unsigned int hwtid = GET_TID(rpl);
2304 struct sock *sk;
2305
2306 sk = lookup_tid(cdev->tids, hwtid);
2307
2308 /* return EINVAL if socket doesn't exist */
2309 if (!sk)
2310 return -EINVAL;
2311
2312 /* Reusing the skb as size of cpl_set_tcb_field structure
2313 * is greater than cpl_abort_req
2314 */
2315 if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG)
2316 chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL);
2317
2318 kfree_skb(skb);
2319 return 0;
2320 }
2321
2322 chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2323 [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl,
2324 [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2325 [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req,
2326 [CPL_PASS_ESTABLISH] = chtls_pass_establish,
2327 [CPL_RX_DATA] = chtls_rx_data,
2328 [CPL_TLS_DATA] = chtls_rx_pdu,
2329 [CPL_RX_TLS_CMP] = chtls_rx_cmp,
2330 [CPL_PEER_CLOSE] = chtls_conn_cpl,
2331 [CPL_CLOSE_CON_RPL] = chtls_conn_cpl,
2332 [CPL_ABORT_REQ_RSS] = chtls_conn_cpl,
2333 [CPL_ABORT_RPL_RSS] = chtls_conn_cpl,
2334 [CPL_FW4_ACK] = chtls_wr_ack,
2335 [CPL_SET_TCB_RPL] = chtls_set_tcb_rpl,
2336 };
2337