1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/skbuff.h>
7 #include <linux/module.h>
8 #include <linux/highmem.h>
9 #include <linux/ip.h>
10 #include <net/ipv6.h>
11 #include <linux/netdevice.h>
12 #include <crypto/aes.h>
13 #include <linux/skbuff_ref.h>
14 #include "chcr_ktls.h"
15
16 static LIST_HEAD(uld_ctx_list);
17 static DEFINE_MUTEX(dev_mutex);
18
19 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
20 * @skb: skb
21 * @start: start offset.
22 * @len: how much data to send after @start
23 */
chcr_get_nfrags_to_send(struct sk_buff * skb,u32 start,u32 len)24 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
25 {
26 struct skb_shared_info *si = skb_shinfo(skb);
27 u32 frag_size, skb_linear_data_len = skb_headlen(skb);
28 u8 nfrags = 0, frag_idx = 0;
29 skb_frag_t *frag;
30
31 /* if its a linear skb then return 1 */
32 if (!skb_is_nonlinear(skb))
33 return 1;
34
35 if (unlikely(start < skb_linear_data_len)) {
36 frag_size = min(len, skb_linear_data_len - start);
37 } else {
38 start -= skb_linear_data_len;
39
40 frag = &si->frags[frag_idx];
41 frag_size = skb_frag_size(frag);
42 while (start >= frag_size) {
43 start -= frag_size;
44 frag_idx++;
45 frag = &si->frags[frag_idx];
46 frag_size = skb_frag_size(frag);
47 }
48 frag_size = min(len, skb_frag_size(frag) - start);
49 }
50 len -= frag_size;
51 nfrags++;
52
53 while (len) {
54 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
55 len -= frag_size;
56 nfrags++;
57 frag_idx++;
58 }
59 return nfrags;
60 }
61
62 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
63 static void clear_conn_resources(struct chcr_ktls_info *tx_info);
64 /*
65 * chcr_ktls_save_keys: calculate and save crypto keys.
66 * @tx_info - driver specific tls info.
67 * @crypto_info - tls crypto information.
68 * @direction - TX/RX direction.
69 * return - SUCCESS/FAILURE.
70 */
chcr_ktls_save_keys(struct chcr_ktls_info * tx_info,struct tls_crypto_info * crypto_info,enum tls_offload_ctx_dir direction)71 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
72 struct tls_crypto_info *crypto_info,
73 enum tls_offload_ctx_dir direction)
74 {
75 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
76 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
77 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
78 struct ktls_key_ctx *kctx = &tx_info->key_ctx;
79 struct crypto_aes_ctx aes_ctx;
80 unsigned char *key, *salt;
81
82 switch (crypto_info->cipher_type) {
83 case TLS_CIPHER_AES_GCM_128:
84 info_128_gcm =
85 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
86 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
87 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
88 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
89 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
90 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
91 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
92
93 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
94 key = info_128_gcm->key;
95 salt = info_128_gcm->salt;
96 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
97
98 /* The SCMD fields used when encrypting a full TLS
99 * record. Its a one time calculation till the
100 * connection exists.
101 */
102 tx_info->scmd0_seqno_numivs =
103 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
104 SCMD_CIPH_AUTH_SEQ_CTRL_F |
105 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
106 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
107 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
108 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
109 SCMD_NUM_IVS_V(1);
110
111 /* keys will be sent inline. */
112 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
113
114 /* The SCMD fields used when encrypting a partial TLS
115 * record (no trailer and possibly a truncated payload).
116 */
117 tx_info->scmd0_short_seqno_numivs =
118 SCMD_CIPH_AUTH_SEQ_CTRL_F |
119 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
120 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
121 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
122
123 tx_info->scmd0_short_ivgen_hdrlen =
124 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
125
126 break;
127
128 default:
129 pr_err("GCM: cipher type 0x%x not supported\n",
130 crypto_info->cipher_type);
131 ret = -EINVAL;
132 goto out;
133 }
134
135 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
136 roundup(keylen, 16) + ghash_size;
137 /* Calculate the H = CIPH(K, 0 repeated 16 times).
138 * It will go in key context
139 */
140
141 ret = aes_expandkey(&aes_ctx, key, keylen);
142 if (ret)
143 goto out;
144
145 memset(ghash_h, 0, ghash_size);
146 aes_encrypt(&aes_ctx, ghash_h, ghash_h);
147 memzero_explicit(&aes_ctx, sizeof(aes_ctx));
148
149 /* fill the Key context */
150 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
151 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
152 mac_key_size,
153 key_ctx_size >> 4);
154 } else {
155 ret = -EINVAL;
156 goto out;
157 }
158
159 memcpy(kctx->salt, salt, tx_info->salt_size);
160 memcpy(kctx->key, key, keylen);
161 memcpy(kctx->key + keylen, ghash_h, ghash_size);
162 tx_info->key_ctx_len = key_ctx_size;
163
164 out:
165 return ret;
166 }
167
168 /*
169 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
170 * @sk - tcp socket.
171 * @tx_info - driver specific tls info.
172 * @atid - connection active tid.
173 * return - send success/failure.
174 */
chcr_ktls_act_open_req(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)175 static int chcr_ktls_act_open_req(struct sock *sk,
176 struct chcr_ktls_info *tx_info,
177 int atid)
178 {
179 struct inet_sock *inet = inet_sk(sk);
180 struct cpl_t6_act_open_req *cpl6;
181 struct cpl_act_open_req *cpl;
182 struct sk_buff *skb;
183 unsigned int len;
184 int qid_atid;
185 u64 options;
186
187 len = sizeof(*cpl6);
188 skb = alloc_skb(len, GFP_KERNEL);
189 if (unlikely(!skb))
190 return -ENOMEM;
191 /* mark it a control pkt */
192 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
193
194 cpl6 = __skb_put_zero(skb, len);
195 cpl = (struct cpl_act_open_req *)cpl6;
196 INIT_TP_WR(cpl6, 0);
197 qid_atid = TID_QID_V(tx_info->rx_qid) |
198 TID_TID_V(atid);
199 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
200 cpl->local_port = inet->inet_sport;
201 cpl->peer_port = inet->inet_dport;
202 cpl->local_ip = inet->inet_rcv_saddr;
203 cpl->peer_ip = inet->inet_daddr;
204
205 /* fill first 64 bit option field. */
206 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
207 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
208 cpl->opt0 = cpu_to_be64(options);
209
210 /* next 64 bit option field. */
211 options =
212 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
213 cpl->opt2 = htonl(options);
214
215 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
216 }
217
218 #if IS_ENABLED(CONFIG_IPV6)
219 /*
220 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
221 * @sk - tcp socket.
222 * @tx_info - driver specific tls info.
223 * @atid - connection active tid.
224 * return - send success/failure.
225 */
chcr_ktls_act_open_req6(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)226 static int chcr_ktls_act_open_req6(struct sock *sk,
227 struct chcr_ktls_info *tx_info,
228 int atid)
229 {
230 struct inet_sock *inet = inet_sk(sk);
231 struct cpl_t6_act_open_req6 *cpl6;
232 struct cpl_act_open_req6 *cpl;
233 struct sk_buff *skb;
234 unsigned int len;
235 int qid_atid;
236 u64 options;
237
238 len = sizeof(*cpl6);
239 skb = alloc_skb(len, GFP_KERNEL);
240 if (unlikely(!skb))
241 return -ENOMEM;
242 /* mark it a control pkt */
243 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
244
245 cpl6 = __skb_put_zero(skb, len);
246 cpl = (struct cpl_act_open_req6 *)cpl6;
247 INIT_TP_WR(cpl6, 0);
248 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
249 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
250 cpl->local_port = inet->inet_sport;
251 cpl->peer_port = inet->inet_dport;
252 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
253 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
254 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
255 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
256
257 /* first 64 bit option field. */
258 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
259 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
260 cpl->opt0 = cpu_to_be64(options);
261 /* next 64 bit option field. */
262 options =
263 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
264 cpl->opt2 = htonl(options);
265
266 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
267 }
268 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
269
270 /*
271 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
272 * @sk - tcp socket.
273 * @tx_info - driver specific tls info.
274 * return: NET_TX_OK/NET_XMIT_DROP
275 */
chcr_setup_connection(struct sock * sk,struct chcr_ktls_info * tx_info)276 static int chcr_setup_connection(struct sock *sk,
277 struct chcr_ktls_info *tx_info)
278 {
279 struct tid_info *t = &tx_info->adap->tids;
280 int atid, ret = 0;
281
282 atid = cxgb4_alloc_atid(t, tx_info);
283 if (atid == -1)
284 return -EINVAL;
285
286 tx_info->atid = atid;
287
288 if (tx_info->ip_family == AF_INET) {
289 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
290 #if IS_ENABLED(CONFIG_IPV6)
291 } else {
292 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
293 &sk->sk_v6_rcv_saddr,
294 1);
295 if (ret)
296 return ret;
297 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
298 #endif
299 }
300
301 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
302 * success, if any other return type clear atid and return that failure.
303 */
304 if (ret) {
305 if (ret == NET_XMIT_CN) {
306 ret = 0;
307 } else {
308 #if IS_ENABLED(CONFIG_IPV6)
309 /* clear clip entry */
310 if (tx_info->ip_family == AF_INET6)
311 cxgb4_clip_release(tx_info->netdev,
312 (const u32 *)
313 &sk->sk_v6_rcv_saddr,
314 1);
315 #endif
316 cxgb4_free_atid(t, atid);
317 }
318 }
319
320 return ret;
321 }
322
323 /*
324 * chcr_set_tcb_field: update tcb fields.
325 * @tx_info - driver specific tls info.
326 * @word - TCB word.
327 * @mask - TCB word related mask.
328 * @val - TCB word related value.
329 * @no_reply - set 1 if not looking for TP response.
330 */
chcr_set_tcb_field(struct chcr_ktls_info * tx_info,u16 word,u64 mask,u64 val,int no_reply)331 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
332 u64 mask, u64 val, int no_reply)
333 {
334 struct cpl_set_tcb_field *req;
335 struct sk_buff *skb;
336
337 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
338 if (!skb)
339 return -ENOMEM;
340
341 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
342 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
343 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
344 NO_REPLY_V(no_reply));
345 req->word_cookie = htons(TCB_WORD_V(word));
346 req->mask = cpu_to_be64(mask);
347 req->val = cpu_to_be64(val);
348
349 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
350 return cxgb4_ofld_send(tx_info->netdev, skb);
351 }
352
353 /*
354 * chcr_ktls_dev_del: call back for tls_dev_del.
355 * Remove the tid and l2t entry and close the connection.
356 * it per connection basis.
357 * @netdev - net device.
358 * @tls_cts - tls context.
359 * @direction - TX/RX crypto direction
360 */
chcr_ktls_dev_del(struct net_device * netdev,struct tls_context * tls_ctx,enum tls_offload_ctx_dir direction)361 static void chcr_ktls_dev_del(struct net_device *netdev,
362 struct tls_context *tls_ctx,
363 enum tls_offload_ctx_dir direction)
364 {
365 struct chcr_ktls_info *tx_info = chcr_get_ktls_tx_info(tls_ctx);
366 struct ch_ktls_port_stats_debug *port_stats;
367 struct chcr_ktls_uld_ctx *u_ctx;
368
369 if (!tx_info)
370 return;
371
372 u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle;
373 if (u_ctx && u_ctx->detach)
374 return;
375 /* clear l2t entry */
376 if (tx_info->l2te)
377 cxgb4_l2t_release(tx_info->l2te);
378
379 #if IS_ENABLED(CONFIG_IPV6)
380 /* clear clip entry */
381 if (tx_info->ip_family == AF_INET6)
382 cxgb4_clip_release(netdev, (const u32 *)
383 &tx_info->sk->sk_v6_rcv_saddr,
384 1);
385 #endif
386
387 /* clear tid */
388 if (tx_info->tid != -1) {
389 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
390 tx_info->tid, tx_info->ip_family);
391
392 xa_erase(&u_ctx->tid_list, tx_info->tid);
393 }
394
395 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
396 atomic64_inc(&port_stats->ktls_tx_connection_close);
397 kvfree(tx_info);
398 chcr_set_ktls_tx_info(tls_ctx, NULL);
399 /* release module refcount */
400 module_put(THIS_MODULE);
401 }
402
403 /*
404 * chcr_ktls_dev_add: call back for tls_dev_add.
405 * Create a tcb entry for TP. Also add l2t entry for the connection. And
406 * generate keys & save those keys locally.
407 * @netdev - net device.
408 * @tls_cts - tls context.
409 * @direction - TX/RX crypto direction
410 * return: SUCCESS/FAILURE.
411 */
chcr_ktls_dev_add(struct net_device * netdev,struct sock * sk,enum tls_offload_ctx_dir direction,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)412 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
413 enum tls_offload_ctx_dir direction,
414 struct tls_crypto_info *crypto_info,
415 u32 start_offload_tcp_sn)
416 {
417 struct tls_context *tls_ctx = tls_get_ctx(sk);
418 struct ch_ktls_port_stats_debug *port_stats;
419 struct chcr_ktls_uld_ctx *u_ctx;
420 struct chcr_ktls_info *tx_info;
421 struct dst_entry *dst;
422 struct adapter *adap;
423 struct port_info *pi;
424 struct neighbour *n;
425 u8 daaddr[16];
426 int ret = -1;
427
428 pi = netdev_priv(netdev);
429 adap = pi->adapter;
430 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
431 atomic64_inc(&port_stats->ktls_tx_connection_open);
432 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
433
434 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
435 pr_err("not expecting for RX direction\n");
436 goto out;
437 }
438
439 if (chcr_get_ktls_tx_info(tls_ctx))
440 goto out;
441
442 if (u_ctx && u_ctx->detach)
443 goto out;
444
445 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
446 if (!tx_info)
447 goto out;
448
449 tx_info->sk = sk;
450 spin_lock_init(&tx_info->lock);
451 /* initialize tid and atid to -1, 0 is a also a valid id. */
452 tx_info->tid = -1;
453 tx_info->atid = -1;
454
455 tx_info->adap = adap;
456 tx_info->netdev = netdev;
457 tx_info->first_qset = pi->first_qset;
458 tx_info->tx_chan = pi->tx_chan;
459 tx_info->smt_idx = pi->smt_idx;
460 tx_info->port_id = pi->port_id;
461 tx_info->prev_ack = 0;
462 tx_info->prev_win = 0;
463
464 tx_info->rx_qid = chcr_get_first_rx_qid(adap);
465 if (unlikely(tx_info->rx_qid < 0))
466 goto free_tx_info;
467
468 tx_info->prev_seq = start_offload_tcp_sn;
469 tx_info->tcp_start_seq_number = start_offload_tcp_sn;
470
471 /* save crypto keys */
472 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
473 if (ret < 0)
474 goto free_tx_info;
475
476 /* get peer ip */
477 if (sk->sk_family == AF_INET) {
478 memcpy(daaddr, &sk->sk_daddr, 4);
479 tx_info->ip_family = AF_INET;
480 #if IS_ENABLED(CONFIG_IPV6)
481 } else {
482 if (!ipv6_only_sock(sk) &&
483 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
484 memcpy(daaddr, &sk->sk_daddr, 4);
485 tx_info->ip_family = AF_INET;
486 } else {
487 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
488 tx_info->ip_family = AF_INET6;
489 }
490 #endif
491 }
492
493 /* get the l2t index */
494 dst = sk_dst_get(sk);
495 if (!dst) {
496 pr_err("DST entry not found\n");
497 goto free_tx_info;
498 }
499 n = dst_neigh_lookup(dst, daaddr);
500 if (!n || !n->dev) {
501 pr_err("neighbour not found\n");
502 dst_release(dst);
503 goto free_tx_info;
504 }
505 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
506
507 neigh_release(n);
508 dst_release(dst);
509
510 if (!tx_info->l2te) {
511 pr_err("l2t entry not found\n");
512 goto free_tx_info;
513 }
514
515 /* Driver shouldn't be removed until any single connection exists */
516 if (!try_module_get(THIS_MODULE))
517 goto free_l2t;
518
519 init_completion(&tx_info->completion);
520 /* create a filter and call cxgb4_l2t_send to send the packet out, which
521 * will take care of updating l2t entry in hw if not already done.
522 */
523 tx_info->open_state = CH_KTLS_OPEN_PENDING;
524
525 if (chcr_setup_connection(sk, tx_info))
526 goto put_module;
527
528 /* Wait for reply */
529 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
530 spin_lock_bh(&tx_info->lock);
531 if (tx_info->open_state) {
532 /* need to wait for hw response, can't free tx_info yet. */
533 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
534 tx_info->pending_close = true;
535 else
536 spin_unlock_bh(&tx_info->lock);
537 /* if in pending close, free the lock after the cleanup */
538 goto put_module;
539 }
540 spin_unlock_bh(&tx_info->lock);
541
542 /* initialize tcb */
543 reinit_completion(&tx_info->completion);
544 /* mark it pending for hw response */
545 tx_info->open_state = CH_KTLS_OPEN_PENDING;
546
547 if (chcr_init_tcb_fields(tx_info))
548 goto free_tid;
549
550 /* Wait for reply */
551 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
552 spin_lock_bh(&tx_info->lock);
553 if (tx_info->open_state) {
554 /* need to wait for hw response, can't free tx_info yet. */
555 tx_info->pending_close = true;
556 /* free the lock after cleanup */
557 goto free_tid;
558 }
559 spin_unlock_bh(&tx_info->lock);
560
561 if (!cxgb4_check_l2t_valid(tx_info->l2te))
562 goto free_tid;
563
564 atomic64_inc(&port_stats->ktls_tx_ctx);
565 chcr_set_ktls_tx_info(tls_ctx, tx_info);
566
567 return 0;
568
569 free_tid:
570 #if IS_ENABLED(CONFIG_IPV6)
571 /* clear clip entry */
572 if (tx_info->ip_family == AF_INET6)
573 cxgb4_clip_release(netdev, (const u32 *)
574 &sk->sk_v6_rcv_saddr,
575 1);
576 #endif
577 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
578 tx_info->tid, tx_info->ip_family);
579
580 xa_erase(&u_ctx->tid_list, tx_info->tid);
581
582 put_module:
583 /* release module refcount */
584 module_put(THIS_MODULE);
585 free_l2t:
586 cxgb4_l2t_release(tx_info->l2te);
587 free_tx_info:
588 if (tx_info->pending_close)
589 spin_unlock_bh(&tx_info->lock);
590 else
591 kvfree(tx_info);
592 out:
593 atomic64_inc(&port_stats->ktls_tx_connection_fail);
594 return -1;
595 }
596
597 /*
598 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
599 * handling.
600 * @tx_info - driver specific tls info.
601 * return: NET_TX_OK/NET_XMIT_DROP
602 */
chcr_init_tcb_fields(struct chcr_ktls_info * tx_info)603 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
604 {
605 int ret = 0;
606
607 /* set tcb in offload and bypass */
608 ret =
609 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
610 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
611 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
612 if (ret)
613 return ret;
614 /* reset snd_una and snd_next fields in tcb */
615 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
616 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
617 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
618 0, 1);
619 if (ret)
620 return ret;
621
622 /* reset send max */
623 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
624 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
625 0, 1);
626 if (ret)
627 return ret;
628
629 /* update l2t index and request for tp reply to confirm tcb is
630 * initialised to handle tx traffic.
631 */
632 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
633 TCB_L2T_IX_V(TCB_L2T_IX_M),
634 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
635 return ret;
636 }
637
638 /*
639 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
640 */
chcr_ktls_cpl_act_open_rpl(struct adapter * adap,unsigned char * input)641 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
642 unsigned char *input)
643 {
644 const struct cpl_act_open_rpl *p = (void *)input;
645 struct chcr_ktls_info *tx_info = NULL;
646 struct tls_offload_context_tx *tx_ctx;
647 struct chcr_ktls_uld_ctx *u_ctx;
648 unsigned int atid, tid, status;
649 struct tls_context *tls_ctx;
650 struct tid_info *t;
651 int ret = 0;
652
653 tid = GET_TID(p);
654 status = AOPEN_STATUS_G(ntohl(p->atid_status));
655 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
656
657 t = &adap->tids;
658 tx_info = lookup_atid(t, atid);
659
660 if (!tx_info || tx_info->atid != atid) {
661 pr_err("%s: incorrect tx_info or atid\n", __func__);
662 return -1;
663 }
664
665 cxgb4_free_atid(t, atid);
666 tx_info->atid = -1;
667
668 spin_lock(&tx_info->lock);
669 /* HW response is very close, finish pending cleanup */
670 if (tx_info->pending_close) {
671 spin_unlock(&tx_info->lock);
672 if (!status) {
673 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
674 tid, tx_info->ip_family);
675 }
676 kvfree(tx_info);
677 return 0;
678 }
679
680 if (!status) {
681 tx_info->tid = tid;
682 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
683 /* Adding tid */
684 tls_ctx = tls_get_ctx(tx_info->sk);
685 tx_ctx = tls_offload_ctx_tx(tls_ctx);
686 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;
687 if (u_ctx) {
688 ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx,
689 GFP_NOWAIT);
690 if (ret < 0) {
691 pr_err("%s: Failed to allocate tid XA entry = %d\n",
692 __func__, tx_info->tid);
693 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
694 goto out;
695 }
696 }
697 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
698 } else {
699 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
700 }
701 out:
702 spin_unlock(&tx_info->lock);
703
704 complete(&tx_info->completion);
705 return ret;
706 }
707
708 /*
709 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
710 */
chcr_ktls_cpl_set_tcb_rpl(struct adapter * adap,unsigned char * input)711 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
712 {
713 const struct cpl_set_tcb_rpl *p = (void *)input;
714 struct chcr_ktls_info *tx_info = NULL;
715 struct tid_info *t;
716 u32 tid;
717
718 tid = GET_TID(p);
719
720 t = &adap->tids;
721 tx_info = lookup_tid(t, tid);
722
723 if (!tx_info || tx_info->tid != tid) {
724 pr_err("%s: incorrect tx_info or tid\n", __func__);
725 return -1;
726 }
727
728 spin_lock(&tx_info->lock);
729 if (tx_info->pending_close) {
730 spin_unlock(&tx_info->lock);
731 kvfree(tx_info);
732 return 0;
733 }
734 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
735 spin_unlock(&tx_info->lock);
736
737 complete(&tx_info->completion);
738 return 0;
739 }
740
__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,u32 tid,void * pos,u16 word,struct sge_eth_txq * q,u64 mask,u64 val,u32 reply)741 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
742 u32 tid, void *pos, u16 word,
743 struct sge_eth_txq *q, u64 mask,
744 u64 val, u32 reply)
745 {
746 struct cpl_set_tcb_field_core *cpl;
747 struct ulptx_idata *idata;
748 struct ulp_txpkt *txpkt;
749
750 /* ULP_TXPKT */
751 txpkt = pos;
752 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
753 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
754 ULP_TXPKT_FID_V(q->q.cntxt_id) |
755 ULP_TXPKT_RO_F);
756 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
757
758 /* ULPTX_IDATA sub-command */
759 idata = (struct ulptx_idata *)(txpkt + 1);
760 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
761 idata->len = htonl(sizeof(*cpl));
762 pos = idata + 1;
763
764 cpl = pos;
765 /* CPL_SET_TCB_FIELD */
766 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
767 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
768 NO_REPLY_V(!reply));
769 cpl->word_cookie = htons(TCB_WORD_V(word));
770 cpl->mask = cpu_to_be64(mask);
771 cpl->val = cpu_to_be64(val);
772
773 /* ULPTX_NOOP */
774 idata = (struct ulptx_idata *)(cpl + 1);
775 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
776 idata->len = htonl(0);
777 pos = idata + 1;
778
779 return pos;
780 }
781
782
783 /*
784 * chcr_write_cpl_set_tcb_ulp: update tcb values.
785 * TCB is responsible to create tcp headers, so all the related values
786 * should be correctly updated.
787 * @tx_info - driver specific tls info.
788 * @q - tx queue on which packet is going out.
789 * @tid - TCB identifier.
790 * @pos - current index where should we start writing.
791 * @word - TCB word.
792 * @mask - TCB word related mask.
793 * @val - TCB word related value.
794 * @reply - set 1 if looking for TP response.
795 * return - next position to write.
796 */
chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tid,void * pos,u16 word,u64 mask,u64 val,u32 reply)797 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
798 struct sge_eth_txq *q, u32 tid,
799 void *pos, u16 word, u64 mask,
800 u64 val, u32 reply)
801 {
802 int left = (void *)q->q.stat - pos;
803
804 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
805 if (!left) {
806 pos = q->q.desc;
807 } else {
808 u8 buf[48] = {0};
809
810 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
811 mask, val, reply);
812
813 return chcr_copy_to_txd(buf, &q->q, pos,
814 CHCR_SET_TCB_FIELD_LEN);
815 }
816 }
817
818 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
819 mask, val, reply);
820
821 /* check again if we are at the end of the queue */
822 if (left == CHCR_SET_TCB_FIELD_LEN)
823 pos = q->q.desc;
824
825 return pos;
826 }
827
828 /*
829 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
830 * with updated values like tcp seq, ack, window etc.
831 * @tx_info - driver specific tls info.
832 * @q - TX queue.
833 * @tcp_seq
834 * @tcp_ack
835 * @tcp_win
836 * return: NETDEV_TX_BUSY/NET_TX_OK.
837 */
chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u64 tcp_seq,u64 tcp_ack,u64 tcp_win,bool offset)838 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
839 struct sge_eth_txq *q, u64 tcp_seq,
840 u64 tcp_ack, u64 tcp_win, bool offset)
841 {
842 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
843 struct ch_ktls_port_stats_debug *port_stats;
844 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
845 struct fw_ulptx_wr *wr;
846 int credits;
847 void *pos;
848
849 wr_len = sizeof(*wr);
850 /* there can be max 4 cpls, check if we have enough credits */
851 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
852 ndesc = DIV_ROUND_UP(len, 64);
853
854 credits = chcr_txq_avail(&q->q) - ndesc;
855 if (unlikely(credits < 0)) {
856 chcr_eth_txq_stop(q);
857 return NETDEV_TX_BUSY;
858 }
859
860 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
861 chcr_eth_txq_stop(q);
862 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
863 }
864
865 pos = &q->q.desc[q->q.pidx];
866 /* make space for WR, we'll fill it later when we know all the cpls
867 * being sent out and have complete length.
868 */
869 wr = pos;
870 pos += wr_len;
871 /* update tx_max if its a re-transmit or the first wr */
872 if (first_wr || tcp_seq != tx_info->prev_seq) {
873 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
874 TCB_TX_MAX_W,
875 TCB_TX_MAX_V(TCB_TX_MAX_M),
876 TCB_TX_MAX_V(tcp_seq), 0);
877 cpl++;
878 }
879 /* reset snd una if it's a re-transmit pkt */
880 if (tcp_seq != tx_info->prev_seq || offset) {
881 /* reset snd_una */
882 port_stats =
883 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
884 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
885 TCB_SND_UNA_RAW_W,
886 TCB_SND_UNA_RAW_V
887 (TCB_SND_UNA_RAW_M),
888 TCB_SND_UNA_RAW_V(0), 0);
889 if (tcp_seq != tx_info->prev_seq)
890 atomic64_inc(&port_stats->ktls_tx_ooo);
891 cpl++;
892 }
893 /* update ack */
894 if (first_wr || tx_info->prev_ack != tcp_ack) {
895 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
896 TCB_RCV_NXT_W,
897 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
898 TCB_RCV_NXT_V(tcp_ack), 0);
899 tx_info->prev_ack = tcp_ack;
900 cpl++;
901 }
902 /* update receive window */
903 if (first_wr || tx_info->prev_win != tcp_win) {
904 chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
905 TCB_RCV_WND_W,
906 TCB_RCV_WND_V(TCB_RCV_WND_M),
907 TCB_RCV_WND_V(tcp_win), 0);
908 tx_info->prev_win = tcp_win;
909 cpl++;
910 }
911
912 if (cpl) {
913 /* get the actual length */
914 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
915 /* ULPTX wr */
916 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
917 wr->cookie = 0;
918 /* fill len in wr field */
919 wr->flowid_len16 = htonl(wr_mid |
920 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
921
922 ndesc = DIV_ROUND_UP(len, 64);
923 chcr_txq_advance(&q->q, ndesc);
924 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
925 }
926 return 0;
927 }
928
929 /*
930 * chcr_ktls_get_tx_flits
931 * returns number of flits to be sent out, it includes key context length, WR
932 * size and skb fragments.
933 */
934 static unsigned int
chcr_ktls_get_tx_flits(u32 nr_frags,unsigned int key_ctx_len)935 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
936 {
937 return chcr_sgl_len(nr_frags) +
938 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
939 }
940
941 /*
942 * chcr_ktls_check_tcp_options: To check if there is any TCP option available
943 * other than timestamp.
944 * @skb - skb contains partial record..
945 * return: 1 / 0
946 */
947 static int
chcr_ktls_check_tcp_options(struct tcphdr * tcp)948 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
949 {
950 int cnt, opt, optlen;
951 u_char *cp;
952
953 cp = (u_char *)(tcp + 1);
954 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
955 for (; cnt > 0; cnt -= optlen, cp += optlen) {
956 opt = cp[0];
957 if (opt == TCPOPT_EOL)
958 break;
959 if (opt == TCPOPT_NOP) {
960 optlen = 1;
961 } else {
962 if (cnt < 2)
963 break;
964 optlen = cp[1];
965 if (optlen < 2 || optlen > cnt)
966 break;
967 }
968 switch (opt) {
969 case TCPOPT_NOP:
970 break;
971 default:
972 return 1;
973 }
974 }
975 return 0;
976 }
977
978 /*
979 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
980 * send out separately.
981 * @tx_info - driver specific tls info.
982 * @skb - skb contains partial record..
983 * @q - TX queue.
984 * @tx_chan - channel number.
985 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
986 */
987 static int
chcr_ktls_write_tcp_options(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q,uint32_t tx_chan)988 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
989 struct sge_eth_txq *q, uint32_t tx_chan)
990 {
991 struct fw_eth_tx_pkt_wr *wr;
992 struct cpl_tx_pkt_core *cpl;
993 u32 ctrl, iplen, maclen;
994 struct ipv6hdr *ip6;
995 unsigned int ndesc;
996 struct tcphdr *tcp;
997 int len16, pktlen;
998 struct iphdr *ip;
999 u32 wr_mid = 0;
1000 int credits;
1001 u8 buf[150];
1002 u64 cntrl1;
1003 void *pos;
1004
1005 iplen = skb_network_header_len(skb);
1006 maclen = skb_mac_header_len(skb);
1007
1008 /* packet length = eth hdr len + ip hdr len + tcp hdr len
1009 * (including options).
1010 */
1011 pktlen = skb_tcp_all_headers(skb);
1012
1013 ctrl = sizeof(*cpl) + pktlen;
1014 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1015 /* check how many descriptors needed */
1016 ndesc = DIV_ROUND_UP(len16, 4);
1017
1018 credits = chcr_txq_avail(&q->q) - ndesc;
1019 if (unlikely(credits < 0)) {
1020 chcr_eth_txq_stop(q);
1021 return NETDEV_TX_BUSY;
1022 }
1023
1024 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1025 chcr_eth_txq_stop(q);
1026 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1027 }
1028
1029 pos = &q->q.desc[q->q.pidx];
1030 wr = pos;
1031
1032 /* Firmware work request header */
1033 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1034 FW_WR_IMMDLEN_V(ctrl));
1035
1036 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1037 wr->r3 = 0;
1038
1039 cpl = (void *)(wr + 1);
1040
1041 /* CPL header */
1042 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1043 TXPKT_PF_V(tx_info->adap->pf));
1044 cpl->pack = 0;
1045 cpl->len = htons(pktlen);
1046
1047 memcpy(buf, skb->data, pktlen);
1048 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1049 /* we need to correct ip header len */
1050 ip = (struct iphdr *)(buf + maclen);
1051 ip->tot_len = htons(pktlen - maclen);
1052 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1053 } else {
1054 ip6 = (struct ipv6hdr *)(buf + maclen);
1055 ip6->payload_len = htons(pktlen - maclen - iplen);
1056 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1057 }
1058
1059 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1060 TXPKT_IPHDR_LEN_V(iplen);
1061 /* checksum offload */
1062 cpl->ctrl1 = cpu_to_be64(cntrl1);
1063
1064 pos = cpl + 1;
1065
1066 /* now take care of the tcp header, if fin is not set then clear push
1067 * bit as well, and if fin is set, it will be sent at the last so we
1068 * need to update the tcp sequence number as per the last packet.
1069 */
1070 tcp = (struct tcphdr *)(buf + maclen + iplen);
1071
1072 if (!tcp->fin)
1073 tcp->psh = 0;
1074 else
1075 tcp->seq = htonl(tx_info->prev_seq);
1076
1077 chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1078
1079 chcr_txq_advance(&q->q, ndesc);
1080 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1081 return 0;
1082 }
1083
1084 /*
1085 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1086 * received has partial end part of the record, send out the complete record, so
1087 * that crypto block will be able to generate TAG/HASH.
1088 * @skb - segment which has complete or partial end part.
1089 * @tx_info - driver specific tls info.
1090 * @q - TX queue.
1091 * @tcp_seq
1092 * @tcp_push - tcp push bit.
1093 * @mss - segment size.
1094 * return: NETDEV_TX_BUSY/NET_TX_OK.
1095 */
chcr_ktls_xmit_wr_complete(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool is_last_wr,u32 data_len,u32 skb_offset,u32 nfrags,bool tcp_push,u32 mss)1096 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1097 struct chcr_ktls_info *tx_info,
1098 struct sge_eth_txq *q, u32 tcp_seq,
1099 bool is_last_wr, u32 data_len,
1100 u32 skb_offset, u32 nfrags,
1101 bool tcp_push, u32 mss)
1102 {
1103 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1104 struct adapter *adap = tx_info->adap;
1105 int credits, left, last_desc;
1106 struct tx_sw_desc *sgl_sdesc;
1107 struct cpl_tx_data *tx_data;
1108 struct cpl_tx_sec_pdu *cpl;
1109 struct ulptx_idata *idata;
1110 struct ulp_txpkt *ulptx;
1111 struct fw_ulptx_wr *wr;
1112 void *pos;
1113 u64 *end;
1114
1115 /* get the number of flits required */
1116 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1117 /* number of descriptors */
1118 ndesc = chcr_flits_to_desc(flits);
1119 /* check if enough credits available */
1120 credits = chcr_txq_avail(&q->q) - ndesc;
1121 if (unlikely(credits < 0)) {
1122 chcr_eth_txq_stop(q);
1123 return NETDEV_TX_BUSY;
1124 }
1125
1126 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1127 /* Credits are below the threshold values, stop the queue after
1128 * injecting the Work Request for this packet.
1129 */
1130 chcr_eth_txq_stop(q);
1131 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1132 }
1133
1134 last_desc = q->q.pidx + ndesc - 1;
1135 if (last_desc >= q->q.size)
1136 last_desc -= q->q.size;
1137 sgl_sdesc = &q->q.sdesc[last_desc];
1138
1139 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1140 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1141 q->mapping_err++;
1142 return NETDEV_TX_BUSY;
1143 }
1144
1145 if (!is_last_wr)
1146 skb_get(skb);
1147
1148 pos = &q->q.desc[q->q.pidx];
1149 end = (u64 *)pos + flits;
1150 /* FW_ULPTX_WR */
1151 wr = pos;
1152 /* WR will need len16 */
1153 len16 = DIV_ROUND_UP(flits, 2);
1154 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1155 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1156 wr->cookie = 0;
1157 pos += sizeof(*wr);
1158 /* ULP_TXPKT */
1159 ulptx = pos;
1160 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1161 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1162 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1163 ULP_TXPKT_RO_F);
1164 ulptx->len = htonl(len16 - 1);
1165 /* ULPTX_IDATA sub-command */
1166 idata = (struct ulptx_idata *)(ulptx + 1);
1167 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1168 /* idata length will include cpl_tx_sec_pdu + key context size +
1169 * cpl_tx_data header.
1170 */
1171 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1172 sizeof(*tx_data));
1173 /* SEC CPL */
1174 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1175 cpl->op_ivinsrtofst =
1176 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1177 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1178 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1179 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1180 cpl->pldlen = htonl(data_len);
1181
1182 /* encryption should start after tls header size + iv size */
1183 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1184
1185 cpl->aadstart_cipherstop_hi =
1186 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1187 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1188 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1189
1190 /* authentication will also start after tls header + iv size */
1191 cpl->cipherstop_lo_authinsert =
1192 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1193 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1194 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1195
1196 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1197 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1198 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1199 cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1200
1201 pos = cpl + 1;
1202 /* check if space left to fill the keys */
1203 left = (void *)q->q.stat - pos;
1204 if (!left) {
1205 left = (void *)end - (void *)q->q.stat;
1206 pos = q->q.desc;
1207 end = pos + left;
1208 }
1209
1210 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1211 tx_info->key_ctx_len);
1212 left = (void *)q->q.stat - pos;
1213
1214 if (!left) {
1215 left = (void *)end - (void *)q->q.stat;
1216 pos = q->q.desc;
1217 end = pos + left;
1218 }
1219 /* CPL_TX_DATA */
1220 tx_data = (void *)pos;
1221 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1222 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1223
1224 tx_data->rsvd = htonl(tcp_seq);
1225
1226 tx_data->flags = htonl(TX_BYPASS_F);
1227 if (tcp_push)
1228 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1229
1230 /* check left again, it might go beyond queue limit */
1231 pos = tx_data + 1;
1232 left = (void *)q->q.stat - pos;
1233
1234 /* check the position again */
1235 if (!left) {
1236 left = (void *)end - (void *)q->q.stat;
1237 pos = q->q.desc;
1238 end = pos + left;
1239 }
1240
1241 /* send the complete packet except the header */
1242 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1243 skb_offset, data_len);
1244 sgl_sdesc->skb = skb;
1245
1246 chcr_txq_advance(&q->q, ndesc);
1247 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1248 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1249
1250 return 0;
1251 }
1252
1253 /*
1254 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1255 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1256 * and then only send it out.
1257 *
1258 * @skb - skb contains partial record..
1259 * @tx_info - driver specific tls info.
1260 * @q - TX queue.
1261 * @tcp_seq
1262 * @tcp_push - tcp push bit.
1263 * @mss - segment size.
1264 * @tls_rec_offset - offset from start of the tls record.
1265 * @perior_data - data before the current segment, required to make this record
1266 * 16 byte aligned.
1267 * @prior_data_len - prior_data length (less than 16)
1268 * return: NETDEV_TX_BUSY/NET_TX_OK.
1269 */
chcr_ktls_xmit_wr_short(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool tcp_push,u32 mss,u32 tls_rec_offset,u8 * prior_data,u32 prior_data_len,u32 data_len,u32 skb_offset)1270 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1271 struct chcr_ktls_info *tx_info,
1272 struct sge_eth_txq *q,
1273 u32 tcp_seq, bool tcp_push, u32 mss,
1274 u32 tls_rec_offset, u8 *prior_data,
1275 u32 prior_data_len, u32 data_len,
1276 u32 skb_offset)
1277 {
1278 u32 len16, wr_mid = 0, cipher_start, nfrags;
1279 struct adapter *adap = tx_info->adap;
1280 unsigned int flits = 0, ndesc;
1281 int credits, left, last_desc;
1282 struct tx_sw_desc *sgl_sdesc;
1283 struct cpl_tx_data *tx_data;
1284 struct cpl_tx_sec_pdu *cpl;
1285 struct ulptx_idata *idata;
1286 struct ulp_txpkt *ulptx;
1287 struct fw_ulptx_wr *wr;
1288 __be64 iv_record;
1289 void *pos;
1290 u64 *end;
1291
1292 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1293 /* get the number of flits required, it's a partial record so 2 flits
1294 * (AES_BLOCK_SIZE) will be added.
1295 */
1296 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1297 /* get the correct 8 byte IV of this record */
1298 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1299 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1300 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1301 * data will be added.
1302 */
1303 if (prior_data_len)
1304 flits += 2;
1305 /* number of descriptors */
1306 ndesc = chcr_flits_to_desc(flits);
1307 /* check if enough credits available */
1308 credits = chcr_txq_avail(&q->q) - ndesc;
1309 if (unlikely(credits < 0)) {
1310 chcr_eth_txq_stop(q);
1311 return NETDEV_TX_BUSY;
1312 }
1313
1314 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1315 chcr_eth_txq_stop(q);
1316 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1317 }
1318
1319 last_desc = q->q.pidx + ndesc - 1;
1320 if (last_desc >= q->q.size)
1321 last_desc -= q->q.size;
1322 sgl_sdesc = &q->q.sdesc[last_desc];
1323
1324 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1325 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1326 q->mapping_err++;
1327 return NETDEV_TX_BUSY;
1328 }
1329
1330 pos = &q->q.desc[q->q.pidx];
1331 end = (u64 *)pos + flits;
1332 /* FW_ULPTX_WR */
1333 wr = pos;
1334 /* WR will need len16 */
1335 len16 = DIV_ROUND_UP(flits, 2);
1336 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1337 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1338 wr->cookie = 0;
1339 pos += sizeof(*wr);
1340 /* ULP_TXPKT */
1341 ulptx = pos;
1342 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1343 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1344 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1345 ULP_TXPKT_RO_F);
1346 ulptx->len = htonl(len16 - 1);
1347 /* ULPTX_IDATA sub-command */
1348 idata = (struct ulptx_idata *)(ulptx + 1);
1349 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1350 /* idata length will include cpl_tx_sec_pdu + key context size +
1351 * cpl_tx_data header.
1352 */
1353 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1354 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1355 /* SEC CPL */
1356 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1357 /* cipher start will have tls header + iv size extra if its a header
1358 * part of tls record. else only 16 byte IV will be added.
1359 */
1360 cipher_start =
1361 AES_BLOCK_LEN + 1 +
1362 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1363
1364 cpl->op_ivinsrtofst =
1365 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1366 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1367 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1368 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1369 cpl->aadstart_cipherstop_hi =
1370 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1371 cpl->cipherstop_lo_authinsert = 0;
1372 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1373 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1374 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1375 cpl->scmd1 = 0;
1376
1377 pos = cpl + 1;
1378 /* check if space left to fill the keys */
1379 left = (void *)q->q.stat - pos;
1380 if (!left) {
1381 left = (void *)end - (void *)q->q.stat;
1382 pos = q->q.desc;
1383 end = pos + left;
1384 }
1385
1386 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1387 tx_info->key_ctx_len);
1388 left = (void *)q->q.stat - pos;
1389
1390 if (!left) {
1391 left = (void *)end - (void *)q->q.stat;
1392 pos = q->q.desc;
1393 end = pos + left;
1394 }
1395 /* CPL_TX_DATA */
1396 tx_data = (void *)pos;
1397 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1398 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1399 TX_LENGTH_V(data_len + prior_data_len));
1400 tx_data->rsvd = htonl(tcp_seq);
1401 tx_data->flags = htonl(TX_BYPASS_F);
1402 if (tcp_push)
1403 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1404
1405 /* check left again, it might go beyond queue limit */
1406 pos = tx_data + 1;
1407 left = (void *)q->q.stat - pos;
1408
1409 /* check the position again */
1410 if (!left) {
1411 left = (void *)end - (void *)q->q.stat;
1412 pos = q->q.desc;
1413 end = pos + left;
1414 }
1415 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1416 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1417 */
1418 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1419 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1420 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1421 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1422 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1423
1424 pos += 16;
1425 /* Prior_data_len will always be less than 16 bytes, fill the
1426 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1427 * to 0.
1428 */
1429 if (prior_data_len)
1430 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1431 /* send the complete packet except the header */
1432 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1433 skb_offset, data_len);
1434 sgl_sdesc->skb = skb;
1435
1436 chcr_txq_advance(&q->q, ndesc);
1437 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1438
1439 return 0;
1440 }
1441
1442 /*
1443 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1444 * only plain text (only tls header and iv)
1445 * @tx_info - driver specific tls info.
1446 * @skb - skb contains partial record..
1447 * @tcp_seq
1448 * @mss - segment size.
1449 * @tcp_push - tcp push bit.
1450 * @q - TX queue.
1451 * @port_id : port number
1452 * @perior_data - data before the current segment, required to make this record
1453 * 16 byte aligned.
1454 * @prior_data_len - prior_data length (less than 16)
1455 * return: NETDEV_TX_BUSY/NET_TX_OK.
1456 */
chcr_ktls_tx_plaintxt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,u32 tcp_seq,u32 mss,bool tcp_push,struct sge_eth_txq * q,u32 port_id,u8 * prior_data,u32 data_len,u32 skb_offset,u32 prior_data_len)1457 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1458 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1459 bool tcp_push, struct sge_eth_txq *q,
1460 u32 port_id, u8 *prior_data,
1461 u32 data_len, u32 skb_offset,
1462 u32 prior_data_len)
1463 {
1464 int credits, left, len16, last_desc;
1465 unsigned int flits = 0, ndesc;
1466 struct tx_sw_desc *sgl_sdesc;
1467 struct cpl_tx_data *tx_data;
1468 struct ulptx_idata *idata;
1469 struct ulp_txpkt *ulptx;
1470 struct fw_ulptx_wr *wr;
1471 u32 wr_mid = 0, nfrags;
1472 void *pos;
1473 u64 *end;
1474
1475 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1476 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1477 flits += chcr_sgl_len(nfrags);
1478 if (prior_data_len)
1479 flits += 2;
1480
1481 /* WR will need len16 */
1482 len16 = DIV_ROUND_UP(flits, 2);
1483 /* check how many descriptors needed */
1484 ndesc = DIV_ROUND_UP(flits, 8);
1485
1486 credits = chcr_txq_avail(&q->q) - ndesc;
1487 if (unlikely(credits < 0)) {
1488 chcr_eth_txq_stop(q);
1489 return NETDEV_TX_BUSY;
1490 }
1491
1492 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1493 chcr_eth_txq_stop(q);
1494 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1495 }
1496
1497 last_desc = q->q.pidx + ndesc - 1;
1498 if (last_desc >= q->q.size)
1499 last_desc -= q->q.size;
1500 sgl_sdesc = &q->q.sdesc[last_desc];
1501
1502 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1503 sgl_sdesc->addr) < 0)) {
1504 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1505 q->mapping_err++;
1506 return NETDEV_TX_BUSY;
1507 }
1508
1509 pos = &q->q.desc[q->q.pidx];
1510 end = (u64 *)pos + flits;
1511 /* FW_ULPTX_WR */
1512 wr = pos;
1513 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1514 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1515 wr->cookie = 0;
1516 /* ULP_TXPKT */
1517 ulptx = (struct ulp_txpkt *)(wr + 1);
1518 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1519 ULP_TXPKT_DATAMODIFY_V(0) |
1520 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1521 ULP_TXPKT_DEST_V(0) |
1522 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1523 ulptx->len = htonl(len16 - 1);
1524 /* ULPTX_IDATA sub-command */
1525 idata = (struct ulptx_idata *)(ulptx + 1);
1526 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1527 idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1528 /* CPL_TX_DATA */
1529 tx_data = (struct cpl_tx_data *)(idata + 1);
1530 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1531 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1532 TX_LENGTH_V(data_len + prior_data_len));
1533 /* set tcp seq number */
1534 tx_data->rsvd = htonl(tcp_seq);
1535 tx_data->flags = htonl(TX_BYPASS_F);
1536 if (tcp_push)
1537 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1538
1539 pos = tx_data + 1;
1540 /* apart from prior_data_len, we should set remaining part of 16 bytes
1541 * to be zero.
1542 */
1543 if (prior_data_len)
1544 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1545
1546 /* check left again, it might go beyond queue limit */
1547 left = (void *)q->q.stat - pos;
1548
1549 /* check the position again */
1550 if (!left) {
1551 left = (void *)end - (void *)q->q.stat;
1552 pos = q->q.desc;
1553 end = pos + left;
1554 }
1555 /* send the complete packet including the header */
1556 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1557 skb_offset, data_len);
1558 sgl_sdesc->skb = skb;
1559
1560 chcr_txq_advance(&q->q, ndesc);
1561 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1562 return 0;
1563 }
1564
chcr_ktls_tunnel_pkt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q)1565 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1566 struct sk_buff *skb,
1567 struct sge_eth_txq *q)
1568 {
1569 u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1570 struct tx_sw_desc *sgl_sdesc;
1571 struct fw_eth_tx_pkt_wr *wr;
1572 struct cpl_tx_pkt_core *cpl;
1573 unsigned int flits, ndesc;
1574 int credits, last_desc;
1575 u64 cntrl1, *end;
1576 void *pos;
1577
1578 ctrl = sizeof(*cpl);
1579 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1580
1581 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1582 len16 = DIV_ROUND_UP(flits, 2);
1583 /* check how many descriptors needed */
1584 ndesc = DIV_ROUND_UP(flits, 8);
1585
1586 credits = chcr_txq_avail(&q->q) - ndesc;
1587 if (unlikely(credits < 0)) {
1588 chcr_eth_txq_stop(q);
1589 return -ENOMEM;
1590 }
1591
1592 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1593 chcr_eth_txq_stop(q);
1594 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1595 }
1596
1597 last_desc = q->q.pidx + ndesc - 1;
1598 if (last_desc >= q->q.size)
1599 last_desc -= q->q.size;
1600 sgl_sdesc = &q->q.sdesc[last_desc];
1601
1602 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1603 sgl_sdesc->addr) < 0)) {
1604 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1605 q->mapping_err++;
1606 return -ENOMEM;
1607 }
1608
1609 iplen = skb_network_header_len(skb);
1610 maclen = skb_mac_header_len(skb);
1611
1612 pos = &q->q.desc[q->q.pidx];
1613 end = (u64 *)pos + flits;
1614 wr = pos;
1615
1616 /* Firmware work request header */
1617 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1618 FW_WR_IMMDLEN_V(ctrl));
1619
1620 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1621 wr->r3 = 0;
1622
1623 cpl = (void *)(wr + 1);
1624
1625 /* CPL header */
1626 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1627 TXPKT_INTF_V(tx_info->tx_chan) |
1628 TXPKT_PF_V(tx_info->adap->pf));
1629 cpl->pack = 0;
1630 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1631 TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1632 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1633 TXPKT_IPHDR_LEN_V(iplen);
1634 /* checksum offload */
1635 cpl->ctrl1 = cpu_to_be64(cntrl1);
1636 cpl->len = htons(skb->len);
1637
1638 pos = cpl + 1;
1639
1640 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1641 sgl_sdesc->skb = skb;
1642 chcr_txq_advance(&q->q, ndesc);
1643 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1644 return 0;
1645 }
1646
1647 /*
1648 * chcr_ktls_copy_record_in_skb
1649 * @nskb - new skb where the frags to be added.
1650 * @skb - old skb, to copy socket and destructor details.
1651 * @record - specific record which has complete 16k record in frags.
1652 */
chcr_ktls_copy_record_in_skb(struct sk_buff * nskb,struct sk_buff * skb,struct tls_record_info * record)1653 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1654 struct sk_buff *skb,
1655 struct tls_record_info *record)
1656 {
1657 int i = 0;
1658
1659 for (i = 0; i < record->num_frags; i++) {
1660 skb_shinfo(nskb)->frags[i] = record->frags[i];
1661 /* increase the frag ref count */
1662 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1663 }
1664
1665 skb_shinfo(nskb)->nr_frags = record->num_frags;
1666 nskb->data_len = record->len;
1667 nskb->len += record->len;
1668 nskb->truesize += record->len;
1669 nskb->sk = skb->sk;
1670 nskb->destructor = skb->destructor;
1671 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1672 }
1673
1674 /*
1675 * chcr_end_part_handler: This handler will handle the record which
1676 * is complete or if record's end part is received. T6 adapter has a issue that
1677 * it can't send out TAG with partial record so if its an end part then we have
1678 * to send TAG as well and for which we need to fetch the complete record and
1679 * send it to crypto module.
1680 * @tx_info - driver specific tls info.
1681 * @skb - skb contains partial record.
1682 * @record - complete record of 16K size.
1683 * @tcp_seq
1684 * @mss - segment size in which TP needs to chop a packet.
1685 * @tcp_push_no_fin - tcp push if fin is not set.
1686 * @q - TX queue.
1687 * @tls_end_offset - offset from end of the record.
1688 * @last wr : check if this is the last part of the skb going out.
1689 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1690 */
chcr_end_part_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,struct sge_eth_txq * q,u32 skb_offset,u32 tls_end_offset,bool last_wr)1691 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1692 struct sk_buff *skb,
1693 struct tls_record_info *record,
1694 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1695 struct sge_eth_txq *q, u32 skb_offset,
1696 u32 tls_end_offset, bool last_wr)
1697 {
1698 bool free_skb_if_tx_fails = false;
1699 struct sk_buff *nskb = NULL;
1700
1701 /* check if it is a complete record */
1702 if (tls_end_offset == record->len) {
1703 nskb = skb;
1704 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1705 } else {
1706 nskb = alloc_skb(0, GFP_ATOMIC);
1707 if (!nskb) {
1708 dev_kfree_skb_any(skb);
1709 return NETDEV_TX_BUSY;
1710 }
1711
1712 /* copy complete record in skb */
1713 chcr_ktls_copy_record_in_skb(nskb, skb, record);
1714 /* packet is being sent from the beginning, update the tcp_seq
1715 * accordingly.
1716 */
1717 tcp_seq = tls_record_start_seq(record);
1718 /* reset skb offset */
1719 skb_offset = 0;
1720
1721 if (last_wr)
1722 dev_kfree_skb_any(skb);
1723 else
1724 free_skb_if_tx_fails = true;
1725
1726 last_wr = true;
1727
1728 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1729 }
1730
1731 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1732 last_wr, record->len, skb_offset,
1733 record->num_frags,
1734 (last_wr && tcp_push_no_fin),
1735 mss)) {
1736 if (free_skb_if_tx_fails)
1737 dev_kfree_skb_any(skb);
1738 goto out;
1739 }
1740 tx_info->prev_seq = record->end_seq;
1741 return 0;
1742 out:
1743 dev_kfree_skb_any(nskb);
1744 return NETDEV_TX_BUSY;
1745 }
1746
1747 /*
1748 * chcr_short_record_handler: This handler will take care of the records which
1749 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1750 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1751 * This partial record might be the first part of the record, or the middle
1752 * part. In case of middle record we should fetch the prior data to make it 16
1753 * byte aligned. If it has a partial tls header or iv then get to the start of
1754 * tls header. And if it has partial TAG, then remove the complete TAG and send
1755 * only the payload.
1756 * There is one more possibility that it gets a partial header, send that
1757 * portion as a plaintext.
1758 * @tx_info - driver specific tls info.
1759 * @skb - skb contains partial record..
1760 * @record - complete record of 16K size.
1761 * @tcp_seq
1762 * @mss - segment size in which TP needs to chop a packet.
1763 * @tcp_push_no_fin - tcp push if fin is not set.
1764 * @q - TX queue.
1765 * @tls_end_offset - offset from end of the record.
1766 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1767 */
chcr_short_record_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,u32 data_len,u32 skb_offset,struct sge_eth_txq * q,u32 tls_end_offset)1768 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1769 struct sk_buff *skb,
1770 struct tls_record_info *record,
1771 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1772 u32 data_len, u32 skb_offset,
1773 struct sge_eth_txq *q, u32 tls_end_offset)
1774 {
1775 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1776 u8 prior_data[16] = {0};
1777 u32 prior_data_len = 0;
1778
1779 /* check if the skb is ending in middle of tag/HASH, its a big
1780 * trouble, send the packet before the HASH.
1781 */
1782 int remaining_record = tls_end_offset - data_len;
1783
1784 if (remaining_record > 0 &&
1785 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1786 int trimmed_len = 0;
1787
1788 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1789 trimmed_len = data_len -
1790 (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1791 remaining_record);
1792 if (!trimmed_len)
1793 return FALLBACK;
1794
1795 WARN_ON(trimmed_len > data_len);
1796
1797 data_len = trimmed_len;
1798 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1799 }
1800
1801 /* check if it is only the header part. */
1802 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1803 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1804 tcp_push_no_fin, q,
1805 tx_info->port_id, prior_data,
1806 data_len, skb_offset, prior_data_len))
1807 goto out;
1808
1809 tx_info->prev_seq = tcp_seq + data_len;
1810 return 0;
1811 }
1812
1813 /* check if the middle record's start point is 16 byte aligned. CTR
1814 * needs 16 byte aligned start point to start encryption.
1815 */
1816 if (tls_rec_offset) {
1817 /* there is an offset from start, means its a middle record */
1818 int remaining = 0;
1819
1820 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1821 prior_data_len = tls_rec_offset;
1822 tls_rec_offset = 0;
1823 remaining = 0;
1824 } else {
1825 prior_data_len =
1826 (tls_rec_offset -
1827 (TLS_HEADER_SIZE + tx_info->iv_size))
1828 % AES_BLOCK_LEN;
1829 remaining = tls_rec_offset - prior_data_len;
1830 }
1831
1832 /* if prior_data_len is not zero, means we need to fetch prior
1833 * data to make this record 16 byte aligned, or we need to reach
1834 * to start offset.
1835 */
1836 if (prior_data_len) {
1837 int i = 0;
1838 skb_frag_t *f;
1839 int frag_size = 0, frag_delta = 0;
1840
1841 while (remaining > 0) {
1842 frag_size = skb_frag_size(&record->frags[i]);
1843 if (remaining < frag_size)
1844 break;
1845
1846 remaining -= frag_size;
1847 i++;
1848 }
1849 f = &record->frags[i];
1850 frag_delta = skb_frag_size(f) - remaining;
1851
1852 if (frag_delta >= prior_data_len) {
1853 memcpy_from_page(prior_data, skb_frag_page(f),
1854 skb_frag_off(f) + remaining,
1855 prior_data_len);
1856 } else {
1857 memcpy_from_page(prior_data, skb_frag_page(f),
1858 skb_frag_off(f) + remaining,
1859 frag_delta);
1860
1861 /* get the next page */
1862 f = &record->frags[i + 1];
1863
1864 memcpy_from_page(prior_data + frag_delta,
1865 skb_frag_page(f),
1866 skb_frag_off(f),
1867 prior_data_len - frag_delta);
1868 }
1869 /* reset tcp_seq as per the prior_data_required len */
1870 tcp_seq -= prior_data_len;
1871 }
1872 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1873 } else {
1874 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1875 }
1876
1877 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1878 mss, tls_rec_offset, prior_data,
1879 prior_data_len, data_len, skb_offset)) {
1880 goto out;
1881 }
1882
1883 tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1884 return 0;
1885 out:
1886 dev_kfree_skb_any(skb);
1887 return NETDEV_TX_BUSY;
1888 }
1889
chcr_ktls_sw_fallback(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1890 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1891 struct chcr_ktls_info *tx_info,
1892 struct sge_eth_txq *q)
1893 {
1894 u32 data_len, skb_offset;
1895 struct sk_buff *nskb;
1896 struct tcphdr *th;
1897
1898 nskb = tls_encrypt_skb(skb);
1899
1900 if (!nskb)
1901 return 0;
1902
1903 th = tcp_hdr(nskb);
1904 skb_offset = skb_tcp_all_headers(nskb);
1905 data_len = nskb->len - skb_offset;
1906 skb_tx_timestamp(nskb);
1907
1908 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1909 goto out;
1910
1911 tx_info->prev_seq = ntohl(th->seq) + data_len;
1912 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1913 return 0;
1914 out:
1915 dev_kfree_skb_any(nskb);
1916 return 0;
1917 }
1918 /* nic tls TX handler */
chcr_ktls_xmit(struct sk_buff * skb,struct net_device * dev)1919 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1920 {
1921 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1922 struct ch_ktls_port_stats_debug *port_stats;
1923 struct tls_offload_context_tx *tx_ctx;
1924 struct ch_ktls_stats_debug *stats;
1925 struct tcphdr *th = tcp_hdr(skb);
1926 int data_len, qidx, ret = 0, mss;
1927 struct tls_record_info *record;
1928 struct chcr_ktls_info *tx_info;
1929 struct net_device *tls_netdev;
1930 struct tls_context *tls_ctx;
1931 struct sge_eth_txq *q;
1932 struct adapter *adap;
1933 unsigned long flags;
1934
1935 tcp_seq = ntohl(th->seq);
1936 skb_offset = skb_tcp_all_headers(skb);
1937 skb_data_len = skb->len - skb_offset;
1938 data_len = skb_data_len;
1939
1940 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1941
1942 tls_ctx = tls_get_ctx(skb->sk);
1943 tx_ctx = tls_offload_ctx_tx(tls_ctx);
1944 tls_netdev = rcu_dereference_bh(tls_ctx->netdev);
1945 /* Don't quit on NULL: if tls_device_down is running in parallel,
1946 * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was
1947 * true. Rather continue processing this packet.
1948 */
1949 if (unlikely(tls_netdev && tls_netdev != dev))
1950 goto out;
1951
1952 tx_info = chcr_get_ktls_tx_info(tls_ctx);
1953
1954 if (unlikely(!tx_info))
1955 goto out;
1956
1957 adap = tx_info->adap;
1958 stats = &adap->ch_ktls_stats;
1959 port_stats = &stats->ktls_port[tx_info->port_id];
1960
1961 qidx = skb->queue_mapping;
1962 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1963 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1964 /* if tcp options are set but finish is not send the options first */
1965 if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1966 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1967 tx_info->tx_chan);
1968 if (ret)
1969 return NETDEV_TX_BUSY;
1970 }
1971
1972 /* TCP segments can be in received either complete or partial.
1973 * chcr_end_part_handler will handle cases if complete record or end
1974 * part of the record is received. In case of partial end part of record,
1975 * we will send the complete record again.
1976 */
1977
1978 spin_lock_irqsave(&tx_ctx->lock, flags);
1979
1980 do {
1981
1982 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1983 /* fetch the tls record */
1984 record = tls_get_record(tx_ctx, tcp_seq,
1985 &tx_info->record_no);
1986 /* By the time packet reached to us, ACK is received, and record
1987 * won't be found in that case, handle it gracefully.
1988 */
1989 if (unlikely(!record)) {
1990 spin_unlock_irqrestore(&tx_ctx->lock, flags);
1991 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
1992 goto out;
1993 }
1994
1995 tls_end_offset = record->end_seq - tcp_seq;
1996
1997 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1998 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1999 /* update tcb for the skb */
2000 if (skb_data_len == data_len) {
2001 u32 tx_max = tcp_seq;
2002
2003 if (!tls_record_is_start_marker(record) &&
2004 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2005 tx_max = record->end_seq -
2006 TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2007
2008 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2009 ntohl(th->ack_seq),
2010 ntohs(th->window),
2011 tls_end_offset !=
2012 record->len);
2013 if (ret) {
2014 spin_unlock_irqrestore(&tx_ctx->lock,
2015 flags);
2016 goto out;
2017 }
2018
2019 if (th->fin)
2020 skb_get(skb);
2021 }
2022
2023 if (unlikely(tls_record_is_start_marker(record))) {
2024 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2025 /* If tls_end_offset < data_len, means there is some
2026 * data after start marker, which needs encryption, send
2027 * plaintext first and take skb refcount. else send out
2028 * complete pkt as plaintext.
2029 */
2030 if (tls_end_offset < data_len)
2031 skb_get(skb);
2032 else
2033 tls_end_offset = data_len;
2034
2035 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2036 (!th->fin && th->psh), q,
2037 tx_info->port_id, NULL,
2038 tls_end_offset, skb_offset,
2039 0);
2040
2041 if (ret) {
2042 /* free the refcount taken earlier */
2043 if (tls_end_offset < data_len)
2044 dev_kfree_skb_any(skb);
2045 spin_unlock_irqrestore(&tx_ctx->lock, flags);
2046 goto out;
2047 }
2048
2049 data_len -= tls_end_offset;
2050 tcp_seq = record->end_seq;
2051 skb_offset += tls_end_offset;
2052 continue;
2053 }
2054
2055 /* if a tls record is finishing in this SKB */
2056 if (tls_end_offset <= data_len) {
2057 ret = chcr_end_part_handler(tx_info, skb, record,
2058 tcp_seq, mss,
2059 (!th->fin && th->psh), q,
2060 skb_offset,
2061 tls_end_offset,
2062 skb_offset +
2063 tls_end_offset == skb->len);
2064
2065 data_len -= tls_end_offset;
2066 /* tcp_seq increment is required to handle next record.
2067 */
2068 tcp_seq += tls_end_offset;
2069 skb_offset += tls_end_offset;
2070 } else {
2071 ret = chcr_short_record_handler(tx_info, skb,
2072 record, tcp_seq, mss,
2073 (!th->fin && th->psh),
2074 data_len, skb_offset,
2075 q, tls_end_offset);
2076 data_len = 0;
2077 }
2078
2079 /* if any failure, come out from the loop. */
2080 if (ret) {
2081 spin_unlock_irqrestore(&tx_ctx->lock, flags);
2082 if (th->fin)
2083 dev_kfree_skb_any(skb);
2084
2085 if (ret == FALLBACK)
2086 return chcr_ktls_sw_fallback(skb, tx_info, q);
2087
2088 return NETDEV_TX_OK;
2089 }
2090
2091 /* length should never be less than 0 */
2092 WARN_ON(data_len < 0);
2093
2094 } while (data_len > 0);
2095
2096 spin_unlock_irqrestore(&tx_ctx->lock, flags);
2097 atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2098 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2099
2100 /* tcp finish is set, send a separate tcp msg including all the options
2101 * as well.
2102 */
2103 if (th->fin) {
2104 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2105 dev_kfree_skb_any(skb);
2106 }
2107
2108 return NETDEV_TX_OK;
2109 out:
2110 dev_kfree_skb_any(skb);
2111 return NETDEV_TX_OK;
2112 }
2113
chcr_ktls_uld_add(const struct cxgb4_lld_info * lldi)2114 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2115 {
2116 struct chcr_ktls_uld_ctx *u_ctx;
2117
2118 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2119 CHCR_KTLS_DRV_VERSION);
2120 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2121 if (!u_ctx) {
2122 u_ctx = ERR_PTR(-ENOMEM);
2123 goto out;
2124 }
2125 u_ctx->lldi = *lldi;
2126 u_ctx->detach = false;
2127 xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH);
2128 out:
2129 return u_ctx;
2130 }
2131
2132 static const struct tlsdev_ops chcr_ktls_ops = {
2133 .tls_dev_add = chcr_ktls_dev_add,
2134 .tls_dev_del = chcr_ktls_dev_del,
2135 };
2136
2137 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2138 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2139 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2140 };
2141
chcr_ktls_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)2142 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2143 const struct pkt_gl *pgl)
2144 {
2145 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2146 struct chcr_ktls_uld_ctx *u_ctx = handle;
2147 u8 opcode = rpl->ot.opcode;
2148 struct adapter *adap;
2149
2150 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2151
2152 if (!work_handlers[opcode]) {
2153 pr_err("Unsupported opcode %d received\n", opcode);
2154 return 0;
2155 }
2156
2157 work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2158 return 0;
2159 }
2160
clear_conn_resources(struct chcr_ktls_info * tx_info)2161 static void clear_conn_resources(struct chcr_ktls_info *tx_info)
2162 {
2163 /* clear l2t entry */
2164 if (tx_info->l2te)
2165 cxgb4_l2t_release(tx_info->l2te);
2166
2167 #if IS_ENABLED(CONFIG_IPV6)
2168 /* clear clip entry */
2169 if (tx_info->ip_family == AF_INET6)
2170 cxgb4_clip_release(tx_info->netdev, (const u32 *)
2171 &tx_info->sk->sk_v6_rcv_saddr,
2172 1);
2173 #endif
2174
2175 /* clear tid */
2176 if (tx_info->tid != -1)
2177 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
2178 tx_info->tid, tx_info->ip_family);
2179 }
2180
ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx * u_ctx)2181 static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx)
2182 {
2183 struct ch_ktls_port_stats_debug *port_stats;
2184 struct tls_offload_context_tx *tx_ctx;
2185 struct chcr_ktls_info *tx_info;
2186 unsigned long index;
2187
2188 xa_for_each(&u_ctx->tid_list, index, tx_ctx) {
2189 tx_info = __chcr_get_ktls_tx_info(tx_ctx);
2190 clear_conn_resources(tx_info);
2191 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
2192 atomic64_inc(&port_stats->ktls_tx_connection_close);
2193 kvfree(tx_info);
2194 memset(tx_ctx->driver_state, 0, TLS_DRIVER_STATE_SIZE_TX);
2195 /* release module refcount */
2196 module_put(THIS_MODULE);
2197 }
2198 }
2199
chcr_ktls_uld_state_change(void * handle,enum cxgb4_state new_state)2200 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2201 {
2202 struct chcr_ktls_uld_ctx *u_ctx = handle;
2203
2204 switch (new_state) {
2205 case CXGB4_STATE_UP:
2206 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2207 mutex_lock(&dev_mutex);
2208 list_add_tail(&u_ctx->entry, &uld_ctx_list);
2209 mutex_unlock(&dev_mutex);
2210 break;
2211 case CXGB4_STATE_START_RECOVERY:
2212 case CXGB4_STATE_DOWN:
2213 case CXGB4_STATE_DETACH:
2214 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2215 mutex_lock(&dev_mutex);
2216 u_ctx->detach = true;
2217 list_del(&u_ctx->entry);
2218 ch_ktls_reset_all_conn(u_ctx);
2219 xa_destroy(&u_ctx->tid_list);
2220 mutex_unlock(&dev_mutex);
2221 break;
2222 default:
2223 break;
2224 }
2225
2226 return 0;
2227 }
2228
2229 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2230 .name = CHCR_KTLS_DRV_MODULE_NAME,
2231 .nrxq = 1,
2232 .rxq_size = 1024,
2233 .add = chcr_ktls_uld_add,
2234 .tx_handler = chcr_ktls_xmit,
2235 .rx_handler = chcr_ktls_uld_rx_handler,
2236 .state_change = chcr_ktls_uld_state_change,
2237 .tlsdev_ops = &chcr_ktls_ops,
2238 };
2239
chcr_ktls_init(void)2240 static int __init chcr_ktls_init(void)
2241 {
2242 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2243 return 0;
2244 }
2245
chcr_ktls_exit(void)2246 static void __exit chcr_ktls_exit(void)
2247 {
2248 struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2249 struct adapter *adap;
2250
2251 pr_info("driver unloaded\n");
2252
2253 mutex_lock(&dev_mutex);
2254 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2255 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2256 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2257 list_del(&u_ctx->entry);
2258 xa_destroy(&u_ctx->tid_list);
2259 kfree(u_ctx);
2260 }
2261 mutex_unlock(&dev_mutex);
2262 cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2263 }
2264
2265 module_init(chcr_ktls_init);
2266 module_exit(chcr_ktls_exit);
2267
2268 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2269 MODULE_LICENSE("GPL");
2270 MODULE_AUTHOR("Chelsio Communications");
2271 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);
2272