Lines Matching full:strp
16 void tls_strp_abort_strp(struct tls_strparser *strp, int err) in tls_strp_abort_strp() argument
18 if (strp->stopped) in tls_strp_abort_strp()
21 strp->stopped = 1; in tls_strp_abort_strp()
24 WRITE_ONCE(strp->sk->sk_err, -err); in tls_strp_abort_strp()
27 sk_error_report(strp->sk); in tls_strp_abort_strp()
30 static void tls_strp_anchor_free(struct tls_strparser *strp) in tls_strp_anchor_free() argument
32 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_anchor_free()
35 if (!strp->copy_mode) in tls_strp_anchor_free()
37 consume_skb(strp->anchor); in tls_strp_anchor_free()
38 strp->anchor = NULL; in tls_strp_anchor_free()
42 tls_strp_skb_copy(struct tls_strparser *strp, struct sk_buff *in_skb, in tls_strp_skb_copy() argument
49 &err, strp->sk->sk_allocation); in tls_strp_skb_copy()
69 static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp) in tls_strp_msg_make_copy() argument
74 skb = tls_strp_skb_copy(strp, strp->anchor, strp->stm.offset, in tls_strp_msg_make_copy()
75 strp->stm.full_len); in tls_strp_msg_make_copy()
87 struct tls_strparser *strp = &ctx->strp; in tls_strp_msg_detach() local
90 DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted); in tls_strp_msg_detach()
98 if (strp->copy_mode) { in tls_strp_msg_detach()
105 skb = alloc_skb(0, strp->sk->sk_allocation); in tls_strp_msg_detach()
109 swap(strp->anchor, skb); in tls_strp_msg_detach()
113 return tls_strp_msg_make_copy(strp); in tls_strp_msg_detach()
122 struct tls_strparser *strp = &ctx->strp; in tls_strp_msg_cow() local
125 if (strp->copy_mode) in tls_strp_msg_cow()
128 skb = tls_strp_msg_make_copy(strp); in tls_strp_msg_cow()
132 tls_strp_anchor_free(strp); in tls_strp_msg_cow()
133 strp->anchor = skb; in tls_strp_msg_cow()
135 tcp_read_done(strp->sk, strp->stm.full_len); in tls_strp_msg_cow()
136 strp->copy_mode = 1; in tls_strp_msg_cow()
145 int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst) in tls_strp_msg_hold() argument
147 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_msg_hold()
149 if (strp->copy_mode) { in tls_strp_msg_hold()
155 skb = alloc_skb(0, strp->sk->sk_allocation); in tls_strp_msg_hold()
159 __skb_queue_tail(dst, strp->anchor); in tls_strp_msg_hold()
160 strp->anchor = skb; in tls_strp_msg_hold()
165 offset = strp->stm.offset; in tls_strp_msg_hold()
166 len = strp->stm.full_len; in tls_strp_msg_hold()
178 clone = skb_clone(iter, strp->sk->sk_allocation); in tls_strp_msg_hold()
192 static void tls_strp_flush_anchor_copy(struct tls_strparser *strp) in tls_strp_flush_anchor_copy() argument
194 struct skb_shared_info *shinfo = skb_shinfo(strp->anchor); in tls_strp_flush_anchor_copy()
202 if (strp->copy_mode) { in tls_strp_flush_anchor_copy()
206 strp->copy_mode = 0; in tls_strp_flush_anchor_copy()
207 strp->mixed_decrypted = 0; in tls_strp_flush_anchor_copy()
210 static int tls_strp_copyin_frag(struct tls_strparser *strp, struct sk_buff *skb, in tls_strp_copyin_frag() argument
228 if (!strp->stm.full_len) { in tls_strp_copyin_frag()
240 sz = tls_rx_msg_size(strp, skb); in tls_strp_copyin_frag()
260 strp->stm.full_len = sz; in tls_strp_copyin_frag()
261 if (!strp->stm.full_len) in tls_strp_copyin_frag()
266 while (len && strp->stm.full_len > skb->len) { in tls_strp_copyin_frag()
267 chunk = min_t(size_t, len, strp->stm.full_len - skb->len); in tls_strp_copyin_frag()
286 static int tls_strp_copyin_skb(struct tls_strparser *strp, struct sk_buff *skb, in tls_strp_copyin_skb() argument
295 if (strp->stm.full_len) in tls_strp_copyin_skb()
296 chunk = strp->stm.full_len - skb->len; in tls_strp_copyin_skb()
301 nskb = tls_strp_skb_copy(strp, in_skb, offset, chunk); in tls_strp_copyin_skb()
319 if (!strp->stm.full_len) { in tls_strp_copyin_skb()
320 sz = tls_rx_msg_size(strp, skb); in tls_strp_copyin_skb()
336 strp->stm.full_len = sz; in tls_strp_copyin_skb()
345 struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data; in tls_strp_copyin() local
349 if (strp->msg_ready) in tls_strp_copyin()
352 skb = strp->anchor; in tls_strp_copyin()
356 strp->mixed_decrypted |= !!skb_cmp_decrypted(skb, in_skb); in tls_strp_copyin()
358 if (IS_ENABLED(CONFIG_TLS_DEVICE) && strp->mixed_decrypted) in tls_strp_copyin()
359 ret = tls_strp_copyin_skb(strp, skb, in_skb, offset, in_len); in tls_strp_copyin()
361 ret = tls_strp_copyin_frag(strp, skb, in_skb, offset, in_len); in tls_strp_copyin()
367 if (strp->stm.full_len && strp->stm.full_len == skb->len) { in tls_strp_copyin()
370 WRITE_ONCE(strp->msg_ready, 1); in tls_strp_copyin()
371 tls_rx_msg_ready(strp); in tls_strp_copyin()
377 static int tls_strp_read_copyin(struct tls_strparser *strp) in tls_strp_read_copyin() argument
381 desc.arg.data = strp; in tls_strp_read_copyin()
386 tcp_read_sock(strp->sk, &desc, tls_strp_copyin); in tls_strp_read_copyin()
391 static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) in tls_strp_read_copy() argument
401 if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX))) in tls_strp_read_copy()
404 shinfo = skb_shinfo(strp->anchor); in tls_strp_read_copy()
407 need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE; in tls_strp_read_copy()
410 page = alloc_page(strp->sk->sk_allocation); in tls_strp_read_copy()
412 tls_strp_flush_anchor_copy(strp); in tls_strp_read_copy()
416 skb_fill_page_desc(strp->anchor, shinfo->nr_frags++, in tls_strp_read_copy()
422 strp->copy_mode = 1; in tls_strp_read_copy()
423 strp->stm.offset = 0; in tls_strp_read_copy()
425 strp->anchor->len = 0; in tls_strp_read_copy()
426 strp->anchor->data_len = 0; in tls_strp_read_copy()
427 strp->anchor->truesize = round_up(need_spc, PAGE_SIZE); in tls_strp_read_copy()
429 tls_strp_read_copyin(strp); in tls_strp_read_copy()
434 static bool tls_strp_check_queue_ok(struct tls_strparser *strp) in tls_strp_check_queue_ok() argument
436 unsigned int len = strp->stm.offset + strp->stm.full_len; in tls_strp_check_queue_ok()
440 first = skb_shinfo(strp->anchor)->frag_list; in tls_strp_check_queue_ok()
461 static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len) in tls_strp_load_anchor_with_queue() argument
463 struct tcp_sock *tp = tcp_sk(strp->sk); in tls_strp_load_anchor_with_queue()
467 first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset); in tls_strp_load_anchor_with_queue()
472 strp->anchor->len = offset + len; in tls_strp_load_anchor_with_queue()
473 strp->anchor->data_len = offset + len; in tls_strp_load_anchor_with_queue()
474 strp->anchor->truesize = offset + len; in tls_strp_load_anchor_with_queue()
476 skb_shinfo(strp->anchor)->frag_list = first; in tls_strp_load_anchor_with_queue()
478 skb_copy_header(strp->anchor, first); in tls_strp_load_anchor_with_queue()
479 strp->anchor->destructor = NULL; in tls_strp_load_anchor_with_queue()
481 strp->stm.offset = offset; in tls_strp_load_anchor_with_queue()
484 bool tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh) in tls_strp_msg_load() argument
489 DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready); in tls_strp_msg_load()
490 DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len); in tls_strp_msg_load()
492 if (!strp->copy_mode && force_refresh) { in tls_strp_msg_load()
493 if (unlikely(tcp_inq(strp->sk) < strp->stm.full_len)) { in tls_strp_msg_load()
494 WRITE_ONCE(strp->msg_ready, 0); in tls_strp_msg_load()
495 memset(&strp->stm, 0, sizeof(strp->stm)); in tls_strp_msg_load()
499 tls_strp_load_anchor_with_queue(strp, strp->stm.full_len); in tls_strp_msg_load()
502 rxm = strp_msg(strp->anchor); in tls_strp_msg_load()
503 rxm->full_len = strp->stm.full_len; in tls_strp_msg_load()
504 rxm->offset = strp->stm.offset; in tls_strp_msg_load()
505 tlm = tls_msg(strp->anchor); in tls_strp_msg_load()
506 tlm->control = strp->mark; in tls_strp_msg_load()
512 static int tls_strp_read_sock(struct tls_strparser *strp) in tls_strp_read_sock() argument
516 inq = tcp_inq(strp->sk); in tls_strp_read_sock()
520 if (unlikely(strp->copy_mode)) in tls_strp_read_sock()
521 return tls_strp_read_copyin(strp); in tls_strp_read_sock()
523 if (inq < strp->stm.full_len) in tls_strp_read_sock()
524 return tls_strp_read_copy(strp, true); in tls_strp_read_sock()
526 tls_strp_load_anchor_with_queue(strp, inq); in tls_strp_read_sock()
527 if (!strp->stm.full_len) { in tls_strp_read_sock()
528 sz = tls_rx_msg_size(strp, strp->anchor); in tls_strp_read_sock()
532 strp->stm.full_len = sz; in tls_strp_read_sock()
534 if (!strp->stm.full_len || inq < strp->stm.full_len) in tls_strp_read_sock()
535 return tls_strp_read_copy(strp, true); in tls_strp_read_sock()
538 if (!tls_strp_check_queue_ok(strp)) in tls_strp_read_sock()
539 return tls_strp_read_copy(strp, false); in tls_strp_read_sock()
541 WRITE_ONCE(strp->msg_ready, 1); in tls_strp_read_sock()
542 tls_rx_msg_ready(strp); in tls_strp_read_sock()
547 void tls_strp_check_rcv(struct tls_strparser *strp) in tls_strp_check_rcv() argument
549 if (unlikely(strp->stopped) || strp->msg_ready) in tls_strp_check_rcv()
552 if (tls_strp_read_sock(strp) == -ENOMEM) in tls_strp_check_rcv()
553 queue_work(tls_strp_wq, &strp->work); in tls_strp_check_rcv()
557 void tls_strp_data_ready(struct tls_strparser *strp) in tls_strp_data_ready() argument
566 if (sock_owned_by_user_nocheck(strp->sk)) { in tls_strp_data_ready()
567 queue_work(tls_strp_wq, &strp->work); in tls_strp_data_ready()
571 tls_strp_check_rcv(strp); in tls_strp_data_ready()
576 struct tls_strparser *strp = in tls_strp_work() local
579 lock_sock(strp->sk); in tls_strp_work()
580 tls_strp_check_rcv(strp); in tls_strp_work()
581 release_sock(strp->sk); in tls_strp_work()
584 void tls_strp_msg_done(struct tls_strparser *strp) in tls_strp_msg_done() argument
586 WARN_ON(!strp->stm.full_len); in tls_strp_msg_done()
588 if (likely(!strp->copy_mode)) in tls_strp_msg_done()
589 tcp_read_done(strp->sk, strp->stm.full_len); in tls_strp_msg_done()
591 tls_strp_flush_anchor_copy(strp); in tls_strp_msg_done()
593 WRITE_ONCE(strp->msg_ready, 0); in tls_strp_msg_done()
594 memset(&strp->stm, 0, sizeof(strp->stm)); in tls_strp_msg_done()
596 tls_strp_check_rcv(strp); in tls_strp_msg_done()
599 void tls_strp_stop(struct tls_strparser *strp) in tls_strp_stop() argument
601 strp->stopped = 1; in tls_strp_stop()
604 int tls_strp_init(struct tls_strparser *strp, struct sock *sk) in tls_strp_init() argument
606 memset(strp, 0, sizeof(*strp)); in tls_strp_init()
608 strp->sk = sk; in tls_strp_init()
610 strp->anchor = alloc_skb(0, GFP_KERNEL); in tls_strp_init()
611 if (!strp->anchor) in tls_strp_init()
614 INIT_WORK(&strp->work, tls_strp_work); in tls_strp_init()
619 /* strp must already be stopped so that tls_strp_recv will no longer be called.
622 void tls_strp_done(struct tls_strparser *strp) in tls_strp_done() argument
624 WARN_ON(!strp->stopped); in tls_strp_done()
626 cancel_work_sync(&strp->work); in tls_strp_done()
627 __tls_strp_done(strp); in tls_strp_done()
631 void __tls_strp_done(struct tls_strparser *strp) in __tls_strp_done() argument
633 tls_strp_anchor_free(strp); in __tls_strp_done()
638 tls_strp_wq = create_workqueue("tls-strp"); in tls_strp_dev_init()