1 // SPDX-License-Identifier: GPL-2.0
2 #include <net/tcp.h>
3 #include <net/strparser.h>
4 #include <net/xfrm.h>
5 #include <net/esp.h>
6 #include <net/espintcp.h>
7 #include <linux/skmsg.h>
8 #include <net/inet_common.h>
9 #include <trace/events/sock.h>
10 #include <net/hotdata.h>
11
handle_nonesp(struct espintcp_ctx * ctx,struct sk_buff * skb,struct sock * sk)12 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb,
13 struct sock *sk)
14 {
15 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf ||
16 !sk_rmem_schedule(sk, skb, skb->truesize)) {
17 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
18 kfree_skb(skb);
19 return;
20 }
21
22 skb_set_owner_r(skb, sk);
23
24 memset(skb->cb, 0, sizeof(skb->cb));
25 skb_queue_tail(&ctx->ike_queue, skb);
26 ctx->saved_data_ready(sk);
27 }
28
handle_esp(struct sk_buff * skb,struct sock * sk)29 static void handle_esp(struct sk_buff *skb, struct sock *sk)
30 {
31 struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb;
32
33 if (!skb_reset_transport_header_careful(skb)) {
34 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
35 kfree_skb(skb);
36 return;
37 }
38
39 /* restore IP CB, we need at least IP6CB->nhoff */
40 memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header));
41
42 rcu_read_lock();
43 skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif);
44 if (!skb->dev) {
45 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR);
46 kfree_skb(skb);
47 goto out;
48 }
49 local_bh_disable();
50 #if IS_ENABLED(CONFIG_IPV6)
51 if (sk->sk_family == AF_INET6)
52 xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
53 else
54 #endif
55 xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP);
56 local_bh_enable();
57 out:
58 rcu_read_unlock();
59 }
60
espintcp_rcv(struct strparser * strp,struct sk_buff * skb)61 static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb)
62 {
63 struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx,
64 strp);
65 struct strp_msg *rxm = strp_msg(skb);
66 int len = rxm->full_len - 2;
67 u32 nonesp_marker;
68 int err;
69
70 /* keepalive packet? */
71 if (unlikely(len == 1)) {
72 u8 data;
73
74 err = skb_copy_bits(skb, rxm->offset + 2, &data, 1);
75 if (err < 0) {
76 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
77 kfree_skb(skb);
78 return;
79 }
80
81 if (data == 0xff) {
82 kfree_skb(skb);
83 return;
84 }
85 }
86
87 /* drop other short messages */
88 if (unlikely(len <= sizeof(nonesp_marker))) {
89 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
90 kfree_skb(skb);
91 return;
92 }
93
94 err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker,
95 sizeof(nonesp_marker));
96 if (err < 0) {
97 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR);
98 kfree_skb(skb);
99 return;
100 }
101
102 /* remove header, leave non-ESP marker/SPI */
103 if (!pskb_pull(skb, rxm->offset + 2)) {
104 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
105 kfree_skb(skb);
106 return;
107 }
108
109 if (pskb_trim(skb, rxm->full_len - 2) != 0) {
110 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
111 kfree_skb(skb);
112 return;
113 }
114
115 if (nonesp_marker == 0)
116 handle_nonesp(ctx, skb, strp->sk);
117 else
118 handle_esp(skb, strp->sk);
119 }
120
espintcp_parse(struct strparser * strp,struct sk_buff * skb)121 static int espintcp_parse(struct strparser *strp, struct sk_buff *skb)
122 {
123 struct strp_msg *rxm = strp_msg(skb);
124 __be16 blen;
125 u16 len;
126 int err;
127
128 if (skb->len < rxm->offset + 2)
129 return 0;
130
131 err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen));
132 if (err < 0)
133 return err;
134
135 len = be16_to_cpu(blen);
136 if (len < 2)
137 return -EINVAL;
138
139 return len;
140 }
141
espintcp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int flags)142 static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
143 int flags)
144 {
145 struct espintcp_ctx *ctx = espintcp_getctx(sk);
146 struct sk_buff *skb;
147 int err = 0;
148 int copied;
149 int off = 0;
150
151 skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err);
152 if (!skb) {
153 if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN)
154 return 0;
155 return err;
156 }
157
158 copied = len;
159 if (copied > skb->len)
160 copied = skb->len;
161 else if (copied < skb->len)
162 msg->msg_flags |= MSG_TRUNC;
163
164 err = skb_copy_datagram_msg(skb, 0, msg, copied);
165 if (unlikely(err)) {
166 kfree_skb(skb);
167 return err;
168 }
169
170 if (flags & MSG_TRUNC)
171 copied = skb->len;
172 kfree_skb(skb);
173 return copied;
174 }
175
espintcp_queue_out(struct sock * sk,struct sk_buff * skb)176 int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
177 {
178 struct espintcp_ctx *ctx = espintcp_getctx(sk);
179
180 if (skb_queue_len(&ctx->out_queue) >=
181 READ_ONCE(net_hotdata.max_backlog)) {
182 kfree_skb(skb);
183 return -ENOBUFS;
184 }
185
186 __skb_queue_tail(&ctx->out_queue, skb);
187
188 return 0;
189 }
190 EXPORT_SYMBOL_GPL(espintcp_queue_out);
191
192 /* espintcp length field is 2B and length includes the length field's size */
193 #define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2)
194
espintcp_sendskb_locked(struct sock * sk,struct espintcp_msg * emsg,int flags)195 static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg,
196 int flags)
197 {
198 do {
199 int ret;
200
201 ret = skb_send_sock_locked(sk, emsg->skb,
202 emsg->offset, emsg->len);
203 if (ret < 0)
204 return ret;
205
206 emsg->len -= ret;
207 emsg->offset += ret;
208 } while (emsg->len > 0);
209
210 kfree_skb(emsg->skb);
211 memset(emsg, 0, sizeof(*emsg));
212
213 return 0;
214 }
215
espintcp_sendskmsg_locked(struct sock * sk,struct espintcp_msg * emsg,int flags)216 static int espintcp_sendskmsg_locked(struct sock *sk,
217 struct espintcp_msg *emsg, int flags)
218 {
219 struct msghdr msghdr = {
220 .msg_flags = flags | MSG_SPLICE_PAGES | MSG_MORE,
221 };
222 struct sk_msg *skmsg = &emsg->skmsg;
223 bool more = flags & MSG_MORE;
224 struct scatterlist *sg;
225 int ret;
226
227 do {
228 struct bio_vec bvec;
229
230 sg = &skmsg->sg.data[skmsg->sg.start];
231 if (sg_is_last(sg) && !more)
232 msghdr.msg_flags &= ~MSG_MORE;
233
234 bvec_set_page(&bvec, sg_page(sg), sg->length, sg->offset);
235 iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, sg->length);
236 ret = tcp_sendmsg_locked(sk, &msghdr, sg->length);
237 if (ret < 0)
238 return ret;
239
240 sk_msg_free_partial(sk, skmsg, ret);
241 } while (skmsg->sg.size);
242
243 memset(emsg, 0, sizeof(*emsg));
244
245 return 0;
246 }
247
espintcp_push_msgs(struct sock * sk,int flags)248 static int espintcp_push_msgs(struct sock *sk, int flags)
249 {
250 struct espintcp_ctx *ctx = espintcp_getctx(sk);
251 struct espintcp_msg *emsg = &ctx->partial;
252 int err;
253
254 if (!emsg->len)
255 return 0;
256
257 if (ctx->tx_running)
258 return -EAGAIN;
259 ctx->tx_running = 1;
260
261 if (emsg->skb)
262 err = espintcp_sendskb_locked(sk, emsg, flags);
263 else
264 err = espintcp_sendskmsg_locked(sk, emsg, flags);
265 if (err == -EAGAIN) {
266 ctx->tx_running = 0;
267 return flags & MSG_DONTWAIT ? -EAGAIN : 0;
268 }
269 if (!err)
270 memset(emsg, 0, sizeof(*emsg));
271
272 ctx->tx_running = 0;
273
274 return err;
275 }
276
espintcp_push_skb(struct sock * sk,struct sk_buff * skb)277 int espintcp_push_skb(struct sock *sk, struct sk_buff *skb)
278 {
279 struct espintcp_ctx *ctx = espintcp_getctx(sk);
280 struct espintcp_msg *emsg = &ctx->partial;
281 unsigned int len;
282 int offset;
283
284 if (sk->sk_state != TCP_ESTABLISHED) {
285 kfree_skb(skb);
286 return -ECONNRESET;
287 }
288
289 offset = skb_transport_offset(skb);
290 len = skb->len - offset;
291
292 espintcp_push_msgs(sk, 0);
293
294 if (emsg->len) {
295 kfree_skb(skb);
296 return -ENOBUFS;
297 }
298
299 skb_set_owner_w(skb, sk);
300
301 emsg->offset = offset;
302 emsg->len = len;
303 emsg->skb = skb;
304
305 espintcp_push_msgs(sk, 0);
306
307 return 0;
308 }
309 EXPORT_SYMBOL_GPL(espintcp_push_skb);
310
espintcp_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)311 static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
312 {
313 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
314 struct espintcp_ctx *ctx = espintcp_getctx(sk);
315 struct espintcp_msg *emsg = &ctx->partial;
316 struct iov_iter pfx_iter;
317 struct kvec pfx_iov = {};
318 size_t msglen = size + 2;
319 char buf[2] = {0};
320 int err, end;
321
322 if (msg->msg_flags & ~MSG_DONTWAIT)
323 return -EOPNOTSUPP;
324
325 if (size > MAX_ESPINTCP_MSG)
326 return -EMSGSIZE;
327
328 if (msg->msg_controllen)
329 return -EOPNOTSUPP;
330
331 lock_sock(sk);
332
333 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
334 if (err < 0) {
335 if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT))
336 err = -ENOBUFS;
337 goto unlock;
338 }
339 if (emsg->len) {
340 err = -ENOBUFS;
341 goto unlock;
342 }
343
344 sk_msg_init(&emsg->skmsg);
345 while (1) {
346 /* only -ENOMEM is possible since we don't coalesce */
347 err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0);
348 if (!err)
349 break;
350
351 err = sk_stream_wait_memory(sk, &timeo);
352 if (err)
353 goto fail;
354 }
355
356 *((__be16 *)buf) = cpu_to_be16(msglen);
357 pfx_iov.iov_base = buf;
358 pfx_iov.iov_len = sizeof(buf);
359 iov_iter_kvec(&pfx_iter, ITER_SOURCE, &pfx_iov, 1, pfx_iov.iov_len);
360
361 err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg,
362 pfx_iov.iov_len);
363 if (err < 0)
364 goto fail;
365
366 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size);
367 if (err < 0)
368 goto fail;
369
370 end = emsg->skmsg.sg.end;
371 emsg->len = size;
372 sk_msg_iter_var_prev(end);
373 sg_mark_end(sk_msg_elem(&emsg->skmsg, end));
374
375 tcp_rate_check_app_limited(sk);
376
377 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT);
378 /* this message could be partially sent, keep it */
379
380 release_sock(sk);
381
382 return size;
383
384 fail:
385 sk_msg_free(sk, &emsg->skmsg);
386 memset(emsg, 0, sizeof(*emsg));
387 unlock:
388 release_sock(sk);
389 return err;
390 }
391
392 static struct proto espintcp_prot __ro_after_init;
393 static struct proto_ops espintcp_ops __ro_after_init;
394 static struct proto espintcp6_prot;
395 static struct proto_ops espintcp6_ops;
396 static DEFINE_MUTEX(tcpv6_prot_mutex);
397
espintcp_data_ready(struct sock * sk)398 static void espintcp_data_ready(struct sock *sk)
399 {
400 struct espintcp_ctx *ctx = espintcp_getctx(sk);
401
402 trace_sk_data_ready(sk);
403
404 strp_data_ready(&ctx->strp);
405 }
406
espintcp_tx_work(struct work_struct * work)407 static void espintcp_tx_work(struct work_struct *work)
408 {
409 struct espintcp_ctx *ctx = container_of(work,
410 struct espintcp_ctx, work);
411 struct sock *sk = ctx->strp.sk;
412
413 lock_sock(sk);
414 if (!ctx->tx_running)
415 espintcp_push_msgs(sk, 0);
416 release_sock(sk);
417 }
418
espintcp_write_space(struct sock * sk)419 static void espintcp_write_space(struct sock *sk)
420 {
421 struct espintcp_ctx *ctx = espintcp_getctx(sk);
422
423 schedule_work(&ctx->work);
424 ctx->saved_write_space(sk);
425 }
426
espintcp_destruct(struct sock * sk)427 static void espintcp_destruct(struct sock *sk)
428 {
429 struct espintcp_ctx *ctx = espintcp_getctx(sk);
430
431 ctx->saved_destruct(sk);
432 kfree(ctx);
433 }
434
tcp_is_ulp_esp(struct sock * sk)435 bool tcp_is_ulp_esp(struct sock *sk)
436 {
437 return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot;
438 }
439 EXPORT_SYMBOL_GPL(tcp_is_ulp_esp);
440
441 static void build_protos(struct proto *espintcp_prot,
442 struct proto_ops *espintcp_ops,
443 const struct proto *orig_prot,
444 const struct proto_ops *orig_ops);
espintcp_init_sk(struct sock * sk)445 static int espintcp_init_sk(struct sock *sk)
446 {
447 struct inet_connection_sock *icsk = inet_csk(sk);
448 struct strp_callbacks cb = {
449 .rcv_msg = espintcp_rcv,
450 .parse_msg = espintcp_parse,
451 };
452 struct espintcp_ctx *ctx;
453 int err;
454
455 /* sockmap is not compatible with espintcp */
456 if (sk->sk_user_data)
457 return -EBUSY;
458
459 ctx = kzalloc_obj(*ctx);
460 if (!ctx)
461 return -ENOMEM;
462
463 err = strp_init(&ctx->strp, sk, &cb);
464 if (err)
465 goto free;
466
467 __sk_dst_reset(sk);
468
469 strp_check_rcv(&ctx->strp);
470 skb_queue_head_init(&ctx->ike_queue);
471 skb_queue_head_init(&ctx->out_queue);
472
473 if (sk->sk_family == AF_INET) {
474 sk->sk_prot = &espintcp_prot;
475 sk->sk_socket->ops = &espintcp_ops;
476 } else {
477 mutex_lock(&tcpv6_prot_mutex);
478 if (!espintcp6_prot.recvmsg)
479 build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops);
480 mutex_unlock(&tcpv6_prot_mutex);
481
482 sk->sk_prot = &espintcp6_prot;
483 sk->sk_socket->ops = &espintcp6_ops;
484 }
485 ctx->saved_data_ready = sk->sk_data_ready;
486 ctx->saved_write_space = sk->sk_write_space;
487 ctx->saved_destruct = sk->sk_destruct;
488 sk->sk_data_ready = espintcp_data_ready;
489 sk->sk_write_space = espintcp_write_space;
490 sk->sk_destruct = espintcp_destruct;
491 rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
492 INIT_WORK(&ctx->work, espintcp_tx_work);
493
494 /* avoid using task_frag */
495 sk->sk_allocation = GFP_ATOMIC;
496 sk->sk_use_task_frag = false;
497
498 return 0;
499
500 free:
501 kfree(ctx);
502 return err;
503 }
504
espintcp_release(struct sock * sk)505 static void espintcp_release(struct sock *sk)
506 {
507 struct espintcp_ctx *ctx = espintcp_getctx(sk);
508 struct sk_buff_head queue;
509 struct sk_buff *skb;
510
511 __skb_queue_head_init(&queue);
512 skb_queue_splice_init(&ctx->out_queue, &queue);
513
514 while ((skb = __skb_dequeue(&queue)))
515 espintcp_push_skb(sk, skb);
516
517 tcp_release_cb(sk);
518 }
519
espintcp_close(struct sock * sk,long timeout)520 static void espintcp_close(struct sock *sk, long timeout)
521 {
522 struct espintcp_ctx *ctx = espintcp_getctx(sk);
523 struct espintcp_msg *emsg = &ctx->partial;
524
525 strp_stop(&ctx->strp);
526
527 sk->sk_prot = &tcp_prot;
528
529 synchronize_rcu();
530
531 disable_work_sync(&ctx->work);
532 strp_done(&ctx->strp);
533
534 skb_queue_purge(&ctx->out_queue);
535 skb_queue_purge(&ctx->ike_queue);
536
537 if (emsg->len) {
538 if (emsg->skb)
539 kfree_skb(emsg->skb);
540 else
541 sk_msg_free(sk, &emsg->skmsg);
542 }
543
544 tcp_close(sk, timeout);
545 }
546
espintcp_poll(struct file * file,struct socket * sock,poll_table * wait)547 static __poll_t espintcp_poll(struct file *file, struct socket *sock,
548 poll_table *wait)
549 {
550 struct sock *sk = sock->sk;
551 struct espintcp_ctx *ctx = espintcp_getctx(sk);
552
553 return datagram_poll_queue(file, sock, wait, &ctx->ike_queue);
554 }
555
build_protos(struct proto * espintcp_prot,struct proto_ops * espintcp_ops,const struct proto * orig_prot,const struct proto_ops * orig_ops)556 static void build_protos(struct proto *espintcp_prot,
557 struct proto_ops *espintcp_ops,
558 const struct proto *orig_prot,
559 const struct proto_ops *orig_ops)
560 {
561 memcpy(espintcp_prot, orig_prot, sizeof(struct proto));
562 memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops));
563 espintcp_prot->sendmsg = espintcp_sendmsg;
564 espintcp_prot->recvmsg = espintcp_recvmsg;
565 espintcp_prot->close = espintcp_close;
566 espintcp_prot->release_cb = espintcp_release;
567 espintcp_ops->poll = espintcp_poll;
568 }
569
570 static struct tcp_ulp_ops espintcp_ulp __read_mostly = {
571 .name = "espintcp",
572 .owner = THIS_MODULE,
573 .init = espintcp_init_sk,
574 };
575
espintcp_init(void)576 void __init espintcp_init(void)
577 {
578 build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops);
579
580 tcp_register_ulp(&espintcp_ulp);
581 }
582