1 // SPDX-License-Identifier: GPL-2.0 2 #include <net/tcp.h> 3 #include <net/strparser.h> 4 #include <net/xfrm.h> 5 #include <net/esp.h> 6 #include <net/espintcp.h> 7 #include <linux/skmsg.h> 8 #include <net/inet_common.h> 9 #include <trace/events/sock.h> 10 #include <net/hotdata.h> 11 12 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb, 13 struct sock *sk) 14 { 15 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf || 16 !sk_rmem_schedule(sk, skb, skb->truesize)) { 17 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 18 kfree_skb(skb); 19 return; 20 } 21 22 skb_set_owner_r(skb, sk); 23 24 memset(skb->cb, 0, sizeof(skb->cb)); 25 skb_queue_tail(&ctx->ike_queue, skb); 26 ctx->saved_data_ready(sk); 27 } 28 29 static void handle_esp(struct sk_buff *skb, struct sock *sk) 30 { 31 struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb; 32 33 if (!skb_reset_transport_header_careful(skb)) { 34 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 35 kfree_skb(skb); 36 return; 37 } 38 39 /* restore IP CB, we need at least IP6CB->nhoff */ 40 memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header)); 41 42 rcu_read_lock(); 43 skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); 44 if (!skb->dev) { 45 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 46 kfree_skb(skb); 47 goto out; 48 } 49 local_bh_disable(); 50 #if IS_ENABLED(CONFIG_IPV6) 51 if (sk->sk_family == AF_INET6) 52 xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 53 else 54 #endif 55 xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 56 local_bh_enable(); 57 out: 58 rcu_read_unlock(); 59 } 60 61 static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb) 62 { 63 struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx, 64 strp); 65 struct strp_msg *rxm = strp_msg(skb); 66 int len = rxm->full_len - 2; 67 u32 nonesp_marker; 68 int err; 69 70 /* keepalive packet? */ 71 if (unlikely(len == 1)) { 72 u8 data; 73 74 err = skb_copy_bits(skb, rxm->offset + 2, &data, 1); 75 if (err < 0) { 76 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 77 kfree_skb(skb); 78 return; 79 } 80 81 if (data == 0xff) { 82 kfree_skb(skb); 83 return; 84 } 85 } 86 87 /* drop other short messages */ 88 if (unlikely(len <= sizeof(nonesp_marker))) { 89 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 90 kfree_skb(skb); 91 return; 92 } 93 94 err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker, 95 sizeof(nonesp_marker)); 96 if (err < 0) { 97 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 98 kfree_skb(skb); 99 return; 100 } 101 102 /* remove header, leave non-ESP marker/SPI */ 103 if (!pskb_pull(skb, rxm->offset + 2)) { 104 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 105 kfree_skb(skb); 106 return; 107 } 108 109 if (pskb_trim(skb, rxm->full_len - 2) != 0) { 110 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 111 kfree_skb(skb); 112 return; 113 } 114 115 if (nonesp_marker == 0) 116 handle_nonesp(ctx, skb, strp->sk); 117 else 118 handle_esp(skb, strp->sk); 119 } 120 121 static int espintcp_parse(struct strparser *strp, struct sk_buff *skb) 122 { 123 struct strp_msg *rxm = strp_msg(skb); 124 __be16 blen; 125 u16 len; 126 int err; 127 128 if (skb->len < rxm->offset + 2) 129 return 0; 130 131 err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen)); 132 if (err < 0) 133 return err; 134 135 len = be16_to_cpu(blen); 136 if (len < 2) 137 return -EINVAL; 138 139 return len; 140 } 141 142 static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 143 int flags) 144 { 145 struct espintcp_ctx *ctx = espintcp_getctx(sk); 146 struct sk_buff *skb; 147 int err = 0; 148 int copied; 149 int off = 0; 150 151 skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err); 152 if (!skb) { 153 if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN) 154 return 0; 155 return err; 156 } 157 158 copied = len; 159 if (copied > skb->len) 160 copied = skb->len; 161 else if (copied < skb->len) 162 msg->msg_flags |= MSG_TRUNC; 163 164 err = skb_copy_datagram_msg(skb, 0, msg, copied); 165 if (unlikely(err)) { 166 kfree_skb(skb); 167 return err; 168 } 169 170 if (flags & MSG_TRUNC) 171 copied = skb->len; 172 kfree_skb(skb); 173 return copied; 174 } 175 176 int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) 177 { 178 struct espintcp_ctx *ctx = espintcp_getctx(sk); 179 180 if (skb_queue_len(&ctx->out_queue) >= 181 READ_ONCE(net_hotdata.max_backlog)) { 182 kfree_skb(skb); 183 return -ENOBUFS; 184 } 185 186 __skb_queue_tail(&ctx->out_queue, skb); 187 188 return 0; 189 } 190 EXPORT_SYMBOL_GPL(espintcp_queue_out); 191 192 /* espintcp length field is 2B and length includes the length field's size */ 193 #define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2) 194 195 static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg, 196 int flags) 197 { 198 do { 199 int ret; 200 201 ret = skb_send_sock_locked(sk, emsg->skb, 202 emsg->offset, emsg->len); 203 if (ret < 0) 204 return ret; 205 206 emsg->len -= ret; 207 emsg->offset += ret; 208 } while (emsg->len > 0); 209 210 kfree_skb(emsg->skb); 211 memset(emsg, 0, sizeof(*emsg)); 212 213 return 0; 214 } 215 216 static int espintcp_sendskmsg_locked(struct sock *sk, 217 struct espintcp_msg *emsg, int flags) 218 { 219 struct msghdr msghdr = { 220 .msg_flags = flags | MSG_SPLICE_PAGES | MSG_MORE, 221 }; 222 struct sk_msg *skmsg = &emsg->skmsg; 223 bool more = flags & MSG_MORE; 224 struct scatterlist *sg; 225 int ret; 226 227 do { 228 struct bio_vec bvec; 229 230 sg = &skmsg->sg.data[skmsg->sg.start]; 231 if (sg_is_last(sg) && !more) 232 msghdr.msg_flags &= ~MSG_MORE; 233 234 bvec_set_page(&bvec, sg_page(sg), sg->length, sg->offset); 235 iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, sg->length); 236 ret = tcp_sendmsg_locked(sk, &msghdr, sg->length); 237 if (ret < 0) 238 return ret; 239 240 sk_msg_free_partial(sk, skmsg, ret); 241 } while (skmsg->sg.size); 242 243 memset(emsg, 0, sizeof(*emsg)); 244 245 return 0; 246 } 247 248 static int espintcp_push_msgs(struct sock *sk, int flags) 249 { 250 struct espintcp_ctx *ctx = espintcp_getctx(sk); 251 struct espintcp_msg *emsg = &ctx->partial; 252 int err; 253 254 if (!emsg->len) 255 return 0; 256 257 if (ctx->tx_running) 258 return -EAGAIN; 259 ctx->tx_running = 1; 260 261 if (emsg->skb) 262 err = espintcp_sendskb_locked(sk, emsg, flags); 263 else 264 err = espintcp_sendskmsg_locked(sk, emsg, flags); 265 if (err == -EAGAIN) { 266 ctx->tx_running = 0; 267 return flags & MSG_DONTWAIT ? -EAGAIN : 0; 268 } 269 if (!err) 270 memset(emsg, 0, sizeof(*emsg)); 271 272 ctx->tx_running = 0; 273 274 return err; 275 } 276 277 int espintcp_push_skb(struct sock *sk, struct sk_buff *skb) 278 { 279 struct espintcp_ctx *ctx = espintcp_getctx(sk); 280 struct espintcp_msg *emsg = &ctx->partial; 281 unsigned int len; 282 int offset; 283 284 if (sk->sk_state != TCP_ESTABLISHED) { 285 kfree_skb(skb); 286 return -ECONNRESET; 287 } 288 289 offset = skb_transport_offset(skb); 290 len = skb->len - offset; 291 292 espintcp_push_msgs(sk, 0); 293 294 if (emsg->len) { 295 kfree_skb(skb); 296 return -ENOBUFS; 297 } 298 299 skb_set_owner_w(skb, sk); 300 301 emsg->offset = offset; 302 emsg->len = len; 303 emsg->skb = skb; 304 305 espintcp_push_msgs(sk, 0); 306 307 return 0; 308 } 309 EXPORT_SYMBOL_GPL(espintcp_push_skb); 310 311 static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 312 { 313 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 314 struct espintcp_ctx *ctx = espintcp_getctx(sk); 315 struct espintcp_msg *emsg = &ctx->partial; 316 struct iov_iter pfx_iter; 317 struct kvec pfx_iov = {}; 318 size_t msglen = size + 2; 319 char buf[2] = {0}; 320 int err, end; 321 322 if (msg->msg_flags & ~MSG_DONTWAIT) 323 return -EOPNOTSUPP; 324 325 if (size > MAX_ESPINTCP_MSG) 326 return -EMSGSIZE; 327 328 if (msg->msg_controllen) 329 return -EOPNOTSUPP; 330 331 lock_sock(sk); 332 333 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 334 if (err < 0) { 335 if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT)) 336 err = -ENOBUFS; 337 goto unlock; 338 } 339 if (emsg->len) { 340 err = -ENOBUFS; 341 goto unlock; 342 } 343 344 sk_msg_init(&emsg->skmsg); 345 while (1) { 346 /* only -ENOMEM is possible since we don't coalesce */ 347 err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0); 348 if (!err) 349 break; 350 351 err = sk_stream_wait_memory(sk, &timeo); 352 if (err) 353 goto fail; 354 } 355 356 *((__be16 *)buf) = cpu_to_be16(msglen); 357 pfx_iov.iov_base = buf; 358 pfx_iov.iov_len = sizeof(buf); 359 iov_iter_kvec(&pfx_iter, ITER_SOURCE, &pfx_iov, 1, pfx_iov.iov_len); 360 361 err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg, 362 pfx_iov.iov_len); 363 if (err < 0) 364 goto fail; 365 366 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size); 367 if (err < 0) 368 goto fail; 369 370 end = emsg->skmsg.sg.end; 371 emsg->len = size; 372 sk_msg_iter_var_prev(end); 373 sg_mark_end(sk_msg_elem(&emsg->skmsg, end)); 374 375 tcp_rate_check_app_limited(sk); 376 377 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 378 /* this message could be partially sent, keep it */ 379 380 release_sock(sk); 381 382 return size; 383 384 fail: 385 sk_msg_free(sk, &emsg->skmsg); 386 memset(emsg, 0, sizeof(*emsg)); 387 unlock: 388 release_sock(sk); 389 return err; 390 } 391 392 static struct proto espintcp_prot __ro_after_init; 393 static struct proto_ops espintcp_ops __ro_after_init; 394 static struct proto espintcp6_prot; 395 static struct proto_ops espintcp6_ops; 396 static DEFINE_MUTEX(tcpv6_prot_mutex); 397 398 static void espintcp_data_ready(struct sock *sk) 399 { 400 struct espintcp_ctx *ctx = espintcp_getctx(sk); 401 402 trace_sk_data_ready(sk); 403 404 strp_data_ready(&ctx->strp); 405 } 406 407 static void espintcp_tx_work(struct work_struct *work) 408 { 409 struct espintcp_ctx *ctx = container_of(work, 410 struct espintcp_ctx, work); 411 struct sock *sk = ctx->strp.sk; 412 413 lock_sock(sk); 414 if (!ctx->tx_running) 415 espintcp_push_msgs(sk, 0); 416 release_sock(sk); 417 } 418 419 static void espintcp_write_space(struct sock *sk) 420 { 421 struct espintcp_ctx *ctx = espintcp_getctx(sk); 422 423 schedule_work(&ctx->work); 424 ctx->saved_write_space(sk); 425 } 426 427 static void espintcp_destruct(struct sock *sk) 428 { 429 struct espintcp_ctx *ctx = espintcp_getctx(sk); 430 431 ctx->saved_destruct(sk); 432 kfree(ctx); 433 } 434 435 bool tcp_is_ulp_esp(struct sock *sk) 436 { 437 return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot; 438 } 439 EXPORT_SYMBOL_GPL(tcp_is_ulp_esp); 440 441 static void build_protos(struct proto *espintcp_prot, 442 struct proto_ops *espintcp_ops, 443 const struct proto *orig_prot, 444 const struct proto_ops *orig_ops); 445 static int espintcp_init_sk(struct sock *sk) 446 { 447 struct inet_connection_sock *icsk = inet_csk(sk); 448 struct strp_callbacks cb = { 449 .rcv_msg = espintcp_rcv, 450 .parse_msg = espintcp_parse, 451 }; 452 struct espintcp_ctx *ctx; 453 int err; 454 455 /* sockmap is not compatible with espintcp */ 456 if (sk->sk_user_data) 457 return -EBUSY; 458 459 ctx = kzalloc_obj(*ctx); 460 if (!ctx) 461 return -ENOMEM; 462 463 err = strp_init(&ctx->strp, sk, &cb); 464 if (err) 465 goto free; 466 467 __sk_dst_reset(sk); 468 469 strp_check_rcv(&ctx->strp); 470 skb_queue_head_init(&ctx->ike_queue); 471 skb_queue_head_init(&ctx->out_queue); 472 473 if (sk->sk_family == AF_INET) { 474 sk->sk_prot = &espintcp_prot; 475 sk->sk_socket->ops = &espintcp_ops; 476 } else { 477 mutex_lock(&tcpv6_prot_mutex); 478 if (!espintcp6_prot.recvmsg) 479 build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops); 480 mutex_unlock(&tcpv6_prot_mutex); 481 482 sk->sk_prot = &espintcp6_prot; 483 sk->sk_socket->ops = &espintcp6_ops; 484 } 485 ctx->saved_data_ready = sk->sk_data_ready; 486 ctx->saved_write_space = sk->sk_write_space; 487 ctx->saved_destruct = sk->sk_destruct; 488 sk->sk_data_ready = espintcp_data_ready; 489 sk->sk_write_space = espintcp_write_space; 490 sk->sk_destruct = espintcp_destruct; 491 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 492 INIT_WORK(&ctx->work, espintcp_tx_work); 493 494 /* avoid using task_frag */ 495 sk->sk_allocation = GFP_ATOMIC; 496 sk->sk_use_task_frag = false; 497 498 return 0; 499 500 free: 501 kfree(ctx); 502 return err; 503 } 504 505 static void espintcp_release(struct sock *sk) 506 { 507 struct espintcp_ctx *ctx = espintcp_getctx(sk); 508 struct sk_buff_head queue; 509 struct sk_buff *skb; 510 511 __skb_queue_head_init(&queue); 512 skb_queue_splice_init(&ctx->out_queue, &queue); 513 514 while ((skb = __skb_dequeue(&queue))) 515 espintcp_push_skb(sk, skb); 516 517 tcp_release_cb(sk); 518 } 519 520 static void espintcp_close(struct sock *sk, long timeout) 521 { 522 struct espintcp_ctx *ctx = espintcp_getctx(sk); 523 struct espintcp_msg *emsg = &ctx->partial; 524 525 strp_stop(&ctx->strp); 526 527 sk->sk_prot = &tcp_prot; 528 529 synchronize_rcu(); 530 531 disable_work_sync(&ctx->work); 532 strp_done(&ctx->strp); 533 534 skb_queue_purge(&ctx->out_queue); 535 skb_queue_purge(&ctx->ike_queue); 536 537 if (emsg->len) { 538 if (emsg->skb) 539 kfree_skb(emsg->skb); 540 else 541 sk_msg_free(sk, &emsg->skmsg); 542 } 543 544 tcp_close(sk, timeout); 545 } 546 547 static __poll_t espintcp_poll(struct file *file, struct socket *sock, 548 poll_table *wait) 549 { 550 struct sock *sk = sock->sk; 551 struct espintcp_ctx *ctx = espintcp_getctx(sk); 552 553 return datagram_poll_queue(file, sock, wait, &ctx->ike_queue); 554 } 555 556 static void build_protos(struct proto *espintcp_prot, 557 struct proto_ops *espintcp_ops, 558 const struct proto *orig_prot, 559 const struct proto_ops *orig_ops) 560 { 561 memcpy(espintcp_prot, orig_prot, sizeof(struct proto)); 562 memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops)); 563 espintcp_prot->sendmsg = espintcp_sendmsg; 564 espintcp_prot->recvmsg = espintcp_recvmsg; 565 espintcp_prot->close = espintcp_close; 566 espintcp_prot->release_cb = espintcp_release; 567 espintcp_ops->poll = espintcp_poll; 568 } 569 570 static struct tcp_ulp_ops espintcp_ulp __read_mostly = { 571 .name = "espintcp", 572 .owner = THIS_MODULE, 573 .init = espintcp_init_sk, 574 }; 575 576 void __init espintcp_init(void) 577 { 578 build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops); 579 580 tcp_register_ulp(&espintcp_ulp); 581 } 582