1 // SPDX-License-Identifier: GPL-2.0 2 #include <net/tcp.h> 3 #include <net/strparser.h> 4 #include <net/xfrm.h> 5 #include <net/esp.h> 6 #include <net/espintcp.h> 7 #include <linux/skmsg.h> 8 #include <net/inet_common.h> 9 #include <trace/events/sock.h> 10 #include <net/hotdata.h> 11 12 static void handle_nonesp(struct espintcp_ctx *ctx, struct sk_buff *skb, 13 struct sock *sk) 14 { 15 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf || 16 !sk_rmem_schedule(sk, skb, skb->truesize)) { 17 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 18 kfree_skb(skb); 19 return; 20 } 21 22 skb_set_owner_r(skb, sk); 23 24 memset(skb->cb, 0, sizeof(skb->cb)); 25 skb_queue_tail(&ctx->ike_queue, skb); 26 ctx->saved_data_ready(sk); 27 } 28 29 static void handle_esp(struct sk_buff *skb, struct sock *sk) 30 { 31 struct tcp_skb_cb *tcp_cb = (struct tcp_skb_cb *)skb->cb; 32 33 skb_reset_transport_header(skb); 34 35 /* restore IP CB, we need at least IP6CB->nhoff */ 36 memmove(skb->cb, &tcp_cb->header, sizeof(tcp_cb->header)); 37 38 rcu_read_lock(); 39 skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); 40 if (!skb->dev) { 41 XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); 42 kfree_skb(skb); 43 goto out; 44 } 45 local_bh_disable(); 46 #if IS_ENABLED(CONFIG_IPV6) 47 if (sk->sk_family == AF_INET6) 48 xfrm6_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 49 else 50 #endif 51 xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); 52 local_bh_enable(); 53 out: 54 rcu_read_unlock(); 55 } 56 57 static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb) 58 { 59 struct espintcp_ctx *ctx = container_of(strp, struct espintcp_ctx, 60 strp); 61 struct strp_msg *rxm = strp_msg(skb); 62 int len = rxm->full_len - 2; 63 u32 nonesp_marker; 64 int err; 65 66 /* keepalive packet? */ 67 if (unlikely(len == 1)) { 68 u8 data; 69 70 err = skb_copy_bits(skb, rxm->offset + 2, &data, 1); 71 if (err < 0) { 72 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 73 kfree_skb(skb); 74 return; 75 } 76 77 if (data == 0xff) { 78 kfree_skb(skb); 79 return; 80 } 81 } 82 83 /* drop other short messages */ 84 if (unlikely(len <= sizeof(nonesp_marker))) { 85 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 86 kfree_skb(skb); 87 return; 88 } 89 90 err = skb_copy_bits(skb, rxm->offset + 2, &nonesp_marker, 91 sizeof(nonesp_marker)); 92 if (err < 0) { 93 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINHDRERROR); 94 kfree_skb(skb); 95 return; 96 } 97 98 /* remove header, leave non-ESP marker/SPI */ 99 if (!pskb_pull(skb, rxm->offset + 2)) { 100 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 101 kfree_skb(skb); 102 return; 103 } 104 105 if (pskb_trim(skb, rxm->full_len - 2) != 0) { 106 XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); 107 kfree_skb(skb); 108 return; 109 } 110 111 if (nonesp_marker == 0) 112 handle_nonesp(ctx, skb, strp->sk); 113 else 114 handle_esp(skb, strp->sk); 115 } 116 117 static int espintcp_parse(struct strparser *strp, struct sk_buff *skb) 118 { 119 struct strp_msg *rxm = strp_msg(skb); 120 __be16 blen; 121 u16 len; 122 int err; 123 124 if (skb->len < rxm->offset + 2) 125 return 0; 126 127 err = skb_copy_bits(skb, rxm->offset, &blen, sizeof(blen)); 128 if (err < 0) 129 return err; 130 131 len = be16_to_cpu(blen); 132 if (len < 2) 133 return -EINVAL; 134 135 return len; 136 } 137 138 static int espintcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 139 int flags) 140 { 141 struct espintcp_ctx *ctx = espintcp_getctx(sk); 142 struct sk_buff *skb; 143 int err = 0; 144 int copied; 145 int off = 0; 146 147 skb = __skb_recv_datagram(sk, &ctx->ike_queue, flags, &off, &err); 148 if (!skb) { 149 if (err == -EAGAIN && sk->sk_shutdown & RCV_SHUTDOWN) 150 return 0; 151 return err; 152 } 153 154 copied = len; 155 if (copied > skb->len) 156 copied = skb->len; 157 else if (copied < skb->len) 158 msg->msg_flags |= MSG_TRUNC; 159 160 err = skb_copy_datagram_msg(skb, 0, msg, copied); 161 if (unlikely(err)) { 162 kfree_skb(skb); 163 return err; 164 } 165 166 if (flags & MSG_TRUNC) 167 copied = skb->len; 168 kfree_skb(skb); 169 return copied; 170 } 171 172 int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) 173 { 174 struct espintcp_ctx *ctx = espintcp_getctx(sk); 175 176 if (skb_queue_len(&ctx->out_queue) >= 177 READ_ONCE(net_hotdata.max_backlog)) { 178 kfree_skb(skb); 179 return -ENOBUFS; 180 } 181 182 __skb_queue_tail(&ctx->out_queue, skb); 183 184 return 0; 185 } 186 EXPORT_SYMBOL_GPL(espintcp_queue_out); 187 188 /* espintcp length field is 2B and length includes the length field's size */ 189 #define MAX_ESPINTCP_MSG (((1 << 16) - 1) - 2) 190 191 static int espintcp_sendskb_locked(struct sock *sk, struct espintcp_msg *emsg, 192 int flags) 193 { 194 do { 195 int ret; 196 197 ret = skb_send_sock_locked(sk, emsg->skb, 198 emsg->offset, emsg->len); 199 if (ret < 0) 200 return ret; 201 202 emsg->len -= ret; 203 emsg->offset += ret; 204 } while (emsg->len > 0); 205 206 kfree_skb(emsg->skb); 207 memset(emsg, 0, sizeof(*emsg)); 208 209 return 0; 210 } 211 212 static int espintcp_sendskmsg_locked(struct sock *sk, 213 struct espintcp_msg *emsg, int flags) 214 { 215 struct msghdr msghdr = { 216 .msg_flags = flags | MSG_SPLICE_PAGES | MSG_MORE, 217 }; 218 struct sk_msg *skmsg = &emsg->skmsg; 219 bool more = flags & MSG_MORE; 220 struct scatterlist *sg; 221 int ret; 222 223 do { 224 struct bio_vec bvec; 225 226 sg = &skmsg->sg.data[skmsg->sg.start]; 227 if (sg_is_last(sg) && !more) 228 msghdr.msg_flags &= ~MSG_MORE; 229 230 bvec_set_page(&bvec, sg_page(sg), sg->length, sg->offset); 231 iov_iter_bvec(&msghdr.msg_iter, ITER_SOURCE, &bvec, 1, sg->length); 232 ret = tcp_sendmsg_locked(sk, &msghdr, sg->length); 233 if (ret < 0) 234 return ret; 235 236 sk_msg_free_partial(sk, skmsg, ret); 237 } while (skmsg->sg.size); 238 239 memset(emsg, 0, sizeof(*emsg)); 240 241 return 0; 242 } 243 244 static int espintcp_push_msgs(struct sock *sk, int flags) 245 { 246 struct espintcp_ctx *ctx = espintcp_getctx(sk); 247 struct espintcp_msg *emsg = &ctx->partial; 248 int err; 249 250 if (!emsg->len) 251 return 0; 252 253 if (ctx->tx_running) 254 return -EAGAIN; 255 ctx->tx_running = 1; 256 257 if (emsg->skb) 258 err = espintcp_sendskb_locked(sk, emsg, flags); 259 else 260 err = espintcp_sendskmsg_locked(sk, emsg, flags); 261 if (err == -EAGAIN) { 262 ctx->tx_running = 0; 263 return flags & MSG_DONTWAIT ? -EAGAIN : 0; 264 } 265 if (!err) 266 memset(emsg, 0, sizeof(*emsg)); 267 268 ctx->tx_running = 0; 269 270 return err; 271 } 272 273 int espintcp_push_skb(struct sock *sk, struct sk_buff *skb) 274 { 275 struct espintcp_ctx *ctx = espintcp_getctx(sk); 276 struct espintcp_msg *emsg = &ctx->partial; 277 unsigned int len; 278 int offset; 279 280 if (sk->sk_state != TCP_ESTABLISHED) { 281 kfree_skb(skb); 282 return -ECONNRESET; 283 } 284 285 offset = skb_transport_offset(skb); 286 len = skb->len - offset; 287 288 espintcp_push_msgs(sk, 0); 289 290 if (emsg->len) { 291 kfree_skb(skb); 292 return -ENOBUFS; 293 } 294 295 skb_set_owner_w(skb, sk); 296 297 emsg->offset = offset; 298 emsg->len = len; 299 emsg->skb = skb; 300 301 espintcp_push_msgs(sk, 0); 302 303 return 0; 304 } 305 EXPORT_SYMBOL_GPL(espintcp_push_skb); 306 307 static int espintcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) 308 { 309 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 310 struct espintcp_ctx *ctx = espintcp_getctx(sk); 311 struct espintcp_msg *emsg = &ctx->partial; 312 struct iov_iter pfx_iter; 313 struct kvec pfx_iov = {}; 314 size_t msglen = size + 2; 315 char buf[2] = {0}; 316 int err, end; 317 318 if (msg->msg_flags & ~MSG_DONTWAIT) 319 return -EOPNOTSUPP; 320 321 if (size > MAX_ESPINTCP_MSG) 322 return -EMSGSIZE; 323 324 if (msg->msg_controllen) 325 return -EOPNOTSUPP; 326 327 lock_sock(sk); 328 329 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 330 if (err < 0) { 331 if (err != -EAGAIN || !(msg->msg_flags & MSG_DONTWAIT)) 332 err = -ENOBUFS; 333 goto unlock; 334 } 335 if (emsg->len) { 336 err = -ENOBUFS; 337 goto unlock; 338 } 339 340 sk_msg_init(&emsg->skmsg); 341 while (1) { 342 /* only -ENOMEM is possible since we don't coalesce */ 343 err = sk_msg_alloc(sk, &emsg->skmsg, msglen, 0); 344 if (!err) 345 break; 346 347 err = sk_stream_wait_memory(sk, &timeo); 348 if (err) 349 goto fail; 350 } 351 352 *((__be16 *)buf) = cpu_to_be16(msglen); 353 pfx_iov.iov_base = buf; 354 pfx_iov.iov_len = sizeof(buf); 355 iov_iter_kvec(&pfx_iter, ITER_SOURCE, &pfx_iov, 1, pfx_iov.iov_len); 356 357 err = sk_msg_memcopy_from_iter(sk, &pfx_iter, &emsg->skmsg, 358 pfx_iov.iov_len); 359 if (err < 0) 360 goto fail; 361 362 err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, &emsg->skmsg, size); 363 if (err < 0) 364 goto fail; 365 366 end = emsg->skmsg.sg.end; 367 emsg->len = size; 368 sk_msg_iter_var_prev(end); 369 sg_mark_end(sk_msg_elem(&emsg->skmsg, end)); 370 371 tcp_rate_check_app_limited(sk); 372 373 err = espintcp_push_msgs(sk, msg->msg_flags & MSG_DONTWAIT); 374 /* this message could be partially sent, keep it */ 375 376 release_sock(sk); 377 378 return size; 379 380 fail: 381 sk_msg_free(sk, &emsg->skmsg); 382 memset(emsg, 0, sizeof(*emsg)); 383 unlock: 384 release_sock(sk); 385 return err; 386 } 387 388 static struct proto espintcp_prot __ro_after_init; 389 static struct proto_ops espintcp_ops __ro_after_init; 390 static struct proto espintcp6_prot; 391 static struct proto_ops espintcp6_ops; 392 static DEFINE_MUTEX(tcpv6_prot_mutex); 393 394 static void espintcp_data_ready(struct sock *sk) 395 { 396 struct espintcp_ctx *ctx = espintcp_getctx(sk); 397 398 trace_sk_data_ready(sk); 399 400 strp_data_ready(&ctx->strp); 401 } 402 403 static void espintcp_tx_work(struct work_struct *work) 404 { 405 struct espintcp_ctx *ctx = container_of(work, 406 struct espintcp_ctx, work); 407 struct sock *sk = ctx->strp.sk; 408 409 lock_sock(sk); 410 if (!ctx->tx_running) 411 espintcp_push_msgs(sk, 0); 412 release_sock(sk); 413 } 414 415 static void espintcp_write_space(struct sock *sk) 416 { 417 struct espintcp_ctx *ctx = espintcp_getctx(sk); 418 419 schedule_work(&ctx->work); 420 ctx->saved_write_space(sk); 421 } 422 423 static void espintcp_destruct(struct sock *sk) 424 { 425 struct espintcp_ctx *ctx = espintcp_getctx(sk); 426 427 ctx->saved_destruct(sk); 428 kfree(ctx); 429 } 430 431 bool tcp_is_ulp_esp(struct sock *sk) 432 { 433 return sk->sk_prot == &espintcp_prot || sk->sk_prot == &espintcp6_prot; 434 } 435 EXPORT_SYMBOL_GPL(tcp_is_ulp_esp); 436 437 static void build_protos(struct proto *espintcp_prot, 438 struct proto_ops *espintcp_ops, 439 const struct proto *orig_prot, 440 const struct proto_ops *orig_ops); 441 static int espintcp_init_sk(struct sock *sk) 442 { 443 struct inet_connection_sock *icsk = inet_csk(sk); 444 struct strp_callbacks cb = { 445 .rcv_msg = espintcp_rcv, 446 .parse_msg = espintcp_parse, 447 }; 448 struct espintcp_ctx *ctx; 449 int err; 450 451 /* sockmap is not compatible with espintcp */ 452 if (sk->sk_user_data) 453 return -EBUSY; 454 455 ctx = kzalloc_obj(*ctx); 456 if (!ctx) 457 return -ENOMEM; 458 459 err = strp_init(&ctx->strp, sk, &cb); 460 if (err) 461 goto free; 462 463 __sk_dst_reset(sk); 464 465 strp_check_rcv(&ctx->strp); 466 skb_queue_head_init(&ctx->ike_queue); 467 skb_queue_head_init(&ctx->out_queue); 468 469 if (sk->sk_family == AF_INET) { 470 sk->sk_prot = &espintcp_prot; 471 sk->sk_socket->ops = &espintcp_ops; 472 } else { 473 mutex_lock(&tcpv6_prot_mutex); 474 if (!espintcp6_prot.recvmsg) 475 build_protos(&espintcp6_prot, &espintcp6_ops, sk->sk_prot, sk->sk_socket->ops); 476 mutex_unlock(&tcpv6_prot_mutex); 477 478 sk->sk_prot = &espintcp6_prot; 479 sk->sk_socket->ops = &espintcp6_ops; 480 } 481 ctx->saved_data_ready = sk->sk_data_ready; 482 ctx->saved_write_space = sk->sk_write_space; 483 ctx->saved_destruct = sk->sk_destruct; 484 sk->sk_data_ready = espintcp_data_ready; 485 sk->sk_write_space = espintcp_write_space; 486 sk->sk_destruct = espintcp_destruct; 487 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 488 INIT_WORK(&ctx->work, espintcp_tx_work); 489 490 /* avoid using task_frag */ 491 sk->sk_allocation = GFP_ATOMIC; 492 sk->sk_use_task_frag = false; 493 494 return 0; 495 496 free: 497 kfree(ctx); 498 return err; 499 } 500 501 static void espintcp_release(struct sock *sk) 502 { 503 struct espintcp_ctx *ctx = espintcp_getctx(sk); 504 struct sk_buff_head queue; 505 struct sk_buff *skb; 506 507 __skb_queue_head_init(&queue); 508 skb_queue_splice_init(&ctx->out_queue, &queue); 509 510 while ((skb = __skb_dequeue(&queue))) 511 espintcp_push_skb(sk, skb); 512 513 tcp_release_cb(sk); 514 } 515 516 static void espintcp_close(struct sock *sk, long timeout) 517 { 518 struct espintcp_ctx *ctx = espintcp_getctx(sk); 519 struct espintcp_msg *emsg = &ctx->partial; 520 521 strp_stop(&ctx->strp); 522 523 sk->sk_prot = &tcp_prot; 524 525 synchronize_rcu(); 526 527 disable_work_sync(&ctx->work); 528 strp_done(&ctx->strp); 529 530 skb_queue_purge(&ctx->out_queue); 531 skb_queue_purge(&ctx->ike_queue); 532 533 if (emsg->len) { 534 if (emsg->skb) 535 kfree_skb(emsg->skb); 536 else 537 sk_msg_free(sk, &emsg->skmsg); 538 } 539 540 tcp_close(sk, timeout); 541 } 542 543 static __poll_t espintcp_poll(struct file *file, struct socket *sock, 544 poll_table *wait) 545 { 546 struct sock *sk = sock->sk; 547 struct espintcp_ctx *ctx = espintcp_getctx(sk); 548 549 return datagram_poll_queue(file, sock, wait, &ctx->ike_queue); 550 } 551 552 static void build_protos(struct proto *espintcp_prot, 553 struct proto_ops *espintcp_ops, 554 const struct proto *orig_prot, 555 const struct proto_ops *orig_ops) 556 { 557 memcpy(espintcp_prot, orig_prot, sizeof(struct proto)); 558 memcpy(espintcp_ops, orig_ops, sizeof(struct proto_ops)); 559 espintcp_prot->sendmsg = espintcp_sendmsg; 560 espintcp_prot->recvmsg = espintcp_recvmsg; 561 espintcp_prot->close = espintcp_close; 562 espintcp_prot->release_cb = espintcp_release; 563 espintcp_ops->poll = espintcp_poll; 564 } 565 566 static struct tcp_ulp_ops espintcp_ulp __read_mostly = { 567 .name = "espintcp", 568 .owner = THIS_MODULE, 569 .init = espintcp_init_sk, 570 }; 571 572 void __init espintcp_init(void) 573 { 574 build_protos(&espintcp_prot, &espintcp_ops, &tcp_prot, &inet_stream_ops); 575 576 tcp_register_ulp(&espintcp_ulp); 577 } 578