1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * PACKET - implements raw packet sockets. 7 * 8 * Authors: Ross Biro 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 10 * Alan Cox, <gw4pts@gw4pts.ampr.org> 11 * 12 * Fixes: 13 * Alan Cox : verify_area() now used correctly 14 * Alan Cox : new skbuff lists, look ma no backlogs! 15 * Alan Cox : tidied skbuff lists. 16 * Alan Cox : Now uses generic datagram routines I 17 * added. Also fixed the peek/read crash 18 * from all old Linux datagram code. 19 * Alan Cox : Uses the improved datagram code. 20 * Alan Cox : Added NULL's for socket options. 21 * Alan Cox : Re-commented the code. 22 * Alan Cox : Use new kernel side addressing 23 * Rob Janssen : Correct MTU usage. 24 * Dave Platt : Counter leaks caused by incorrect 25 * interrupt locking and some slightly 26 * dubious gcc output. Can you read 27 * compiler: it said _VOLATILE_ 28 * Richard Kooijman : Timestamp fixes. 29 * Alan Cox : New buffers. Use sk->mac.raw. 30 * Alan Cox : sendmsg/recvmsg support. 31 * Alan Cox : Protocol setting support 32 * Alexey Kuznetsov : Untied from IPv4 stack. 33 * Cyrus Durgin : Fixed kerneld for kmod. 34 * Michal Ostrowski : Module initialization cleanup. 35 * Ulises Alonso : Frame number limit removal and 36 * packet_set_ring memory leak. 37 * Eric Biederman : Allow for > 8 byte hardware addresses. 38 * The convention is that longer addresses 39 * will simply extend the hardware address 40 * byte arrays at the end of sockaddr_ll 41 * and packet_mreq. 42 * Johann Baudy : Added TX RING. 43 * 44 * This program is free software; you can redistribute it and/or 45 * modify it under the terms of the GNU General Public License 46 * as published by the Free Software Foundation; either version 47 * 2 of the License, or (at your option) any later version. 48 * 49 */ 50 51 #include <linux/types.h> 52 #include <linux/mm.h> 53 #include <linux/capability.h> 54 #include <linux/fcntl.h> 55 #include <linux/socket.h> 56 #include <linux/in.h> 57 #include <linux/inet.h> 58 #include <linux/netdevice.h> 59 #include <linux/if_packet.h> 60 #include <linux/wireless.h> 61 #include <linux/kernel.h> 62 #include <linux/kmod.h> 63 #include <linux/slab.h> 64 #include <net/net_namespace.h> 65 #include <net/ip.h> 66 #include <net/protocol.h> 67 #include <linux/skbuff.h> 68 #include <net/sock.h> 69 #include <linux/errno.h> 70 #include <linux/timer.h> 71 #include <asm/system.h> 72 #include <asm/uaccess.h> 73 #include <asm/ioctls.h> 74 #include <asm/page.h> 75 #include <asm/cacheflush.h> 76 #include <asm/io.h> 77 #include <linux/proc_fs.h> 78 #include <linux/seq_file.h> 79 #include <linux/poll.h> 80 #include <linux/module.h> 81 #include <linux/init.h> 82 #include <linux/mutex.h> 83 #include <linux/if_vlan.h> 84 #include <linux/virtio_net.h> 85 #include <linux/errqueue.h> 86 #include <linux/net_tstamp.h> 87 88 #ifdef CONFIG_INET 89 #include <net/inet_common.h> 90 #endif 91 92 /* 93 Assumptions: 94 - if device has no dev->hard_header routine, it adds and removes ll header 95 inside itself. In this case ll header is invisible outside of device, 96 but higher levels still should reserve dev->hard_header_len. 97 Some devices are enough clever to reallocate skb, when header 98 will not fit to reserved space (tunnel), another ones are silly 99 (PPP). 100 - packet socket receives packets with pulled ll header, 101 so that SOCK_RAW should push it back. 102 103 On receive: 104 ----------- 105 106 Incoming, dev->hard_header!=NULL 107 mac_header -> ll header 108 data -> data 109 110 Outgoing, dev->hard_header!=NULL 111 mac_header -> ll header 112 data -> ll header 113 114 Incoming, dev->hard_header==NULL 115 mac_header -> UNKNOWN position. It is very likely, that it points to ll 116 header. PPP makes it, that is wrong, because introduce 117 assymetry between rx and tx paths. 118 data -> data 119 120 Outgoing, dev->hard_header==NULL 121 mac_header -> data. ll header is still not built! 122 data -> data 123 124 Resume 125 If dev->hard_header==NULL we are unlikely to restore sensible ll header. 126 127 128 On transmit: 129 ------------ 130 131 dev->hard_header != NULL 132 mac_header -> ll header 133 data -> ll header 134 135 dev->hard_header == NULL (ll header is added by device, we cannot control it) 136 mac_header -> data 137 data -> data 138 139 We should set nh.raw on output to correct posistion, 140 packet classifier depends on it. 141 */ 142 143 /* Private packet socket structures. */ 144 145 struct packet_mclist { 146 struct packet_mclist *next; 147 int ifindex; 148 int count; 149 unsigned short type; 150 unsigned short alen; 151 unsigned char addr[MAX_ADDR_LEN]; 152 }; 153 /* identical to struct packet_mreq except it has 154 * a longer address field. 155 */ 156 struct packet_mreq_max { 157 int mr_ifindex; 158 unsigned short mr_type; 159 unsigned short mr_alen; 160 unsigned char mr_address[MAX_ADDR_LEN]; 161 }; 162 163 static int packet_set_ring(struct sock *sk, struct tpacket_req *req, 164 int closing, int tx_ring); 165 166 struct packet_ring_buffer { 167 char **pg_vec; 168 unsigned int head; 169 unsigned int frames_per_block; 170 unsigned int frame_size; 171 unsigned int frame_max; 172 173 unsigned int pg_vec_order; 174 unsigned int pg_vec_pages; 175 unsigned int pg_vec_len; 176 177 atomic_t pending; 178 }; 179 180 struct packet_sock; 181 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg); 182 183 static void packet_flush_mclist(struct sock *sk); 184 185 struct packet_sock { 186 /* struct sock has to be the first member of packet_sock */ 187 struct sock sk; 188 struct tpacket_stats stats; 189 struct packet_ring_buffer rx_ring; 190 struct packet_ring_buffer tx_ring; 191 int copy_thresh; 192 spinlock_t bind_lock; 193 struct mutex pg_vec_lock; 194 unsigned int running:1, /* prot_hook is attached*/ 195 auxdata:1, 196 origdev:1, 197 has_vnet_hdr:1; 198 int ifindex; /* bound device */ 199 __be16 num; 200 struct packet_mclist *mclist; 201 atomic_t mapped; 202 enum tpacket_versions tp_version; 203 unsigned int tp_hdrlen; 204 unsigned int tp_reserve; 205 unsigned int tp_loss:1; 206 unsigned int tp_tstamp; 207 struct packet_type prot_hook ____cacheline_aligned_in_smp; 208 }; 209 210 struct packet_skb_cb { 211 unsigned int origlen; 212 union { 213 struct sockaddr_pkt pkt; 214 struct sockaddr_ll ll; 215 } sa; 216 }; 217 218 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb)) 219 220 static void __packet_set_status(struct packet_sock *po, void *frame, int status) 221 { 222 union { 223 struct tpacket_hdr *h1; 224 struct tpacket2_hdr *h2; 225 void *raw; 226 } h; 227 228 h.raw = frame; 229 switch (po->tp_version) { 230 case TPACKET_V1: 231 h.h1->tp_status = status; 232 flush_dcache_page(virt_to_page(&h.h1->tp_status)); 233 break; 234 case TPACKET_V2: 235 h.h2->tp_status = status; 236 flush_dcache_page(virt_to_page(&h.h2->tp_status)); 237 break; 238 default: 239 pr_err("TPACKET version not supported\n"); 240 BUG(); 241 } 242 243 smp_wmb(); 244 } 245 246 static int __packet_get_status(struct packet_sock *po, void *frame) 247 { 248 union { 249 struct tpacket_hdr *h1; 250 struct tpacket2_hdr *h2; 251 void *raw; 252 } h; 253 254 smp_rmb(); 255 256 h.raw = frame; 257 switch (po->tp_version) { 258 case TPACKET_V1: 259 flush_dcache_page(virt_to_page(&h.h1->tp_status)); 260 return h.h1->tp_status; 261 case TPACKET_V2: 262 flush_dcache_page(virt_to_page(&h.h2->tp_status)); 263 return h.h2->tp_status; 264 default: 265 pr_err("TPACKET version not supported\n"); 266 BUG(); 267 return 0; 268 } 269 } 270 271 static void *packet_lookup_frame(struct packet_sock *po, 272 struct packet_ring_buffer *rb, 273 unsigned int position, 274 int status) 275 { 276 unsigned int pg_vec_pos, frame_offset; 277 union { 278 struct tpacket_hdr *h1; 279 struct tpacket2_hdr *h2; 280 void *raw; 281 } h; 282 283 pg_vec_pos = position / rb->frames_per_block; 284 frame_offset = position % rb->frames_per_block; 285 286 h.raw = rb->pg_vec[pg_vec_pos] + (frame_offset * rb->frame_size); 287 288 if (status != __packet_get_status(po, h.raw)) 289 return NULL; 290 291 return h.raw; 292 } 293 294 static inline void *packet_current_frame(struct packet_sock *po, 295 struct packet_ring_buffer *rb, 296 int status) 297 { 298 return packet_lookup_frame(po, rb, rb->head, status); 299 } 300 301 static inline void *packet_previous_frame(struct packet_sock *po, 302 struct packet_ring_buffer *rb, 303 int status) 304 { 305 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max; 306 return packet_lookup_frame(po, rb, previous, status); 307 } 308 309 static inline void packet_increment_head(struct packet_ring_buffer *buff) 310 { 311 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0; 312 } 313 314 static inline struct packet_sock *pkt_sk(struct sock *sk) 315 { 316 return (struct packet_sock *)sk; 317 } 318 319 static void packet_sock_destruct(struct sock *sk) 320 { 321 skb_queue_purge(&sk->sk_error_queue); 322 323 WARN_ON(atomic_read(&sk->sk_rmem_alloc)); 324 WARN_ON(atomic_read(&sk->sk_wmem_alloc)); 325 326 if (!sock_flag(sk, SOCK_DEAD)) { 327 pr_err("Attempt to release alive packet socket: %p\n", sk); 328 return; 329 } 330 331 sk_refcnt_debug_dec(sk); 332 } 333 334 335 static const struct proto_ops packet_ops; 336 337 static const struct proto_ops packet_ops_spkt; 338 339 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev, 340 struct packet_type *pt, struct net_device *orig_dev) 341 { 342 struct sock *sk; 343 struct sockaddr_pkt *spkt; 344 345 /* 346 * When we registered the protocol we saved the socket in the data 347 * field for just this event. 348 */ 349 350 sk = pt->af_packet_priv; 351 352 /* 353 * Yank back the headers [hope the device set this 354 * right or kerboom...] 355 * 356 * Incoming packets have ll header pulled, 357 * push it back. 358 * 359 * For outgoing ones skb->data == skb_mac_header(skb) 360 * so that this procedure is noop. 361 */ 362 363 if (skb->pkt_type == PACKET_LOOPBACK) 364 goto out; 365 366 if (!net_eq(dev_net(dev), sock_net(sk))) 367 goto out; 368 369 skb = skb_share_check(skb, GFP_ATOMIC); 370 if (skb == NULL) 371 goto oom; 372 373 /* drop any routing info */ 374 skb_dst_drop(skb); 375 376 /* drop conntrack reference */ 377 nf_reset(skb); 378 379 spkt = &PACKET_SKB_CB(skb)->sa.pkt; 380 381 skb_push(skb, skb->data - skb_mac_header(skb)); 382 383 /* 384 * The SOCK_PACKET socket receives _all_ frames. 385 */ 386 387 spkt->spkt_family = dev->type; 388 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device)); 389 spkt->spkt_protocol = skb->protocol; 390 391 /* 392 * Charge the memory to the socket. This is done specifically 393 * to prevent sockets using all the memory up. 394 */ 395 396 if (sock_queue_rcv_skb(sk, skb) == 0) 397 return 0; 398 399 out: 400 kfree_skb(skb); 401 oom: 402 return 0; 403 } 404 405 406 /* 407 * Output a raw packet to a device layer. This bypasses all the other 408 * protocol layers and you must therefore supply it with a complete frame 409 */ 410 411 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock, 412 struct msghdr *msg, size_t len) 413 { 414 struct sock *sk = sock->sk; 415 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name; 416 struct sk_buff *skb = NULL; 417 struct net_device *dev; 418 __be16 proto = 0; 419 int err; 420 421 /* 422 * Get and verify the address. 423 */ 424 425 if (saddr) { 426 if (msg->msg_namelen < sizeof(struct sockaddr)) 427 return -EINVAL; 428 if (msg->msg_namelen == sizeof(struct sockaddr_pkt)) 429 proto = saddr->spkt_protocol; 430 } else 431 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */ 432 433 /* 434 * Find the device first to size check it 435 */ 436 437 saddr->spkt_device[13] = 0; 438 retry: 439 rcu_read_lock(); 440 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device); 441 err = -ENODEV; 442 if (dev == NULL) 443 goto out_unlock; 444 445 err = -ENETDOWN; 446 if (!(dev->flags & IFF_UP)) 447 goto out_unlock; 448 449 /* 450 * You may not queue a frame bigger than the mtu. This is the lowest level 451 * raw protocol and you must do your own fragmentation at this level. 452 */ 453 454 err = -EMSGSIZE; 455 if (len > dev->mtu + dev->hard_header_len) 456 goto out_unlock; 457 458 if (!skb) { 459 size_t reserved = LL_RESERVED_SPACE(dev); 460 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0; 461 462 rcu_read_unlock(); 463 skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL); 464 if (skb == NULL) 465 return -ENOBUFS; 466 /* FIXME: Save some space for broken drivers that write a hard 467 * header at transmission time by themselves. PPP is the notable 468 * one here. This should really be fixed at the driver level. 469 */ 470 skb_reserve(skb, reserved); 471 skb_reset_network_header(skb); 472 473 /* Try to align data part correctly */ 474 if (hhlen) { 475 skb->data -= hhlen; 476 skb->tail -= hhlen; 477 if (len < hhlen) 478 skb_reset_network_header(skb); 479 } 480 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len); 481 if (err) 482 goto out_free; 483 goto retry; 484 } 485 486 487 skb->protocol = proto; 488 skb->dev = dev; 489 skb->priority = sk->sk_priority; 490 skb->mark = sk->sk_mark; 491 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags); 492 if (err < 0) 493 goto out_unlock; 494 495 dev_queue_xmit(skb); 496 rcu_read_unlock(); 497 return len; 498 499 out_unlock: 500 rcu_read_unlock(); 501 out_free: 502 kfree_skb(skb); 503 return err; 504 } 505 506 static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk, 507 unsigned int res) 508 { 509 struct sk_filter *filter; 510 511 rcu_read_lock_bh(); 512 filter = rcu_dereference_bh(sk->sk_filter); 513 if (filter != NULL) 514 res = sk_run_filter(skb, filter->insns, filter->len); 515 rcu_read_unlock_bh(); 516 517 return res; 518 } 519 520 /* 521 This function makes lazy skb cloning in hope that most of packets 522 are discarded by BPF. 523 524 Note tricky part: we DO mangle shared skb! skb->data, skb->len 525 and skb->cb are mangled. It works because (and until) packets 526 falling here are owned by current CPU. Output packets are cloned 527 by dev_queue_xmit_nit(), input packets are processed by net_bh 528 sequencially, so that if we return skb to original state on exit, 529 we will not harm anyone. 530 */ 531 532 static int packet_rcv(struct sk_buff *skb, struct net_device *dev, 533 struct packet_type *pt, struct net_device *orig_dev) 534 { 535 struct sock *sk; 536 struct sockaddr_ll *sll; 537 struct packet_sock *po; 538 u8 *skb_head = skb->data; 539 int skb_len = skb->len; 540 unsigned int snaplen, res; 541 542 if (skb->pkt_type == PACKET_LOOPBACK) 543 goto drop; 544 545 sk = pt->af_packet_priv; 546 po = pkt_sk(sk); 547 548 if (!net_eq(dev_net(dev), sock_net(sk))) 549 goto drop; 550 551 skb->dev = dev; 552 553 if (dev->header_ops) { 554 /* The device has an explicit notion of ll header, 555 exported to higher levels. 556 557 Otherwise, the device hides datails of it frame 558 structure, so that corresponding packet head 559 never delivered to user. 560 */ 561 if (sk->sk_type != SOCK_DGRAM) 562 skb_push(skb, skb->data - skb_mac_header(skb)); 563 else if (skb->pkt_type == PACKET_OUTGOING) { 564 /* Special case: outgoing packets have ll header at head */ 565 skb_pull(skb, skb_network_offset(skb)); 566 } 567 } 568 569 snaplen = skb->len; 570 571 res = run_filter(skb, sk, snaplen); 572 if (!res) 573 goto drop_n_restore; 574 if (snaplen > res) 575 snaplen = res; 576 577 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 578 (unsigned)sk->sk_rcvbuf) 579 goto drop_n_acct; 580 581 if (skb_shared(skb)) { 582 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); 583 if (nskb == NULL) 584 goto drop_n_acct; 585 586 if (skb_head != skb->data) { 587 skb->data = skb_head; 588 skb->len = skb_len; 589 } 590 kfree_skb(skb); 591 skb = nskb; 592 } 593 594 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 > 595 sizeof(skb->cb)); 596 597 sll = &PACKET_SKB_CB(skb)->sa.ll; 598 sll->sll_family = AF_PACKET; 599 sll->sll_hatype = dev->type; 600 sll->sll_protocol = skb->protocol; 601 sll->sll_pkttype = skb->pkt_type; 602 if (unlikely(po->origdev)) 603 sll->sll_ifindex = orig_dev->ifindex; 604 else 605 sll->sll_ifindex = dev->ifindex; 606 607 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); 608 609 PACKET_SKB_CB(skb)->origlen = skb->len; 610 611 if (pskb_trim(skb, snaplen)) 612 goto drop_n_acct; 613 614 skb_set_owner_r(skb, sk); 615 skb->dev = NULL; 616 skb_dst_drop(skb); 617 618 /* drop conntrack reference */ 619 nf_reset(skb); 620 621 spin_lock(&sk->sk_receive_queue.lock); 622 po->stats.tp_packets++; 623 skb->dropcount = atomic_read(&sk->sk_drops); 624 __skb_queue_tail(&sk->sk_receive_queue, skb); 625 spin_unlock(&sk->sk_receive_queue.lock); 626 sk->sk_data_ready(sk, skb->len); 627 return 0; 628 629 drop_n_acct: 630 po->stats.tp_drops = atomic_inc_return(&sk->sk_drops); 631 632 drop_n_restore: 633 if (skb_head != skb->data && skb_shared(skb)) { 634 skb->data = skb_head; 635 skb->len = skb_len; 636 } 637 drop: 638 consume_skb(skb); 639 return 0; 640 } 641 642 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev, 643 struct packet_type *pt, struct net_device *orig_dev) 644 { 645 struct sock *sk; 646 struct packet_sock *po; 647 struct sockaddr_ll *sll; 648 union { 649 struct tpacket_hdr *h1; 650 struct tpacket2_hdr *h2; 651 void *raw; 652 } h; 653 u8 *skb_head = skb->data; 654 int skb_len = skb->len; 655 unsigned int snaplen, res; 656 unsigned long status = TP_STATUS_LOSING|TP_STATUS_USER; 657 unsigned short macoff, netoff, hdrlen; 658 struct sk_buff *copy_skb = NULL; 659 struct timeval tv; 660 struct timespec ts; 661 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb); 662 663 if (skb->pkt_type == PACKET_LOOPBACK) 664 goto drop; 665 666 sk = pt->af_packet_priv; 667 po = pkt_sk(sk); 668 669 if (!net_eq(dev_net(dev), sock_net(sk))) 670 goto drop; 671 672 if (dev->header_ops) { 673 if (sk->sk_type != SOCK_DGRAM) 674 skb_push(skb, skb->data - skb_mac_header(skb)); 675 else if (skb->pkt_type == PACKET_OUTGOING) { 676 /* Special case: outgoing packets have ll header at head */ 677 skb_pull(skb, skb_network_offset(skb)); 678 } 679 } 680 681 if (skb->ip_summed == CHECKSUM_PARTIAL) 682 status |= TP_STATUS_CSUMNOTREADY; 683 684 snaplen = skb->len; 685 686 res = run_filter(skb, sk, snaplen); 687 if (!res) 688 goto drop_n_restore; 689 if (snaplen > res) 690 snaplen = res; 691 692 if (sk->sk_type == SOCK_DGRAM) { 693 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + 694 po->tp_reserve; 695 } else { 696 unsigned maclen = skb_network_offset(skb); 697 netoff = TPACKET_ALIGN(po->tp_hdrlen + 698 (maclen < 16 ? 16 : maclen)) + 699 po->tp_reserve; 700 macoff = netoff - maclen; 701 } 702 703 if (macoff + snaplen > po->rx_ring.frame_size) { 704 if (po->copy_thresh && 705 atomic_read(&sk->sk_rmem_alloc) + skb->truesize < 706 (unsigned)sk->sk_rcvbuf) { 707 if (skb_shared(skb)) { 708 copy_skb = skb_clone(skb, GFP_ATOMIC); 709 } else { 710 copy_skb = skb_get(skb); 711 skb_head = skb->data; 712 } 713 if (copy_skb) 714 skb_set_owner_r(copy_skb, sk); 715 } 716 snaplen = po->rx_ring.frame_size - macoff; 717 if ((int)snaplen < 0) 718 snaplen = 0; 719 } 720 721 spin_lock(&sk->sk_receive_queue.lock); 722 h.raw = packet_current_frame(po, &po->rx_ring, TP_STATUS_KERNEL); 723 if (!h.raw) 724 goto ring_is_full; 725 packet_increment_head(&po->rx_ring); 726 po->stats.tp_packets++; 727 if (copy_skb) { 728 status |= TP_STATUS_COPY; 729 __skb_queue_tail(&sk->sk_receive_queue, copy_skb); 730 } 731 if (!po->stats.tp_drops) 732 status &= ~TP_STATUS_LOSING; 733 spin_unlock(&sk->sk_receive_queue.lock); 734 735 skb_copy_bits(skb, 0, h.raw + macoff, snaplen); 736 737 switch (po->tp_version) { 738 case TPACKET_V1: 739 h.h1->tp_len = skb->len; 740 h.h1->tp_snaplen = snaplen; 741 h.h1->tp_mac = macoff; 742 h.h1->tp_net = netoff; 743 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE) 744 && shhwtstamps->syststamp.tv64) 745 tv = ktime_to_timeval(shhwtstamps->syststamp); 746 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE) 747 && shhwtstamps->hwtstamp.tv64) 748 tv = ktime_to_timeval(shhwtstamps->hwtstamp); 749 else if (skb->tstamp.tv64) 750 tv = ktime_to_timeval(skb->tstamp); 751 else 752 do_gettimeofday(&tv); 753 h.h1->tp_sec = tv.tv_sec; 754 h.h1->tp_usec = tv.tv_usec; 755 hdrlen = sizeof(*h.h1); 756 break; 757 case TPACKET_V2: 758 h.h2->tp_len = skb->len; 759 h.h2->tp_snaplen = snaplen; 760 h.h2->tp_mac = macoff; 761 h.h2->tp_net = netoff; 762 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE) 763 && shhwtstamps->syststamp.tv64) 764 ts = ktime_to_timespec(shhwtstamps->syststamp); 765 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE) 766 && shhwtstamps->hwtstamp.tv64) 767 ts = ktime_to_timespec(shhwtstamps->hwtstamp); 768 else if (skb->tstamp.tv64) 769 ts = ktime_to_timespec(skb->tstamp); 770 else 771 getnstimeofday(&ts); 772 h.h2->tp_sec = ts.tv_sec; 773 h.h2->tp_nsec = ts.tv_nsec; 774 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb); 775 hdrlen = sizeof(*h.h2); 776 break; 777 default: 778 BUG(); 779 } 780 781 sll = h.raw + TPACKET_ALIGN(hdrlen); 782 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); 783 sll->sll_family = AF_PACKET; 784 sll->sll_hatype = dev->type; 785 sll->sll_protocol = skb->protocol; 786 sll->sll_pkttype = skb->pkt_type; 787 if (unlikely(po->origdev)) 788 sll->sll_ifindex = orig_dev->ifindex; 789 else 790 sll->sll_ifindex = dev->ifindex; 791 792 __packet_set_status(po, h.raw, status); 793 smp_mb(); 794 { 795 struct page *p_start, *p_end; 796 u8 *h_end = h.raw + macoff + snaplen - 1; 797 798 p_start = virt_to_page(h.raw); 799 p_end = virt_to_page(h_end); 800 while (p_start <= p_end) { 801 flush_dcache_page(p_start); 802 p_start++; 803 } 804 } 805 806 sk->sk_data_ready(sk, 0); 807 808 drop_n_restore: 809 if (skb_head != skb->data && skb_shared(skb)) { 810 skb->data = skb_head; 811 skb->len = skb_len; 812 } 813 drop: 814 kfree_skb(skb); 815 return 0; 816 817 ring_is_full: 818 po->stats.tp_drops++; 819 spin_unlock(&sk->sk_receive_queue.lock); 820 821 sk->sk_data_ready(sk, 0); 822 kfree_skb(copy_skb); 823 goto drop_n_restore; 824 } 825 826 static void tpacket_destruct_skb(struct sk_buff *skb) 827 { 828 struct packet_sock *po = pkt_sk(skb->sk); 829 void *ph; 830 831 BUG_ON(skb == NULL); 832 833 if (likely(po->tx_ring.pg_vec)) { 834 ph = skb_shinfo(skb)->destructor_arg; 835 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING); 836 BUG_ON(atomic_read(&po->tx_ring.pending) == 0); 837 atomic_dec(&po->tx_ring.pending); 838 __packet_set_status(po, ph, TP_STATUS_AVAILABLE); 839 } 840 841 sock_wfree(skb); 842 } 843 844 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, 845 void *frame, struct net_device *dev, int size_max, 846 __be16 proto, unsigned char *addr) 847 { 848 union { 849 struct tpacket_hdr *h1; 850 struct tpacket2_hdr *h2; 851 void *raw; 852 } ph; 853 int to_write, offset, len, tp_len, nr_frags, len_max; 854 struct socket *sock = po->sk.sk_socket; 855 struct page *page; 856 void *data; 857 int err; 858 859 ph.raw = frame; 860 861 skb->protocol = proto; 862 skb->dev = dev; 863 skb->priority = po->sk.sk_priority; 864 skb->mark = po->sk.sk_mark; 865 skb_shinfo(skb)->destructor_arg = ph.raw; 866 867 switch (po->tp_version) { 868 case TPACKET_V2: 869 tp_len = ph.h2->tp_len; 870 break; 871 default: 872 tp_len = ph.h1->tp_len; 873 break; 874 } 875 if (unlikely(tp_len > size_max)) { 876 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max); 877 return -EMSGSIZE; 878 } 879 880 skb_reserve(skb, LL_RESERVED_SPACE(dev)); 881 skb_reset_network_header(skb); 882 883 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll); 884 to_write = tp_len; 885 886 if (sock->type == SOCK_DGRAM) { 887 err = dev_hard_header(skb, dev, ntohs(proto), addr, 888 NULL, tp_len); 889 if (unlikely(err < 0)) 890 return -EINVAL; 891 } else if (dev->hard_header_len) { 892 /* net device doesn't like empty head */ 893 if (unlikely(tp_len <= dev->hard_header_len)) { 894 pr_err("packet size is too short (%d < %d)\n", 895 tp_len, dev->hard_header_len); 896 return -EINVAL; 897 } 898 899 skb_push(skb, dev->hard_header_len); 900 err = skb_store_bits(skb, 0, data, 901 dev->hard_header_len); 902 if (unlikely(err)) 903 return err; 904 905 data += dev->hard_header_len; 906 to_write -= dev->hard_header_len; 907 } 908 909 err = -EFAULT; 910 page = virt_to_page(data); 911 offset = offset_in_page(data); 912 len_max = PAGE_SIZE - offset; 913 len = ((to_write > len_max) ? len_max : to_write); 914 915 skb->data_len = to_write; 916 skb->len += to_write; 917 skb->truesize += to_write; 918 atomic_add(to_write, &po->sk.sk_wmem_alloc); 919 920 while (likely(to_write)) { 921 nr_frags = skb_shinfo(skb)->nr_frags; 922 923 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) { 924 pr_err("Packet exceed the number of skb frags(%lu)\n", 925 MAX_SKB_FRAGS); 926 return -EFAULT; 927 } 928 929 flush_dcache_page(page); 930 get_page(page); 931 skb_fill_page_desc(skb, 932 nr_frags, 933 page++, offset, len); 934 to_write -= len; 935 offset = 0; 936 len_max = PAGE_SIZE; 937 len = ((to_write > len_max) ? len_max : to_write); 938 } 939 940 return tp_len; 941 } 942 943 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) 944 { 945 struct socket *sock; 946 struct sk_buff *skb; 947 struct net_device *dev; 948 __be16 proto; 949 int ifindex, err, reserve = 0; 950 void *ph; 951 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; 952 int tp_len, size_max; 953 unsigned char *addr; 954 int len_sum = 0; 955 int status = 0; 956 957 sock = po->sk.sk_socket; 958 959 mutex_lock(&po->pg_vec_lock); 960 961 err = -EBUSY; 962 if (saddr == NULL) { 963 ifindex = po->ifindex; 964 proto = po->num; 965 addr = NULL; 966 } else { 967 err = -EINVAL; 968 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) 969 goto out; 970 if (msg->msg_namelen < (saddr->sll_halen 971 + offsetof(struct sockaddr_ll, 972 sll_addr))) 973 goto out; 974 ifindex = saddr->sll_ifindex; 975 proto = saddr->sll_protocol; 976 addr = saddr->sll_addr; 977 } 978 979 dev = dev_get_by_index(sock_net(&po->sk), ifindex); 980 err = -ENXIO; 981 if (unlikely(dev == NULL)) 982 goto out; 983 984 reserve = dev->hard_header_len; 985 986 err = -ENETDOWN; 987 if (unlikely(!(dev->flags & IFF_UP))) 988 goto out_put; 989 990 size_max = po->tx_ring.frame_size 991 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); 992 993 if (size_max > dev->mtu + reserve) 994 size_max = dev->mtu + reserve; 995 996 do { 997 ph = packet_current_frame(po, &po->tx_ring, 998 TP_STATUS_SEND_REQUEST); 999 1000 if (unlikely(ph == NULL)) { 1001 schedule(); 1002 continue; 1003 } 1004 1005 status = TP_STATUS_SEND_REQUEST; 1006 skb = sock_alloc_send_skb(&po->sk, 1007 LL_ALLOCATED_SPACE(dev) 1008 + sizeof(struct sockaddr_ll), 1009 0, &err); 1010 1011 if (unlikely(skb == NULL)) 1012 goto out_status; 1013 1014 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto, 1015 addr); 1016 1017 if (unlikely(tp_len < 0)) { 1018 if (po->tp_loss) { 1019 __packet_set_status(po, ph, 1020 TP_STATUS_AVAILABLE); 1021 packet_increment_head(&po->tx_ring); 1022 kfree_skb(skb); 1023 continue; 1024 } else { 1025 status = TP_STATUS_WRONG_FORMAT; 1026 err = tp_len; 1027 goto out_status; 1028 } 1029 } 1030 1031 skb->destructor = tpacket_destruct_skb; 1032 __packet_set_status(po, ph, TP_STATUS_SENDING); 1033 atomic_inc(&po->tx_ring.pending); 1034 1035 status = TP_STATUS_SEND_REQUEST; 1036 err = dev_queue_xmit(skb); 1037 if (unlikely(err > 0)) { 1038 err = net_xmit_errno(err); 1039 if (err && __packet_get_status(po, ph) == 1040 TP_STATUS_AVAILABLE) { 1041 /* skb was destructed already */ 1042 skb = NULL; 1043 goto out_status; 1044 } 1045 /* 1046 * skb was dropped but not destructed yet; 1047 * let's treat it like congestion or err < 0 1048 */ 1049 err = 0; 1050 } 1051 packet_increment_head(&po->tx_ring); 1052 len_sum += tp_len; 1053 } while (likely((ph != NULL) || 1054 ((!(msg->msg_flags & MSG_DONTWAIT)) && 1055 (atomic_read(&po->tx_ring.pending)))) 1056 ); 1057 1058 err = len_sum; 1059 goto out_put; 1060 1061 out_status: 1062 __packet_set_status(po, ph, status); 1063 kfree_skb(skb); 1064 out_put: 1065 dev_put(dev); 1066 out: 1067 mutex_unlock(&po->pg_vec_lock); 1068 return err; 1069 } 1070 1071 static inline struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad, 1072 size_t reserve, size_t len, 1073 size_t linear, int noblock, 1074 int *err) 1075 { 1076 struct sk_buff *skb; 1077 1078 /* Under a page? Don't bother with paged skb. */ 1079 if (prepad + len < PAGE_SIZE || !linear) 1080 linear = len; 1081 1082 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 1083 err); 1084 if (!skb) 1085 return NULL; 1086 1087 skb_reserve(skb, reserve); 1088 skb_put(skb, linear); 1089 skb->data_len = len - linear; 1090 skb->len += len - linear; 1091 1092 return skb; 1093 } 1094 1095 static int packet_snd(struct socket *sock, 1096 struct msghdr *msg, size_t len) 1097 { 1098 struct sock *sk = sock->sk; 1099 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; 1100 struct sk_buff *skb; 1101 struct net_device *dev; 1102 __be16 proto; 1103 unsigned char *addr; 1104 int ifindex, err, reserve = 0; 1105 struct virtio_net_hdr vnet_hdr = { 0 }; 1106 int offset = 0; 1107 int vnet_hdr_len; 1108 struct packet_sock *po = pkt_sk(sk); 1109 unsigned short gso_type = 0; 1110 1111 /* 1112 * Get and verify the address. 1113 */ 1114 1115 if (saddr == NULL) { 1116 ifindex = po->ifindex; 1117 proto = po->num; 1118 addr = NULL; 1119 } else { 1120 err = -EINVAL; 1121 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) 1122 goto out; 1123 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr))) 1124 goto out; 1125 ifindex = saddr->sll_ifindex; 1126 proto = saddr->sll_protocol; 1127 addr = saddr->sll_addr; 1128 } 1129 1130 1131 dev = dev_get_by_index(sock_net(sk), ifindex); 1132 err = -ENXIO; 1133 if (dev == NULL) 1134 goto out_unlock; 1135 if (sock->type == SOCK_RAW) 1136 reserve = dev->hard_header_len; 1137 1138 err = -ENETDOWN; 1139 if (!(dev->flags & IFF_UP)) 1140 goto out_unlock; 1141 1142 if (po->has_vnet_hdr) { 1143 vnet_hdr_len = sizeof(vnet_hdr); 1144 1145 err = -EINVAL; 1146 if (len < vnet_hdr_len) 1147 goto out_unlock; 1148 1149 len -= vnet_hdr_len; 1150 1151 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov, 1152 vnet_hdr_len); 1153 if (err < 0) 1154 goto out_unlock; 1155 1156 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 1157 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 > 1158 vnet_hdr.hdr_len)) 1159 vnet_hdr.hdr_len = vnet_hdr.csum_start + 1160 vnet_hdr.csum_offset + 2; 1161 1162 err = -EINVAL; 1163 if (vnet_hdr.hdr_len > len) 1164 goto out_unlock; 1165 1166 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) { 1167 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 1168 case VIRTIO_NET_HDR_GSO_TCPV4: 1169 gso_type = SKB_GSO_TCPV4; 1170 break; 1171 case VIRTIO_NET_HDR_GSO_TCPV6: 1172 gso_type = SKB_GSO_TCPV6; 1173 break; 1174 case VIRTIO_NET_HDR_GSO_UDP: 1175 gso_type = SKB_GSO_UDP; 1176 break; 1177 default: 1178 goto out_unlock; 1179 } 1180 1181 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN) 1182 gso_type |= SKB_GSO_TCP_ECN; 1183 1184 if (vnet_hdr.gso_size == 0) 1185 goto out_unlock; 1186 1187 } 1188 } 1189 1190 err = -EMSGSIZE; 1191 if (!gso_type && (len > dev->mtu+reserve)) 1192 goto out_unlock; 1193 1194 err = -ENOBUFS; 1195 skb = packet_alloc_skb(sk, LL_ALLOCATED_SPACE(dev), 1196 LL_RESERVED_SPACE(dev), len, vnet_hdr.hdr_len, 1197 msg->msg_flags & MSG_DONTWAIT, &err); 1198 if (skb == NULL) 1199 goto out_unlock; 1200 1201 skb_set_network_header(skb, reserve); 1202 1203 err = -EINVAL; 1204 if (sock->type == SOCK_DGRAM && 1205 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0) 1206 goto out_free; 1207 1208 /* Returns -EFAULT on error */ 1209 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len); 1210 if (err) 1211 goto out_free; 1212 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags); 1213 if (err < 0) 1214 goto out_free; 1215 1216 skb->protocol = proto; 1217 skb->dev = dev; 1218 skb->priority = sk->sk_priority; 1219 skb->mark = sk->sk_mark; 1220 1221 if (po->has_vnet_hdr) { 1222 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 1223 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start, 1224 vnet_hdr.csum_offset)) { 1225 err = -EINVAL; 1226 goto out_free; 1227 } 1228 } 1229 1230 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size; 1231 skb_shinfo(skb)->gso_type = gso_type; 1232 1233 /* Header must be checked, and gso_segs computed. */ 1234 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 1235 skb_shinfo(skb)->gso_segs = 0; 1236 1237 len += vnet_hdr_len; 1238 } 1239 1240 /* 1241 * Now send it 1242 */ 1243 1244 err = dev_queue_xmit(skb); 1245 if (err > 0 && (err = net_xmit_errno(err)) != 0) 1246 goto out_unlock; 1247 1248 dev_put(dev); 1249 1250 return len; 1251 1252 out_free: 1253 kfree_skb(skb); 1254 out_unlock: 1255 if (dev) 1256 dev_put(dev); 1257 out: 1258 return err; 1259 } 1260 1261 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock, 1262 struct msghdr *msg, size_t len) 1263 { 1264 struct sock *sk = sock->sk; 1265 struct packet_sock *po = pkt_sk(sk); 1266 if (po->tx_ring.pg_vec) 1267 return tpacket_snd(po, msg); 1268 else 1269 return packet_snd(sock, msg, len); 1270 } 1271 1272 /* 1273 * Close a PACKET socket. This is fairly simple. We immediately go 1274 * to 'closed' state and remove our protocol entry in the device list. 1275 */ 1276 1277 static int packet_release(struct socket *sock) 1278 { 1279 struct sock *sk = sock->sk; 1280 struct packet_sock *po; 1281 struct net *net; 1282 struct tpacket_req req; 1283 1284 if (!sk) 1285 return 0; 1286 1287 net = sock_net(sk); 1288 po = pkt_sk(sk); 1289 1290 spin_lock_bh(&net->packet.sklist_lock); 1291 sk_del_node_init_rcu(sk); 1292 sock_prot_inuse_add(net, sk->sk_prot, -1); 1293 spin_unlock_bh(&net->packet.sklist_lock); 1294 1295 spin_lock(&po->bind_lock); 1296 if (po->running) { 1297 /* 1298 * Remove from protocol table 1299 */ 1300 po->running = 0; 1301 po->num = 0; 1302 __dev_remove_pack(&po->prot_hook); 1303 __sock_put(sk); 1304 } 1305 spin_unlock(&po->bind_lock); 1306 1307 packet_flush_mclist(sk); 1308 1309 memset(&req, 0, sizeof(req)); 1310 1311 if (po->rx_ring.pg_vec) 1312 packet_set_ring(sk, &req, 1, 0); 1313 1314 if (po->tx_ring.pg_vec) 1315 packet_set_ring(sk, &req, 1, 1); 1316 1317 synchronize_net(); 1318 /* 1319 * Now the socket is dead. No more input will appear. 1320 */ 1321 sock_orphan(sk); 1322 sock->sk = NULL; 1323 1324 /* Purge queues */ 1325 1326 skb_queue_purge(&sk->sk_receive_queue); 1327 sk_refcnt_debug_release(sk); 1328 1329 sock_put(sk); 1330 return 0; 1331 } 1332 1333 /* 1334 * Attach a packet hook. 1335 */ 1336 1337 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol) 1338 { 1339 struct packet_sock *po = pkt_sk(sk); 1340 /* 1341 * Detach an existing hook if present. 1342 */ 1343 1344 lock_sock(sk); 1345 1346 spin_lock(&po->bind_lock); 1347 if (po->running) { 1348 __sock_put(sk); 1349 po->running = 0; 1350 po->num = 0; 1351 spin_unlock(&po->bind_lock); 1352 dev_remove_pack(&po->prot_hook); 1353 spin_lock(&po->bind_lock); 1354 } 1355 1356 po->num = protocol; 1357 po->prot_hook.type = protocol; 1358 po->prot_hook.dev = dev; 1359 1360 po->ifindex = dev ? dev->ifindex : 0; 1361 1362 if (protocol == 0) 1363 goto out_unlock; 1364 1365 if (!dev || (dev->flags & IFF_UP)) { 1366 dev_add_pack(&po->prot_hook); 1367 sock_hold(sk); 1368 po->running = 1; 1369 } else { 1370 sk->sk_err = ENETDOWN; 1371 if (!sock_flag(sk, SOCK_DEAD)) 1372 sk->sk_error_report(sk); 1373 } 1374 1375 out_unlock: 1376 spin_unlock(&po->bind_lock); 1377 release_sock(sk); 1378 return 0; 1379 } 1380 1381 /* 1382 * Bind a packet socket to a device 1383 */ 1384 1385 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr, 1386 int addr_len) 1387 { 1388 struct sock *sk = sock->sk; 1389 char name[15]; 1390 struct net_device *dev; 1391 int err = -ENODEV; 1392 1393 /* 1394 * Check legality 1395 */ 1396 1397 if (addr_len != sizeof(struct sockaddr)) 1398 return -EINVAL; 1399 strlcpy(name, uaddr->sa_data, sizeof(name)); 1400 1401 dev = dev_get_by_name(sock_net(sk), name); 1402 if (dev) { 1403 err = packet_do_bind(sk, dev, pkt_sk(sk)->num); 1404 dev_put(dev); 1405 } 1406 return err; 1407 } 1408 1409 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 1410 { 1411 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr; 1412 struct sock *sk = sock->sk; 1413 struct net_device *dev = NULL; 1414 int err; 1415 1416 1417 /* 1418 * Check legality 1419 */ 1420 1421 if (addr_len < sizeof(struct sockaddr_ll)) 1422 return -EINVAL; 1423 if (sll->sll_family != AF_PACKET) 1424 return -EINVAL; 1425 1426 if (sll->sll_ifindex) { 1427 err = -ENODEV; 1428 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex); 1429 if (dev == NULL) 1430 goto out; 1431 } 1432 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num); 1433 if (dev) 1434 dev_put(dev); 1435 1436 out: 1437 return err; 1438 } 1439 1440 static struct proto packet_proto = { 1441 .name = "PACKET", 1442 .owner = THIS_MODULE, 1443 .obj_size = sizeof(struct packet_sock), 1444 }; 1445 1446 /* 1447 * Create a packet of type SOCK_PACKET. 1448 */ 1449 1450 static int packet_create(struct net *net, struct socket *sock, int protocol, 1451 int kern) 1452 { 1453 struct sock *sk; 1454 struct packet_sock *po; 1455 __be16 proto = (__force __be16)protocol; /* weird, but documented */ 1456 int err; 1457 1458 if (!capable(CAP_NET_RAW)) 1459 return -EPERM; 1460 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW && 1461 sock->type != SOCK_PACKET) 1462 return -ESOCKTNOSUPPORT; 1463 1464 sock->state = SS_UNCONNECTED; 1465 1466 err = -ENOBUFS; 1467 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto); 1468 if (sk == NULL) 1469 goto out; 1470 1471 sock->ops = &packet_ops; 1472 if (sock->type == SOCK_PACKET) 1473 sock->ops = &packet_ops_spkt; 1474 1475 sock_init_data(sock, sk); 1476 1477 po = pkt_sk(sk); 1478 sk->sk_family = PF_PACKET; 1479 po->num = proto; 1480 1481 sk->sk_destruct = packet_sock_destruct; 1482 sk_refcnt_debug_inc(sk); 1483 1484 /* 1485 * Attach a protocol block 1486 */ 1487 1488 spin_lock_init(&po->bind_lock); 1489 mutex_init(&po->pg_vec_lock); 1490 po->prot_hook.func = packet_rcv; 1491 1492 if (sock->type == SOCK_PACKET) 1493 po->prot_hook.func = packet_rcv_spkt; 1494 1495 po->prot_hook.af_packet_priv = sk; 1496 1497 if (proto) { 1498 po->prot_hook.type = proto; 1499 dev_add_pack(&po->prot_hook); 1500 sock_hold(sk); 1501 po->running = 1; 1502 } 1503 1504 spin_lock_bh(&net->packet.sklist_lock); 1505 sk_add_node_rcu(sk, &net->packet.sklist); 1506 sock_prot_inuse_add(net, &packet_proto, 1); 1507 spin_unlock_bh(&net->packet.sklist_lock); 1508 1509 return 0; 1510 out: 1511 return err; 1512 } 1513 1514 static int packet_recv_error(struct sock *sk, struct msghdr *msg, int len) 1515 { 1516 struct sock_exterr_skb *serr; 1517 struct sk_buff *skb, *skb2; 1518 int copied, err; 1519 1520 err = -EAGAIN; 1521 skb = skb_dequeue(&sk->sk_error_queue); 1522 if (skb == NULL) 1523 goto out; 1524 1525 copied = skb->len; 1526 if (copied > len) { 1527 msg->msg_flags |= MSG_TRUNC; 1528 copied = len; 1529 } 1530 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied); 1531 if (err) 1532 goto out_free_skb; 1533 1534 sock_recv_timestamp(msg, sk, skb); 1535 1536 serr = SKB_EXT_ERR(skb); 1537 put_cmsg(msg, SOL_PACKET, PACKET_TX_TIMESTAMP, 1538 sizeof(serr->ee), &serr->ee); 1539 1540 msg->msg_flags |= MSG_ERRQUEUE; 1541 err = copied; 1542 1543 /* Reset and regenerate socket error */ 1544 spin_lock_bh(&sk->sk_error_queue.lock); 1545 sk->sk_err = 0; 1546 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) { 1547 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno; 1548 spin_unlock_bh(&sk->sk_error_queue.lock); 1549 sk->sk_error_report(sk); 1550 } else 1551 spin_unlock_bh(&sk->sk_error_queue.lock); 1552 1553 out_free_skb: 1554 kfree_skb(skb); 1555 out: 1556 return err; 1557 } 1558 1559 /* 1560 * Pull a packet from our receive queue and hand it to the user. 1561 * If necessary we block. 1562 */ 1563 1564 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock, 1565 struct msghdr *msg, size_t len, int flags) 1566 { 1567 struct sock *sk = sock->sk; 1568 struct sk_buff *skb; 1569 int copied, err; 1570 struct sockaddr_ll *sll; 1571 int vnet_hdr_len = 0; 1572 1573 err = -EINVAL; 1574 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE)) 1575 goto out; 1576 1577 #if 0 1578 /* What error should we return now? EUNATTACH? */ 1579 if (pkt_sk(sk)->ifindex < 0) 1580 return -ENODEV; 1581 #endif 1582 1583 if (flags & MSG_ERRQUEUE) { 1584 err = packet_recv_error(sk, msg, len); 1585 goto out; 1586 } 1587 1588 /* 1589 * Call the generic datagram receiver. This handles all sorts 1590 * of horrible races and re-entrancy so we can forget about it 1591 * in the protocol layers. 1592 * 1593 * Now it will return ENETDOWN, if device have just gone down, 1594 * but then it will block. 1595 */ 1596 1597 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err); 1598 1599 /* 1600 * An error occurred so return it. Because skb_recv_datagram() 1601 * handles the blocking we don't see and worry about blocking 1602 * retries. 1603 */ 1604 1605 if (skb == NULL) 1606 goto out; 1607 1608 if (pkt_sk(sk)->has_vnet_hdr) { 1609 struct virtio_net_hdr vnet_hdr = { 0 }; 1610 1611 err = -EINVAL; 1612 vnet_hdr_len = sizeof(vnet_hdr); 1613 if (len < vnet_hdr_len) 1614 goto out_free; 1615 1616 len -= vnet_hdr_len; 1617 1618 if (skb_is_gso(skb)) { 1619 struct skb_shared_info *sinfo = skb_shinfo(skb); 1620 1621 /* This is a hint as to how much should be linear. */ 1622 vnet_hdr.hdr_len = skb_headlen(skb); 1623 vnet_hdr.gso_size = sinfo->gso_size; 1624 if (sinfo->gso_type & SKB_GSO_TCPV4) 1625 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1626 else if (sinfo->gso_type & SKB_GSO_TCPV6) 1627 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 1628 else if (sinfo->gso_type & SKB_GSO_UDP) 1629 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP; 1630 else if (sinfo->gso_type & SKB_GSO_FCOE) 1631 goto out_free; 1632 else 1633 BUG(); 1634 if (sinfo->gso_type & SKB_GSO_TCP_ECN) 1635 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN; 1636 } else 1637 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE; 1638 1639 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1640 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 1641 vnet_hdr.csum_start = skb->csum_start - 1642 skb_headroom(skb); 1643 vnet_hdr.csum_offset = skb->csum_offset; 1644 } /* else everything is zero */ 1645 1646 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr, 1647 vnet_hdr_len); 1648 if (err < 0) 1649 goto out_free; 1650 } 1651 1652 /* 1653 * If the address length field is there to be filled in, we fill 1654 * it in now. 1655 */ 1656 1657 sll = &PACKET_SKB_CB(skb)->sa.ll; 1658 if (sock->type == SOCK_PACKET) 1659 msg->msg_namelen = sizeof(struct sockaddr_pkt); 1660 else 1661 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr); 1662 1663 /* 1664 * You lose any data beyond the buffer you gave. If it worries a 1665 * user program they can ask the device for its MTU anyway. 1666 */ 1667 1668 copied = skb->len; 1669 if (copied > len) { 1670 copied = len; 1671 msg->msg_flags |= MSG_TRUNC; 1672 } 1673 1674 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied); 1675 if (err) 1676 goto out_free; 1677 1678 sock_recv_ts_and_drops(msg, sk, skb); 1679 1680 if (msg->msg_name) 1681 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, 1682 msg->msg_namelen); 1683 1684 if (pkt_sk(sk)->auxdata) { 1685 struct tpacket_auxdata aux; 1686 1687 aux.tp_status = TP_STATUS_USER; 1688 if (skb->ip_summed == CHECKSUM_PARTIAL) 1689 aux.tp_status |= TP_STATUS_CSUMNOTREADY; 1690 aux.tp_len = PACKET_SKB_CB(skb)->origlen; 1691 aux.tp_snaplen = skb->len; 1692 aux.tp_mac = 0; 1693 aux.tp_net = skb_network_offset(skb); 1694 aux.tp_vlan_tci = vlan_tx_tag_get(skb); 1695 1696 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux); 1697 } 1698 1699 /* 1700 * Free or return the buffer as appropriate. Again this 1701 * hides all the races and re-entrancy issues from us. 1702 */ 1703 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied); 1704 1705 out_free: 1706 skb_free_datagram(sk, skb); 1707 out: 1708 return err; 1709 } 1710 1711 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr, 1712 int *uaddr_len, int peer) 1713 { 1714 struct net_device *dev; 1715 struct sock *sk = sock->sk; 1716 1717 if (peer) 1718 return -EOPNOTSUPP; 1719 1720 uaddr->sa_family = AF_PACKET; 1721 rcu_read_lock(); 1722 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex); 1723 if (dev) 1724 strncpy(uaddr->sa_data, dev->name, 14); 1725 else 1726 memset(uaddr->sa_data, 0, 14); 1727 rcu_read_unlock(); 1728 *uaddr_len = sizeof(*uaddr); 1729 1730 return 0; 1731 } 1732 1733 static int packet_getname(struct socket *sock, struct sockaddr *uaddr, 1734 int *uaddr_len, int peer) 1735 { 1736 struct net_device *dev; 1737 struct sock *sk = sock->sk; 1738 struct packet_sock *po = pkt_sk(sk); 1739 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr); 1740 1741 if (peer) 1742 return -EOPNOTSUPP; 1743 1744 sll->sll_family = AF_PACKET; 1745 sll->sll_ifindex = po->ifindex; 1746 sll->sll_protocol = po->num; 1747 sll->sll_pkttype = 0; 1748 rcu_read_lock(); 1749 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex); 1750 if (dev) { 1751 sll->sll_hatype = dev->type; 1752 sll->sll_halen = dev->addr_len; 1753 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len); 1754 } else { 1755 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */ 1756 sll->sll_halen = 0; 1757 } 1758 rcu_read_unlock(); 1759 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen; 1760 1761 return 0; 1762 } 1763 1764 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i, 1765 int what) 1766 { 1767 switch (i->type) { 1768 case PACKET_MR_MULTICAST: 1769 if (i->alen != dev->addr_len) 1770 return -EINVAL; 1771 if (what > 0) 1772 return dev_mc_add(dev, i->addr); 1773 else 1774 return dev_mc_del(dev, i->addr); 1775 break; 1776 case PACKET_MR_PROMISC: 1777 return dev_set_promiscuity(dev, what); 1778 break; 1779 case PACKET_MR_ALLMULTI: 1780 return dev_set_allmulti(dev, what); 1781 break; 1782 case PACKET_MR_UNICAST: 1783 if (i->alen != dev->addr_len) 1784 return -EINVAL; 1785 if (what > 0) 1786 return dev_uc_add(dev, i->addr); 1787 else 1788 return dev_uc_del(dev, i->addr); 1789 break; 1790 default: 1791 break; 1792 } 1793 return 0; 1794 } 1795 1796 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what) 1797 { 1798 for ( ; i; i = i->next) { 1799 if (i->ifindex == dev->ifindex) 1800 packet_dev_mc(dev, i, what); 1801 } 1802 } 1803 1804 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq) 1805 { 1806 struct packet_sock *po = pkt_sk(sk); 1807 struct packet_mclist *ml, *i; 1808 struct net_device *dev; 1809 int err; 1810 1811 rtnl_lock(); 1812 1813 err = -ENODEV; 1814 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex); 1815 if (!dev) 1816 goto done; 1817 1818 err = -EINVAL; 1819 if (mreq->mr_alen > dev->addr_len) 1820 goto done; 1821 1822 err = -ENOBUFS; 1823 i = kmalloc(sizeof(*i), GFP_KERNEL); 1824 if (i == NULL) 1825 goto done; 1826 1827 err = 0; 1828 for (ml = po->mclist; ml; ml = ml->next) { 1829 if (ml->ifindex == mreq->mr_ifindex && 1830 ml->type == mreq->mr_type && 1831 ml->alen == mreq->mr_alen && 1832 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { 1833 ml->count++; 1834 /* Free the new element ... */ 1835 kfree(i); 1836 goto done; 1837 } 1838 } 1839 1840 i->type = mreq->mr_type; 1841 i->ifindex = mreq->mr_ifindex; 1842 i->alen = mreq->mr_alen; 1843 memcpy(i->addr, mreq->mr_address, i->alen); 1844 i->count = 1; 1845 i->next = po->mclist; 1846 po->mclist = i; 1847 err = packet_dev_mc(dev, i, 1); 1848 if (err) { 1849 po->mclist = i->next; 1850 kfree(i); 1851 } 1852 1853 done: 1854 rtnl_unlock(); 1855 return err; 1856 } 1857 1858 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq) 1859 { 1860 struct packet_mclist *ml, **mlp; 1861 1862 rtnl_lock(); 1863 1864 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) { 1865 if (ml->ifindex == mreq->mr_ifindex && 1866 ml->type == mreq->mr_type && 1867 ml->alen == mreq->mr_alen && 1868 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { 1869 if (--ml->count == 0) { 1870 struct net_device *dev; 1871 *mlp = ml->next; 1872 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); 1873 if (dev) 1874 packet_dev_mc(dev, ml, -1); 1875 kfree(ml); 1876 } 1877 rtnl_unlock(); 1878 return 0; 1879 } 1880 } 1881 rtnl_unlock(); 1882 return -EADDRNOTAVAIL; 1883 } 1884 1885 static void packet_flush_mclist(struct sock *sk) 1886 { 1887 struct packet_sock *po = pkt_sk(sk); 1888 struct packet_mclist *ml; 1889 1890 if (!po->mclist) 1891 return; 1892 1893 rtnl_lock(); 1894 while ((ml = po->mclist) != NULL) { 1895 struct net_device *dev; 1896 1897 po->mclist = ml->next; 1898 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); 1899 if (dev != NULL) 1900 packet_dev_mc(dev, ml, -1); 1901 kfree(ml); 1902 } 1903 rtnl_unlock(); 1904 } 1905 1906 static int 1907 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen) 1908 { 1909 struct sock *sk = sock->sk; 1910 struct packet_sock *po = pkt_sk(sk); 1911 int ret; 1912 1913 if (level != SOL_PACKET) 1914 return -ENOPROTOOPT; 1915 1916 switch (optname) { 1917 case PACKET_ADD_MEMBERSHIP: 1918 case PACKET_DROP_MEMBERSHIP: 1919 { 1920 struct packet_mreq_max mreq; 1921 int len = optlen; 1922 memset(&mreq, 0, sizeof(mreq)); 1923 if (len < sizeof(struct packet_mreq)) 1924 return -EINVAL; 1925 if (len > sizeof(mreq)) 1926 len = sizeof(mreq); 1927 if (copy_from_user(&mreq, optval, len)) 1928 return -EFAULT; 1929 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address))) 1930 return -EINVAL; 1931 if (optname == PACKET_ADD_MEMBERSHIP) 1932 ret = packet_mc_add(sk, &mreq); 1933 else 1934 ret = packet_mc_drop(sk, &mreq); 1935 return ret; 1936 } 1937 1938 case PACKET_RX_RING: 1939 case PACKET_TX_RING: 1940 { 1941 struct tpacket_req req; 1942 1943 if (optlen < sizeof(req)) 1944 return -EINVAL; 1945 if (pkt_sk(sk)->has_vnet_hdr) 1946 return -EINVAL; 1947 if (copy_from_user(&req, optval, sizeof(req))) 1948 return -EFAULT; 1949 return packet_set_ring(sk, &req, 0, optname == PACKET_TX_RING); 1950 } 1951 case PACKET_COPY_THRESH: 1952 { 1953 int val; 1954 1955 if (optlen != sizeof(val)) 1956 return -EINVAL; 1957 if (copy_from_user(&val, optval, sizeof(val))) 1958 return -EFAULT; 1959 1960 pkt_sk(sk)->copy_thresh = val; 1961 return 0; 1962 } 1963 case PACKET_VERSION: 1964 { 1965 int val; 1966 1967 if (optlen != sizeof(val)) 1968 return -EINVAL; 1969 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) 1970 return -EBUSY; 1971 if (copy_from_user(&val, optval, sizeof(val))) 1972 return -EFAULT; 1973 switch (val) { 1974 case TPACKET_V1: 1975 case TPACKET_V2: 1976 po->tp_version = val; 1977 return 0; 1978 default: 1979 return -EINVAL; 1980 } 1981 } 1982 case PACKET_RESERVE: 1983 { 1984 unsigned int val; 1985 1986 if (optlen != sizeof(val)) 1987 return -EINVAL; 1988 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) 1989 return -EBUSY; 1990 if (copy_from_user(&val, optval, sizeof(val))) 1991 return -EFAULT; 1992 po->tp_reserve = val; 1993 return 0; 1994 } 1995 case PACKET_LOSS: 1996 { 1997 unsigned int val; 1998 1999 if (optlen != sizeof(val)) 2000 return -EINVAL; 2001 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) 2002 return -EBUSY; 2003 if (copy_from_user(&val, optval, sizeof(val))) 2004 return -EFAULT; 2005 po->tp_loss = !!val; 2006 return 0; 2007 } 2008 case PACKET_AUXDATA: 2009 { 2010 int val; 2011 2012 if (optlen < sizeof(val)) 2013 return -EINVAL; 2014 if (copy_from_user(&val, optval, sizeof(val))) 2015 return -EFAULT; 2016 2017 po->auxdata = !!val; 2018 return 0; 2019 } 2020 case PACKET_ORIGDEV: 2021 { 2022 int val; 2023 2024 if (optlen < sizeof(val)) 2025 return -EINVAL; 2026 if (copy_from_user(&val, optval, sizeof(val))) 2027 return -EFAULT; 2028 2029 po->origdev = !!val; 2030 return 0; 2031 } 2032 case PACKET_VNET_HDR: 2033 { 2034 int val; 2035 2036 if (sock->type != SOCK_RAW) 2037 return -EINVAL; 2038 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) 2039 return -EBUSY; 2040 if (optlen < sizeof(val)) 2041 return -EINVAL; 2042 if (copy_from_user(&val, optval, sizeof(val))) 2043 return -EFAULT; 2044 2045 po->has_vnet_hdr = !!val; 2046 return 0; 2047 } 2048 case PACKET_TIMESTAMP: 2049 { 2050 int val; 2051 2052 if (optlen != sizeof(val)) 2053 return -EINVAL; 2054 if (copy_from_user(&val, optval, sizeof(val))) 2055 return -EFAULT; 2056 2057 po->tp_tstamp = val; 2058 return 0; 2059 } 2060 default: 2061 return -ENOPROTOOPT; 2062 } 2063 } 2064 2065 static int packet_getsockopt(struct socket *sock, int level, int optname, 2066 char __user *optval, int __user *optlen) 2067 { 2068 int len; 2069 int val; 2070 struct sock *sk = sock->sk; 2071 struct packet_sock *po = pkt_sk(sk); 2072 void *data; 2073 struct tpacket_stats st; 2074 2075 if (level != SOL_PACKET) 2076 return -ENOPROTOOPT; 2077 2078 if (get_user(len, optlen)) 2079 return -EFAULT; 2080 2081 if (len < 0) 2082 return -EINVAL; 2083 2084 switch (optname) { 2085 case PACKET_STATISTICS: 2086 if (len > sizeof(struct tpacket_stats)) 2087 len = sizeof(struct tpacket_stats); 2088 spin_lock_bh(&sk->sk_receive_queue.lock); 2089 st = po->stats; 2090 memset(&po->stats, 0, sizeof(st)); 2091 spin_unlock_bh(&sk->sk_receive_queue.lock); 2092 st.tp_packets += st.tp_drops; 2093 2094 data = &st; 2095 break; 2096 case PACKET_AUXDATA: 2097 if (len > sizeof(int)) 2098 len = sizeof(int); 2099 val = po->auxdata; 2100 2101 data = &val; 2102 break; 2103 case PACKET_ORIGDEV: 2104 if (len > sizeof(int)) 2105 len = sizeof(int); 2106 val = po->origdev; 2107 2108 data = &val; 2109 break; 2110 case PACKET_VNET_HDR: 2111 if (len > sizeof(int)) 2112 len = sizeof(int); 2113 val = po->has_vnet_hdr; 2114 2115 data = &val; 2116 break; 2117 case PACKET_VERSION: 2118 if (len > sizeof(int)) 2119 len = sizeof(int); 2120 val = po->tp_version; 2121 data = &val; 2122 break; 2123 case PACKET_HDRLEN: 2124 if (len > sizeof(int)) 2125 len = sizeof(int); 2126 if (copy_from_user(&val, optval, len)) 2127 return -EFAULT; 2128 switch (val) { 2129 case TPACKET_V1: 2130 val = sizeof(struct tpacket_hdr); 2131 break; 2132 case TPACKET_V2: 2133 val = sizeof(struct tpacket2_hdr); 2134 break; 2135 default: 2136 return -EINVAL; 2137 } 2138 data = &val; 2139 break; 2140 case PACKET_RESERVE: 2141 if (len > sizeof(unsigned int)) 2142 len = sizeof(unsigned int); 2143 val = po->tp_reserve; 2144 data = &val; 2145 break; 2146 case PACKET_LOSS: 2147 if (len > sizeof(unsigned int)) 2148 len = sizeof(unsigned int); 2149 val = po->tp_loss; 2150 data = &val; 2151 break; 2152 case PACKET_TIMESTAMP: 2153 if (len > sizeof(int)) 2154 len = sizeof(int); 2155 val = po->tp_tstamp; 2156 data = &val; 2157 break; 2158 default: 2159 return -ENOPROTOOPT; 2160 } 2161 2162 if (put_user(len, optlen)) 2163 return -EFAULT; 2164 if (copy_to_user(optval, data, len)) 2165 return -EFAULT; 2166 return 0; 2167 } 2168 2169 2170 static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data) 2171 { 2172 struct sock *sk; 2173 struct hlist_node *node; 2174 struct net_device *dev = data; 2175 struct net *net = dev_net(dev); 2176 2177 rcu_read_lock(); 2178 sk_for_each_rcu(sk, node, &net->packet.sklist) { 2179 struct packet_sock *po = pkt_sk(sk); 2180 2181 switch (msg) { 2182 case NETDEV_UNREGISTER: 2183 if (po->mclist) 2184 packet_dev_mclist(dev, po->mclist, -1); 2185 /* fallthrough */ 2186 2187 case NETDEV_DOWN: 2188 if (dev->ifindex == po->ifindex) { 2189 spin_lock(&po->bind_lock); 2190 if (po->running) { 2191 __dev_remove_pack(&po->prot_hook); 2192 __sock_put(sk); 2193 po->running = 0; 2194 sk->sk_err = ENETDOWN; 2195 if (!sock_flag(sk, SOCK_DEAD)) 2196 sk->sk_error_report(sk); 2197 } 2198 if (msg == NETDEV_UNREGISTER) { 2199 po->ifindex = -1; 2200 po->prot_hook.dev = NULL; 2201 } 2202 spin_unlock(&po->bind_lock); 2203 } 2204 break; 2205 case NETDEV_UP: 2206 if (dev->ifindex == po->ifindex) { 2207 spin_lock(&po->bind_lock); 2208 if (po->num && !po->running) { 2209 dev_add_pack(&po->prot_hook); 2210 sock_hold(sk); 2211 po->running = 1; 2212 } 2213 spin_unlock(&po->bind_lock); 2214 } 2215 break; 2216 } 2217 } 2218 rcu_read_unlock(); 2219 return NOTIFY_DONE; 2220 } 2221 2222 2223 static int packet_ioctl(struct socket *sock, unsigned int cmd, 2224 unsigned long arg) 2225 { 2226 struct sock *sk = sock->sk; 2227 2228 switch (cmd) { 2229 case SIOCOUTQ: 2230 { 2231 int amount = sk_wmem_alloc_get(sk); 2232 2233 return put_user(amount, (int __user *)arg); 2234 } 2235 case SIOCINQ: 2236 { 2237 struct sk_buff *skb; 2238 int amount = 0; 2239 2240 spin_lock_bh(&sk->sk_receive_queue.lock); 2241 skb = skb_peek(&sk->sk_receive_queue); 2242 if (skb) 2243 amount = skb->len; 2244 spin_unlock_bh(&sk->sk_receive_queue.lock); 2245 return put_user(amount, (int __user *)arg); 2246 } 2247 case SIOCGSTAMP: 2248 return sock_get_timestamp(sk, (struct timeval __user *)arg); 2249 case SIOCGSTAMPNS: 2250 return sock_get_timestampns(sk, (struct timespec __user *)arg); 2251 2252 #ifdef CONFIG_INET 2253 case SIOCADDRT: 2254 case SIOCDELRT: 2255 case SIOCDARP: 2256 case SIOCGARP: 2257 case SIOCSARP: 2258 case SIOCGIFADDR: 2259 case SIOCSIFADDR: 2260 case SIOCGIFBRDADDR: 2261 case SIOCSIFBRDADDR: 2262 case SIOCGIFNETMASK: 2263 case SIOCSIFNETMASK: 2264 case SIOCGIFDSTADDR: 2265 case SIOCSIFDSTADDR: 2266 case SIOCSIFFLAGS: 2267 return inet_dgram_ops.ioctl(sock, cmd, arg); 2268 #endif 2269 2270 default: 2271 return -ENOIOCTLCMD; 2272 } 2273 return 0; 2274 } 2275 2276 static unsigned int packet_poll(struct file *file, struct socket *sock, 2277 poll_table *wait) 2278 { 2279 struct sock *sk = sock->sk; 2280 struct packet_sock *po = pkt_sk(sk); 2281 unsigned int mask = datagram_poll(file, sock, wait); 2282 2283 spin_lock_bh(&sk->sk_receive_queue.lock); 2284 if (po->rx_ring.pg_vec) { 2285 if (!packet_previous_frame(po, &po->rx_ring, TP_STATUS_KERNEL)) 2286 mask |= POLLIN | POLLRDNORM; 2287 } 2288 spin_unlock_bh(&sk->sk_receive_queue.lock); 2289 spin_lock_bh(&sk->sk_write_queue.lock); 2290 if (po->tx_ring.pg_vec) { 2291 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) 2292 mask |= POLLOUT | POLLWRNORM; 2293 } 2294 spin_unlock_bh(&sk->sk_write_queue.lock); 2295 return mask; 2296 } 2297 2298 2299 /* Dirty? Well, I still did not learn better way to account 2300 * for user mmaps. 2301 */ 2302 2303 static void packet_mm_open(struct vm_area_struct *vma) 2304 { 2305 struct file *file = vma->vm_file; 2306 struct socket *sock = file->private_data; 2307 struct sock *sk = sock->sk; 2308 2309 if (sk) 2310 atomic_inc(&pkt_sk(sk)->mapped); 2311 } 2312 2313 static void packet_mm_close(struct vm_area_struct *vma) 2314 { 2315 struct file *file = vma->vm_file; 2316 struct socket *sock = file->private_data; 2317 struct sock *sk = sock->sk; 2318 2319 if (sk) 2320 atomic_dec(&pkt_sk(sk)->mapped); 2321 } 2322 2323 static const struct vm_operations_struct packet_mmap_ops = { 2324 .open = packet_mm_open, 2325 .close = packet_mm_close, 2326 }; 2327 2328 static void free_pg_vec(char **pg_vec, unsigned int order, unsigned int len) 2329 { 2330 int i; 2331 2332 for (i = 0; i < len; i++) { 2333 if (likely(pg_vec[i])) 2334 free_pages((unsigned long) pg_vec[i], order); 2335 } 2336 kfree(pg_vec); 2337 } 2338 2339 static inline char *alloc_one_pg_vec_page(unsigned long order) 2340 { 2341 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN; 2342 2343 return (char *) __get_free_pages(gfp_flags, order); 2344 } 2345 2346 static char **alloc_pg_vec(struct tpacket_req *req, int order) 2347 { 2348 unsigned int block_nr = req->tp_block_nr; 2349 char **pg_vec; 2350 int i; 2351 2352 pg_vec = kzalloc(block_nr * sizeof(char *), GFP_KERNEL); 2353 if (unlikely(!pg_vec)) 2354 goto out; 2355 2356 for (i = 0; i < block_nr; i++) { 2357 pg_vec[i] = alloc_one_pg_vec_page(order); 2358 if (unlikely(!pg_vec[i])) 2359 goto out_free_pgvec; 2360 } 2361 2362 out: 2363 return pg_vec; 2364 2365 out_free_pgvec: 2366 free_pg_vec(pg_vec, order, block_nr); 2367 pg_vec = NULL; 2368 goto out; 2369 } 2370 2371 static int packet_set_ring(struct sock *sk, struct tpacket_req *req, 2372 int closing, int tx_ring) 2373 { 2374 char **pg_vec = NULL; 2375 struct packet_sock *po = pkt_sk(sk); 2376 int was_running, order = 0; 2377 struct packet_ring_buffer *rb; 2378 struct sk_buff_head *rb_queue; 2379 __be16 num; 2380 int err; 2381 2382 rb = tx_ring ? &po->tx_ring : &po->rx_ring; 2383 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue; 2384 2385 err = -EBUSY; 2386 if (!closing) { 2387 if (atomic_read(&po->mapped)) 2388 goto out; 2389 if (atomic_read(&rb->pending)) 2390 goto out; 2391 } 2392 2393 if (req->tp_block_nr) { 2394 /* Sanity tests and some calculations */ 2395 err = -EBUSY; 2396 if (unlikely(rb->pg_vec)) 2397 goto out; 2398 2399 switch (po->tp_version) { 2400 case TPACKET_V1: 2401 po->tp_hdrlen = TPACKET_HDRLEN; 2402 break; 2403 case TPACKET_V2: 2404 po->tp_hdrlen = TPACKET2_HDRLEN; 2405 break; 2406 } 2407 2408 err = -EINVAL; 2409 if (unlikely((int)req->tp_block_size <= 0)) 2410 goto out; 2411 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1))) 2412 goto out; 2413 if (unlikely(req->tp_frame_size < po->tp_hdrlen + 2414 po->tp_reserve)) 2415 goto out; 2416 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1))) 2417 goto out; 2418 2419 rb->frames_per_block = req->tp_block_size/req->tp_frame_size; 2420 if (unlikely(rb->frames_per_block <= 0)) 2421 goto out; 2422 if (unlikely((rb->frames_per_block * req->tp_block_nr) != 2423 req->tp_frame_nr)) 2424 goto out; 2425 2426 err = -ENOMEM; 2427 order = get_order(req->tp_block_size); 2428 pg_vec = alloc_pg_vec(req, order); 2429 if (unlikely(!pg_vec)) 2430 goto out; 2431 } 2432 /* Done */ 2433 else { 2434 err = -EINVAL; 2435 if (unlikely(req->tp_frame_nr)) 2436 goto out; 2437 } 2438 2439 lock_sock(sk); 2440 2441 /* Detach socket from network */ 2442 spin_lock(&po->bind_lock); 2443 was_running = po->running; 2444 num = po->num; 2445 if (was_running) { 2446 __dev_remove_pack(&po->prot_hook); 2447 po->num = 0; 2448 po->running = 0; 2449 __sock_put(sk); 2450 } 2451 spin_unlock(&po->bind_lock); 2452 2453 synchronize_net(); 2454 2455 err = -EBUSY; 2456 mutex_lock(&po->pg_vec_lock); 2457 if (closing || atomic_read(&po->mapped) == 0) { 2458 err = 0; 2459 #define XC(a, b) ({ __typeof__ ((a)) __t; __t = (a); (a) = (b); __t; }) 2460 spin_lock_bh(&rb_queue->lock); 2461 pg_vec = XC(rb->pg_vec, pg_vec); 2462 rb->frame_max = (req->tp_frame_nr - 1); 2463 rb->head = 0; 2464 rb->frame_size = req->tp_frame_size; 2465 spin_unlock_bh(&rb_queue->lock); 2466 2467 order = XC(rb->pg_vec_order, order); 2468 req->tp_block_nr = XC(rb->pg_vec_len, req->tp_block_nr); 2469 2470 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; 2471 po->prot_hook.func = (po->rx_ring.pg_vec) ? 2472 tpacket_rcv : packet_rcv; 2473 skb_queue_purge(rb_queue); 2474 #undef XC 2475 if (atomic_read(&po->mapped)) 2476 pr_err("packet_mmap: vma is busy: %d\n", 2477 atomic_read(&po->mapped)); 2478 } 2479 mutex_unlock(&po->pg_vec_lock); 2480 2481 spin_lock(&po->bind_lock); 2482 if (was_running && !po->running) { 2483 sock_hold(sk); 2484 po->running = 1; 2485 po->num = num; 2486 dev_add_pack(&po->prot_hook); 2487 } 2488 spin_unlock(&po->bind_lock); 2489 2490 release_sock(sk); 2491 2492 if (pg_vec) 2493 free_pg_vec(pg_vec, order, req->tp_block_nr); 2494 out: 2495 return err; 2496 } 2497 2498 static int packet_mmap(struct file *file, struct socket *sock, 2499 struct vm_area_struct *vma) 2500 { 2501 struct sock *sk = sock->sk; 2502 struct packet_sock *po = pkt_sk(sk); 2503 unsigned long size, expected_size; 2504 struct packet_ring_buffer *rb; 2505 unsigned long start; 2506 int err = -EINVAL; 2507 int i; 2508 2509 if (vma->vm_pgoff) 2510 return -EINVAL; 2511 2512 mutex_lock(&po->pg_vec_lock); 2513 2514 expected_size = 0; 2515 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { 2516 if (rb->pg_vec) { 2517 expected_size += rb->pg_vec_len 2518 * rb->pg_vec_pages 2519 * PAGE_SIZE; 2520 } 2521 } 2522 2523 if (expected_size == 0) 2524 goto out; 2525 2526 size = vma->vm_end - vma->vm_start; 2527 if (size != expected_size) 2528 goto out; 2529 2530 start = vma->vm_start; 2531 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { 2532 if (rb->pg_vec == NULL) 2533 continue; 2534 2535 for (i = 0; i < rb->pg_vec_len; i++) { 2536 struct page *page = virt_to_page(rb->pg_vec[i]); 2537 int pg_num; 2538 2539 for (pg_num = 0; pg_num < rb->pg_vec_pages; 2540 pg_num++, page++) { 2541 err = vm_insert_page(vma, start, page); 2542 if (unlikely(err)) 2543 goto out; 2544 start += PAGE_SIZE; 2545 } 2546 } 2547 } 2548 2549 atomic_inc(&po->mapped); 2550 vma->vm_ops = &packet_mmap_ops; 2551 err = 0; 2552 2553 out: 2554 mutex_unlock(&po->pg_vec_lock); 2555 return err; 2556 } 2557 2558 static const struct proto_ops packet_ops_spkt = { 2559 .family = PF_PACKET, 2560 .owner = THIS_MODULE, 2561 .release = packet_release, 2562 .bind = packet_bind_spkt, 2563 .connect = sock_no_connect, 2564 .socketpair = sock_no_socketpair, 2565 .accept = sock_no_accept, 2566 .getname = packet_getname_spkt, 2567 .poll = datagram_poll, 2568 .ioctl = packet_ioctl, 2569 .listen = sock_no_listen, 2570 .shutdown = sock_no_shutdown, 2571 .setsockopt = sock_no_setsockopt, 2572 .getsockopt = sock_no_getsockopt, 2573 .sendmsg = packet_sendmsg_spkt, 2574 .recvmsg = packet_recvmsg, 2575 .mmap = sock_no_mmap, 2576 .sendpage = sock_no_sendpage, 2577 }; 2578 2579 static const struct proto_ops packet_ops = { 2580 .family = PF_PACKET, 2581 .owner = THIS_MODULE, 2582 .release = packet_release, 2583 .bind = packet_bind, 2584 .connect = sock_no_connect, 2585 .socketpair = sock_no_socketpair, 2586 .accept = sock_no_accept, 2587 .getname = packet_getname, 2588 .poll = packet_poll, 2589 .ioctl = packet_ioctl, 2590 .listen = sock_no_listen, 2591 .shutdown = sock_no_shutdown, 2592 .setsockopt = packet_setsockopt, 2593 .getsockopt = packet_getsockopt, 2594 .sendmsg = packet_sendmsg, 2595 .recvmsg = packet_recvmsg, 2596 .mmap = packet_mmap, 2597 .sendpage = sock_no_sendpage, 2598 }; 2599 2600 static const struct net_proto_family packet_family_ops = { 2601 .family = PF_PACKET, 2602 .create = packet_create, 2603 .owner = THIS_MODULE, 2604 }; 2605 2606 static struct notifier_block packet_netdev_notifier = { 2607 .notifier_call = packet_notifier, 2608 }; 2609 2610 #ifdef CONFIG_PROC_FS 2611 2612 static void *packet_seq_start(struct seq_file *seq, loff_t *pos) 2613 __acquires(RCU) 2614 { 2615 struct net *net = seq_file_net(seq); 2616 2617 rcu_read_lock(); 2618 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos); 2619 } 2620 2621 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos) 2622 { 2623 struct net *net = seq_file_net(seq); 2624 return seq_hlist_next_rcu(v, &net->packet.sklist, pos); 2625 } 2626 2627 static void packet_seq_stop(struct seq_file *seq, void *v) 2628 __releases(RCU) 2629 { 2630 rcu_read_unlock(); 2631 } 2632 2633 static int packet_seq_show(struct seq_file *seq, void *v) 2634 { 2635 if (v == SEQ_START_TOKEN) 2636 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n"); 2637 else { 2638 struct sock *s = sk_entry(v); 2639 const struct packet_sock *po = pkt_sk(s); 2640 2641 seq_printf(seq, 2642 "%p %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n", 2643 s, 2644 atomic_read(&s->sk_refcnt), 2645 s->sk_type, 2646 ntohs(po->num), 2647 po->ifindex, 2648 po->running, 2649 atomic_read(&s->sk_rmem_alloc), 2650 sock_i_uid(s), 2651 sock_i_ino(s)); 2652 } 2653 2654 return 0; 2655 } 2656 2657 static const struct seq_operations packet_seq_ops = { 2658 .start = packet_seq_start, 2659 .next = packet_seq_next, 2660 .stop = packet_seq_stop, 2661 .show = packet_seq_show, 2662 }; 2663 2664 static int packet_seq_open(struct inode *inode, struct file *file) 2665 { 2666 return seq_open_net(inode, file, &packet_seq_ops, 2667 sizeof(struct seq_net_private)); 2668 } 2669 2670 static const struct file_operations packet_seq_fops = { 2671 .owner = THIS_MODULE, 2672 .open = packet_seq_open, 2673 .read = seq_read, 2674 .llseek = seq_lseek, 2675 .release = seq_release_net, 2676 }; 2677 2678 #endif 2679 2680 static int __net_init packet_net_init(struct net *net) 2681 { 2682 spin_lock_init(&net->packet.sklist_lock); 2683 INIT_HLIST_HEAD(&net->packet.sklist); 2684 2685 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops)) 2686 return -ENOMEM; 2687 2688 return 0; 2689 } 2690 2691 static void __net_exit packet_net_exit(struct net *net) 2692 { 2693 proc_net_remove(net, "packet"); 2694 } 2695 2696 static struct pernet_operations packet_net_ops = { 2697 .init = packet_net_init, 2698 .exit = packet_net_exit, 2699 }; 2700 2701 2702 static void __exit packet_exit(void) 2703 { 2704 unregister_netdevice_notifier(&packet_netdev_notifier); 2705 unregister_pernet_subsys(&packet_net_ops); 2706 sock_unregister(PF_PACKET); 2707 proto_unregister(&packet_proto); 2708 } 2709 2710 static int __init packet_init(void) 2711 { 2712 int rc = proto_register(&packet_proto, 0); 2713 2714 if (rc != 0) 2715 goto out; 2716 2717 sock_register(&packet_family_ops); 2718 register_pernet_subsys(&packet_net_ops); 2719 register_netdevice_notifier(&packet_netdev_notifier); 2720 out: 2721 return rc; 2722 } 2723 2724 module_init(packet_init); 2725 module_exit(packet_exit); 2726 MODULE_LICENSE("GPL"); 2727 MODULE_ALIAS_NETPROTO(PF_PACKET); 2728