1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __PACKET_INTERNAL_H__ 3 #define __PACKET_INTERNAL_H__ 4 5 #include <linux/refcount.h> 6 7 struct packet_mclist { 8 struct packet_mclist *next; 9 int ifindex; 10 int count; 11 unsigned short type; 12 unsigned short alen; 13 unsigned char addr[MAX_ADDR_LEN]; 14 }; 15 16 /* kbdq - kernel block descriptor queue */ 17 struct tpacket_kbdq_core { 18 struct pgv *pkbdq; 19 unsigned int feature_req_word; 20 unsigned int hdrlen; 21 unsigned char reset_pending_on_curr_blk; 22 unsigned char delete_blk_timer; 23 unsigned short kactive_blk_num; 24 unsigned short blk_sizeof_priv; 25 26 /* last_kactive_blk_num: 27 * trick to see if user-space has caught up 28 * in order to avoid refreshing timer when every single pkt arrives. 29 */ 30 unsigned short last_kactive_blk_num; 31 32 char *pkblk_start; 33 char *pkblk_end; 34 int kblk_size; 35 unsigned int max_frame_len; 36 unsigned int knum_blocks; 37 uint64_t knxt_seq_num; 38 char *prev; 39 char *nxt_offset; 40 struct sk_buff *skb; 41 42 rwlock_t blk_fill_in_prog_lock; 43 44 /* Default is set to 8ms */ 45 #define DEFAULT_PRB_RETIRE_TOV (8) 46 47 unsigned short retire_blk_tov; 48 unsigned short version; 49 unsigned long tov_in_jiffies; 50 51 /* timer to retire an outstanding block */ 52 struct timer_list retire_blk_timer; 53 }; 54 55 struct pgv { 56 char *buffer; 57 }; 58 59 struct packet_ring_buffer { 60 struct pgv *pg_vec; 61 62 unsigned int head; 63 unsigned int frames_per_block; 64 unsigned int frame_size; 65 unsigned int frame_max; 66 67 unsigned int pg_vec_order; 68 unsigned int pg_vec_pages; 69 unsigned int pg_vec_len; 70 71 unsigned int __percpu *pending_refcnt; 72 73 union { 74 unsigned long *rx_owner_map; 75 struct tpacket_kbdq_core prb_bdqc; 76 }; 77 }; 78 79 extern struct mutex fanout_mutex; 80 #define PACKET_FANOUT_MAX (1 << 16) 81 82 struct packet_fanout { 83 possible_net_t net; 84 unsigned int num_members; 85 u32 max_num_members; 86 u16 id; 87 u8 type; 88 u8 flags; 89 union { 90 atomic_t rr_cur; 91 struct bpf_prog __rcu *bpf_prog; 92 }; 93 struct list_head list; 94 spinlock_t lock; 95 refcount_t sk_ref; 96 struct packet_type prot_hook ____cacheline_aligned_in_smp; 97 struct sock __rcu *arr[] __counted_by(max_num_members); 98 }; 99 100 struct packet_rollover { 101 int sock; 102 atomic_long_t num; 103 atomic_long_t num_huge; 104 atomic_long_t num_failed; 105 #define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32)) 106 u32 history[ROLLOVER_HLEN] ____cacheline_aligned; 107 } ____cacheline_aligned_in_smp; 108 109 struct packet_sock { 110 /* struct sock has to be the first member of packet_sock */ 111 struct sock sk; 112 struct packet_fanout *fanout; 113 union tpacket_stats_u stats; 114 struct packet_ring_buffer rx_ring; 115 struct packet_ring_buffer tx_ring; 116 int copy_thresh; 117 spinlock_t bind_lock; 118 struct mutex pg_vec_lock; 119 unsigned long flags; 120 int ifindex; /* bound device */ 121 u8 vnet_hdr_sz; 122 __be16 num; 123 struct packet_rollover *rollover; 124 struct packet_mclist *mclist; 125 atomic_long_t mapped; 126 enum tpacket_versions tp_version; 127 unsigned int tp_hdrlen; 128 unsigned int tp_reserve; 129 unsigned int tp_tstamp; 130 struct completion skb_completion; 131 struct net_device __rcu *cached_dev; 132 struct packet_type prot_hook ____cacheline_aligned_in_smp; 133 atomic_t tp_drops ____cacheline_aligned_in_smp; 134 }; 135 136 #define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk) 137 138 enum packet_sock_flags { 139 PACKET_SOCK_ORIGDEV, 140 PACKET_SOCK_AUXDATA, 141 PACKET_SOCK_TX_HAS_OFF, 142 PACKET_SOCK_TP_LOSS, 143 PACKET_SOCK_RUNNING, 144 PACKET_SOCK_PRESSURE, 145 PACKET_SOCK_QDISC_BYPASS, 146 }; 147 148 static inline void packet_sock_flag_set(struct packet_sock *po, 149 enum packet_sock_flags flag, 150 bool val) 151 { 152 if (val) 153 set_bit(flag, &po->flags); 154 else 155 clear_bit(flag, &po->flags); 156 } 157 158 static inline bool packet_sock_flag(const struct packet_sock *po, 159 enum packet_sock_flags flag) 160 { 161 return test_bit(flag, &po->flags); 162 } 163 164 #endif 165