1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * net busy poll support 4 * Copyright(c) 2013 Intel Corporation. 5 * 6 * Author: Eliezer Tamir 7 * 8 * Contact Information: 9 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 10 */ 11 12 #ifndef _LINUX_NET_BUSY_POLL_H 13 #define _LINUX_NET_BUSY_POLL_H 14 15 #include <linux/netdevice.h> 16 #include <linux/sched/clock.h> 17 #include <linux/sched/signal.h> 18 #include <net/ip.h> 19 #include <net/xdp.h> 20 21 /* 0 - Reserved to indicate value not set 22 * 1..NR_CPUS - Reserved for sender_cpu 23 * NR_CPUS+1..~0 - Region available for NAPI IDs 24 */ 25 #define MIN_NAPI_ID ((unsigned int)(NR_CPUS + 1)) 26 27 #define BUSY_POLL_BUDGET 8 28 29 #ifdef CONFIG_NET_RX_BUSY_POLL 30 31 struct napi_struct; 32 extern unsigned int sysctl_net_busy_read __read_mostly; 33 extern unsigned int sysctl_net_busy_poll __read_mostly; 34 35 static inline bool net_busy_loop_on(void) 36 { 37 return READ_ONCE(sysctl_net_busy_poll); 38 } 39 40 static inline bool sk_can_busy_loop(const struct sock *sk) 41 { 42 return READ_ONCE(sk->sk_ll_usec) && !signal_pending(current); 43 } 44 45 bool sk_busy_loop_end(void *p, unsigned long start_time); 46 47 void napi_busy_loop(unsigned int napi_id, 48 bool (*loop_end)(void *, unsigned long), 49 void *loop_end_arg, bool prefer_busy_poll, u16 budget); 50 51 void napi_busy_loop_rcu(unsigned int napi_id, 52 bool (*loop_end)(void *, unsigned long), 53 void *loop_end_arg, bool prefer_busy_poll, u16 budget); 54 55 void napi_suspend_irqs(unsigned int napi_id); 56 void napi_resume_irqs(unsigned int napi_id); 57 58 #else /* CONFIG_NET_RX_BUSY_POLL */ 59 static inline unsigned long net_busy_loop_on(void) 60 { 61 return 0; 62 } 63 64 static inline bool sk_can_busy_loop(struct sock *sk) 65 { 66 return false; 67 } 68 69 #endif /* CONFIG_NET_RX_BUSY_POLL */ 70 71 static inline unsigned long busy_loop_current_time(void) 72 { 73 #ifdef CONFIG_NET_RX_BUSY_POLL 74 return (unsigned long)(ktime_get_ns() >> 10); 75 #else 76 return 0; 77 #endif 78 } 79 80 /* in poll/select we use the global sysctl_net_ll_poll value */ 81 static inline bool busy_loop_timeout(unsigned long start_time) 82 { 83 #ifdef CONFIG_NET_RX_BUSY_POLL 84 unsigned long bp_usec = READ_ONCE(sysctl_net_busy_poll); 85 86 if (bp_usec) { 87 unsigned long end_time = start_time + bp_usec; 88 unsigned long now = busy_loop_current_time(); 89 90 return time_after(now, end_time); 91 } 92 #endif 93 return true; 94 } 95 96 static inline bool sk_busy_loop_timeout(struct sock *sk, 97 unsigned long start_time) 98 { 99 #ifdef CONFIG_NET_RX_BUSY_POLL 100 unsigned long bp_usec = READ_ONCE(sk->sk_ll_usec); 101 102 if (bp_usec) { 103 unsigned long end_time = start_time + bp_usec; 104 unsigned long now = busy_loop_current_time(); 105 106 return time_after(now, end_time); 107 } 108 #endif 109 return true; 110 } 111 112 static inline void sk_busy_loop(struct sock *sk, int nonblock) 113 { 114 #ifdef CONFIG_NET_RX_BUSY_POLL 115 unsigned int napi_id = READ_ONCE(sk->sk_napi_id); 116 117 if (napi_id >= MIN_NAPI_ID) 118 napi_busy_loop(napi_id, nonblock ? NULL : sk_busy_loop_end, sk, 119 READ_ONCE(sk->sk_prefer_busy_poll), 120 READ_ONCE(sk->sk_busy_poll_budget) ?: BUSY_POLL_BUDGET); 121 #endif 122 } 123 124 /* used in the NIC receive handler to mark the skb */ 125 static inline void skb_mark_napi_id(struct sk_buff *skb, 126 struct napi_struct *napi) 127 { 128 #ifdef CONFIG_NET_RX_BUSY_POLL 129 /* If the skb was already marked with a valid NAPI ID, avoid overwriting 130 * it. 131 */ 132 if (skb->napi_id < MIN_NAPI_ID) 133 skb->napi_id = napi->napi_id; 134 #endif 135 } 136 137 /* used in the protocol handler to propagate the napi_id to the socket */ 138 static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb) 139 { 140 #ifdef CONFIG_NET_RX_BUSY_POLL 141 if (unlikely(READ_ONCE(sk->sk_napi_id) != skb->napi_id)) 142 WRITE_ONCE(sk->sk_napi_id, skb->napi_id); 143 #endif 144 sk_rx_queue_update(sk, skb); 145 } 146 147 /* Variant of sk_mark_napi_id() for passive flow setup, 148 * as sk->sk_napi_id and sk->sk_rx_queue_mapping content 149 * needs to be set. 150 */ 151 static inline void sk_mark_napi_id_set(struct sock *sk, 152 const struct sk_buff *skb) 153 { 154 #ifdef CONFIG_NET_RX_BUSY_POLL 155 WRITE_ONCE(sk->sk_napi_id, skb->napi_id); 156 #endif 157 sk_rx_queue_set(sk, skb); 158 } 159 160 static inline void __sk_mark_napi_id_once(struct sock *sk, unsigned int napi_id) 161 { 162 #ifdef CONFIG_NET_RX_BUSY_POLL 163 if (!READ_ONCE(sk->sk_napi_id)) 164 WRITE_ONCE(sk->sk_napi_id, napi_id); 165 #endif 166 } 167 168 /* variant used for unconnected sockets */ 169 static inline void sk_mark_napi_id_once(struct sock *sk, 170 const struct sk_buff *skb) 171 { 172 #ifdef CONFIG_NET_RX_BUSY_POLL 173 __sk_mark_napi_id_once(sk, skb->napi_id); 174 #endif 175 } 176 177 static inline void sk_mark_napi_id_once_xdp(struct sock *sk, 178 const struct xdp_buff *xdp) 179 { 180 #ifdef CONFIG_NET_RX_BUSY_POLL 181 __sk_mark_napi_id_once(sk, xdp->rxq->napi_id); 182 #endif 183 } 184 185 #endif /* _LINUX_NET_BUSY_POLL_H */ 186