1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "io_uring.h" 4 #include "napi.h" 5 6 #ifdef CONFIG_NET_RX_BUSY_POLL 7 8 /* Timeout for cleanout of stale entries. */ 9 #define NAPI_TIMEOUT (60 * HZ) 10 11 struct io_napi_entry { 12 unsigned int napi_id; 13 struct list_head list; 14 15 unsigned long timeout; 16 struct hlist_node node; 17 18 struct rcu_head rcu; 19 }; 20 21 static struct io_napi_entry *io_napi_hash_find(struct hlist_head *hash_list, 22 unsigned int napi_id) 23 { 24 struct io_napi_entry *e; 25 26 hlist_for_each_entry_rcu(e, hash_list, node) { 27 if (e->napi_id != napi_id) 28 continue; 29 return e; 30 } 31 32 return NULL; 33 } 34 35 static inline ktime_t net_to_ktime(unsigned long t) 36 { 37 /* napi approximating usecs, reverse busy_loop_current_time */ 38 return ns_to_ktime(t << 10); 39 } 40 41 int __io_napi_add_id(struct io_ring_ctx *ctx, unsigned int napi_id, 42 unsigned int mode) 43 { 44 struct hlist_head *hash_list; 45 struct io_napi_entry *e; 46 47 /* Non-NAPI IDs can be rejected. */ 48 if (!napi_id_valid(napi_id)) 49 return -EINVAL; 50 51 hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; 52 53 scoped_guard(rcu) { 54 e = io_napi_hash_find(hash_list, napi_id); 55 if (e) { 56 WRITE_ONCE(e->timeout, jiffies + NAPI_TIMEOUT); 57 return -EEXIST; 58 } 59 } 60 61 e = kmalloc_obj(*e, GFP_NOWAIT); 62 if (!e) 63 return -ENOMEM; 64 65 e->napi_id = napi_id; 66 e->timeout = jiffies + NAPI_TIMEOUT; 67 68 /* 69 * guard(spinlock) is not used to manually unlock it before calling 70 * kfree() 71 */ 72 spin_lock(&ctx->napi_lock); 73 if (unlikely(READ_ONCE(ctx->napi_track_mode) != mode)) { 74 spin_unlock(&ctx->napi_lock); 75 kfree(e); 76 return -EINVAL; 77 } 78 if (unlikely(io_napi_hash_find(hash_list, napi_id))) { 79 spin_unlock(&ctx->napi_lock); 80 kfree(e); 81 return -EEXIST; 82 } 83 84 hlist_add_tail_rcu(&e->node, hash_list); 85 list_add_tail_rcu(&e->list, &ctx->napi_list); 86 spin_unlock(&ctx->napi_lock); 87 return 0; 88 } 89 90 static int __io_napi_del_id(struct io_ring_ctx *ctx, unsigned int napi_id) 91 { 92 struct hlist_head *hash_list; 93 struct io_napi_entry *e; 94 95 /* Non-NAPI IDs can be rejected. */ 96 if (!napi_id_valid(napi_id)) 97 return -EINVAL; 98 99 hash_list = &ctx->napi_ht[hash_min(napi_id, HASH_BITS(ctx->napi_ht))]; 100 guard(spinlock)(&ctx->napi_lock); 101 e = io_napi_hash_find(hash_list, napi_id); 102 if (!e) 103 return -ENOENT; 104 105 list_del_rcu(&e->list); 106 hash_del_rcu(&e->node); 107 kfree_rcu(e, rcu); 108 return 0; 109 } 110 111 static void __io_napi_remove_stale(struct io_ring_ctx *ctx) 112 { 113 struct io_napi_entry *e, *next; 114 115 guard(spinlock)(&ctx->napi_lock); 116 list_for_each_entry_safe(e, next, &ctx->napi_list, list) { 117 if (time_after(jiffies, READ_ONCE(e->timeout))) { 118 list_del_rcu(&e->list); 119 hash_del_rcu(&e->node); 120 kfree_rcu(e, rcu); 121 } 122 } 123 } 124 125 static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale) 126 { 127 if (is_stale) 128 __io_napi_remove_stale(ctx); 129 } 130 131 static inline bool io_napi_busy_loop_timeout(ktime_t start_time, 132 ktime_t bp) 133 { 134 if (bp) { 135 ktime_t end_time = ktime_add(start_time, bp); 136 ktime_t now = net_to_ktime(busy_loop_current_time()); 137 138 return ktime_after(now, end_time); 139 } 140 141 return true; 142 } 143 144 static bool io_napi_busy_loop_should_end(void *data, 145 unsigned long start_time) 146 { 147 struct io_wait_queue *iowq = data; 148 149 if (signal_pending(current)) 150 return true; 151 if (io_should_wake(iowq) || io_has_work(iowq->ctx)) 152 return true; 153 if (io_napi_busy_loop_timeout(net_to_ktime(start_time), 154 iowq->napi_busy_poll_dt)) 155 return true; 156 157 return false; 158 } 159 160 /* 161 * never report stale entries 162 */ 163 static bool static_tracking_do_busy_loop(struct io_ring_ctx *ctx, 164 bool (*loop_end)(void *, unsigned long), 165 void *loop_end_arg) 166 { 167 struct io_napi_entry *e; 168 169 list_for_each_entry_rcu(e, &ctx->napi_list, list) 170 napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, 171 ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); 172 return false; 173 } 174 175 static bool 176 dynamic_tracking_do_busy_loop(struct io_ring_ctx *ctx, 177 bool (*loop_end)(void *, unsigned long), 178 void *loop_end_arg) 179 { 180 struct io_napi_entry *e; 181 bool is_stale = false; 182 183 list_for_each_entry_rcu(e, &ctx->napi_list, list) { 184 napi_busy_loop_rcu(e->napi_id, loop_end, loop_end_arg, 185 ctx->napi_prefer_busy_poll, BUSY_POLL_BUDGET); 186 187 if (time_after(jiffies, READ_ONCE(e->timeout))) 188 is_stale = true; 189 } 190 191 return is_stale; 192 } 193 194 static inline bool 195 __io_napi_do_busy_loop(struct io_ring_ctx *ctx, 196 bool (*loop_end)(void *, unsigned long), 197 void *loop_end_arg) 198 { 199 switch (READ_ONCE(ctx->napi_track_mode)) { 200 case IO_URING_NAPI_TRACKING_STATIC: 201 return static_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); 202 case IO_URING_NAPI_TRACKING_DYNAMIC: 203 return dynamic_tracking_do_busy_loop(ctx, loop_end, loop_end_arg); 204 default: 205 return false; 206 } 207 } 208 209 static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx, 210 struct io_wait_queue *iowq) 211 { 212 unsigned long start_time = busy_loop_current_time(); 213 bool (*loop_end)(void *, unsigned long) = NULL; 214 void *loop_end_arg = NULL; 215 bool is_stale = false; 216 217 /* Singular lists use a different napi loop end check function and are 218 * only executed once. 219 */ 220 if (list_is_singular(&ctx->napi_list)) { 221 loop_end = io_napi_busy_loop_should_end; 222 loop_end_arg = iowq; 223 } 224 225 scoped_guard(rcu) { 226 do { 227 is_stale = __io_napi_do_busy_loop(ctx, loop_end, 228 loop_end_arg); 229 } while (!io_napi_busy_loop_should_end(iowq, start_time) && 230 !loop_end_arg); 231 } 232 233 io_napi_remove_stale(ctx, is_stale); 234 } 235 236 /* 237 * io_napi_init() - Init napi settings 238 * @ctx: pointer to io-uring context structure 239 * 240 * Init napi settings in the io-uring context. 241 */ 242 void io_napi_init(struct io_ring_ctx *ctx) 243 { 244 u64 sys_dt = READ_ONCE(sysctl_net_busy_poll) * NSEC_PER_USEC; 245 246 INIT_LIST_HEAD(&ctx->napi_list); 247 spin_lock_init(&ctx->napi_lock); 248 ctx->napi_prefer_busy_poll = false; 249 ctx->napi_busy_poll_dt = ns_to_ktime(sys_dt); 250 ctx->napi_track_mode = IO_URING_NAPI_TRACKING_INACTIVE; 251 } 252 253 /* 254 * io_napi_free() - Deallocate napi 255 * @ctx: pointer to io-uring context structure 256 * 257 * Free the napi list and the hash table in the io-uring context. 258 */ 259 void io_napi_free(struct io_ring_ctx *ctx) 260 { 261 struct io_napi_entry *e, *next; 262 263 guard(spinlock)(&ctx->napi_lock); 264 list_for_each_entry_safe(e, next, &ctx->napi_list, list) { 265 hash_del_rcu(&e->node); 266 kfree_rcu(e, rcu); 267 } 268 INIT_LIST_HEAD_RCU(&ctx->napi_list); 269 } 270 271 static int io_napi_register_napi(struct io_ring_ctx *ctx, 272 struct io_uring_napi *napi) 273 { 274 switch (napi->op_param) { 275 case IO_URING_NAPI_TRACKING_DYNAMIC: 276 case IO_URING_NAPI_TRACKING_STATIC: 277 break; 278 default: 279 return -EINVAL; 280 } 281 WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE); 282 io_napi_free(ctx); 283 /* cap NAPI at 10 msec of spin time */ 284 napi->busy_poll_to = min(10000, napi->busy_poll_to); 285 WRITE_ONCE(ctx->napi_busy_poll_dt, napi->busy_poll_to * NSEC_PER_USEC); 286 WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi->prefer_busy_poll); 287 WRITE_ONCE(ctx->napi_track_mode, napi->op_param); 288 return 0; 289 } 290 291 /* 292 * io_napi_register() - Register napi with io-uring 293 * @ctx: pointer to io-uring context structure 294 * @arg: pointer to io_uring_napi structure 295 * 296 * Register napi in the io-uring context. 297 */ 298 int io_register_napi(struct io_ring_ctx *ctx, void __user *arg) 299 { 300 const struct io_uring_napi curr = { 301 .busy_poll_to = ktime_to_us(ctx->napi_busy_poll_dt), 302 .prefer_busy_poll = ctx->napi_prefer_busy_poll, 303 .op_param = ctx->napi_track_mode 304 }; 305 struct io_uring_napi napi; 306 307 if (ctx->flags & IORING_SETUP_IOPOLL) 308 return -EINVAL; 309 if (copy_from_user(&napi, arg, sizeof(napi))) 310 return -EFAULT; 311 if (napi.pad[0] || napi.pad[1] || napi.resv) 312 return -EINVAL; 313 314 if (copy_to_user(arg, &curr, sizeof(curr))) 315 return -EFAULT; 316 317 switch (napi.opcode) { 318 case IO_URING_NAPI_REGISTER_OP: 319 return io_napi_register_napi(ctx, &napi); 320 case IO_URING_NAPI_STATIC_ADD_ID: 321 if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) 322 return -EINVAL; 323 return __io_napi_add_id(ctx, napi.op_param, 324 IO_URING_NAPI_TRACKING_STATIC); 325 case IO_URING_NAPI_STATIC_DEL_ID: 326 if (curr.op_param != IO_URING_NAPI_TRACKING_STATIC) 327 return -EINVAL; 328 return __io_napi_del_id(ctx, napi.op_param); 329 default: 330 return -EINVAL; 331 } 332 } 333 334 /* 335 * io_napi_unregister() - Unregister napi with io-uring 336 * @ctx: pointer to io-uring context structure 337 * @arg: pointer to io_uring_napi structure 338 * 339 * Unregister napi. If arg has been specified copy the busy poll timeout and 340 * prefer busy poll setting to the passed in structure. 341 */ 342 int io_unregister_napi(struct io_ring_ctx *ctx, void __user *arg) 343 { 344 const struct io_uring_napi curr = { 345 .busy_poll_to = ktime_to_us(ctx->napi_busy_poll_dt), 346 .prefer_busy_poll = ctx->napi_prefer_busy_poll 347 }; 348 349 if (arg && copy_to_user(arg, &curr, sizeof(curr))) 350 return -EFAULT; 351 352 WRITE_ONCE(ctx->napi_track_mode, IO_URING_NAPI_TRACKING_INACTIVE); 353 WRITE_ONCE(ctx->napi_busy_poll_dt, 0); 354 WRITE_ONCE(ctx->napi_prefer_busy_poll, false); 355 io_napi_free(ctx); 356 return 0; 357 } 358 359 /* 360 * __io_napi_busy_loop() - execute busy poll loop 361 * @ctx: pointer to io-uring context structure 362 * @iowq: pointer to io wait queue 363 * 364 * Execute the busy poll loop and merge the spliced off list. 365 */ 366 void __io_napi_busy_loop(struct io_ring_ctx *ctx, struct io_wait_queue *iowq) 367 { 368 if (ctx->flags & IORING_SETUP_SQPOLL) 369 return; 370 371 iowq->napi_busy_poll_dt = READ_ONCE(ctx->napi_busy_poll_dt); 372 if (iowq->timeout != KTIME_MAX) { 373 ktime_t dt = ktime_sub(iowq->timeout, io_get_time(ctx)); 374 375 /* an absolute timeout may have passed already */ 376 dt = max_t(ktime_t, dt, 0); 377 iowq->napi_busy_poll_dt = min_t(u64, iowq->napi_busy_poll_dt, dt); 378 } 379 380 iowq->napi_prefer_busy_poll = READ_ONCE(ctx->napi_prefer_busy_poll); 381 io_napi_blocking_busy_loop(ctx, iowq); 382 } 383 384 /* 385 * io_napi_sqpoll_busy_poll() - busy poll loop for sqpoll 386 * @ctx: pointer to io-uring context structure 387 * 388 * Splice of the napi list and execute the napi busy poll loop. 389 */ 390 int io_napi_sqpoll_busy_poll(struct io_ring_ctx *ctx) 391 { 392 bool is_stale = false; 393 394 if (!READ_ONCE(ctx->napi_busy_poll_dt)) 395 return 0; 396 if (list_empty_careful(&ctx->napi_list)) 397 return 0; 398 399 scoped_guard(rcu) { 400 is_stale = __io_napi_do_busy_loop(ctx, NULL, NULL); 401 } 402 403 io_napi_remove_stale(ctx, is_stale); 404 return 1; 405 } 406 407 #endif 408