1 /*- 2 * Copyright (c) 2016-2018 Netflix, Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 #include <sys/cdefs.h> 27 #include "opt_inet.h" 28 #include "opt_inet6.h" 29 #include "opt_rss.h" 30 31 /** 32 * Some notes about usage. 33 * 34 * The tcp_hpts system is designed to provide a high precision timer 35 * system for tcp. Its main purpose is to provide a mechanism for 36 * pacing packets out onto the wire. It can be used in two ways 37 * by a given TCP stack (and those two methods can be used simultaneously). 38 * 39 * First, and probably the main thing its used by Rack and BBR, it can 40 * be used to call tcp_output() of a transport stack at some time in the future. 41 * The normal way this is done is that tcp_output() of the stack schedules 42 * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The 43 * slot is the time from now that the stack wants to be called but it 44 * must be converted to tcp_hpts's notion of slot. This is done with 45 * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical 46 * call from the tcp_output() routine might look like: 47 * 48 * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550)); 49 * 50 * The above would schedule tcp_ouput() to be called in 550 useconds. 51 * Note that if using this mechanism the stack will want to add near 52 * its top a check to prevent unwanted calls (from user land or the 53 * arrival of incoming ack's). So it would add something like: 54 * 55 * if (tcp_in_hpts(inp)) 56 * return; 57 * 58 * to prevent output processing until the time alotted has gone by. 59 * Of course this is a bare bones example and the stack will probably 60 * have more consideration then just the above. 61 * 62 * In order to run input queued segments from the HPTS context the 63 * tcp stack must define an input function for 64 * tfb_do_queued_segments(). This function understands 65 * how to dequeue a array of packets that were input and 66 * knows how to call the correct processing routine. 67 * 68 * Locking in this is important as well so most likely the 69 * stack will need to define the tfb_do_segment_nounlock() 70 * splitting tfb_do_segment() into two parts. The main processing 71 * part that does not unlock the INP and returns a value of 1 or 0. 72 * It returns 0 if all is well and the lock was not released. It 73 * returns 1 if we had to destroy the TCB (a reset received etc). 74 * The remains of tfb_do_segment() then become just a simple call 75 * to the tfb_do_segment_nounlock() function and check the return 76 * code and possibly unlock. 77 * 78 * The stack must also set the flag on the INP that it supports this 79 * feature i.e. INP_SUPPORTS_MBUFQ. The LRO code recoginizes 80 * this flag as well and will queue packets when it is set. 81 * There are other flags as well INP_MBUF_QUEUE_READY and 82 * INP_DONT_SACK_QUEUE. The first flag tells the LRO code 83 * that we are in the pacer for output so there is no 84 * need to wake up the hpts system to get immediate 85 * input. The second tells the LRO code that its okay 86 * if a SACK arrives you can still defer input and let 87 * the current hpts timer run (this is usually set when 88 * a rack timer is up so we know SACK's are happening 89 * on the connection already and don't want to wakeup yet). 90 * 91 * There is a common functions within the rack_bbr_common code 92 * version i.e. ctf_do_queued_segments(). This function 93 * knows how to take the input queue of packets from tp->t_inqueue 94 * and process them digging out all the arguments, calling any bpf tap and 95 * calling into tfb_do_segment_nounlock(). The common 96 * function (ctf_do_queued_segments()) requires that 97 * you have defined the tfb_do_segment_nounlock() as 98 * described above. 99 */ 100 101 #include <sys/param.h> 102 #include <sys/bus.h> 103 #include <sys/interrupt.h> 104 #include <sys/module.h> 105 #include <sys/kernel.h> 106 #include <sys/hhook.h> 107 #include <sys/malloc.h> 108 #include <sys/mbuf.h> 109 #include <sys/proc.h> /* for proc0 declaration */ 110 #include <sys/socket.h> 111 #include <sys/socketvar.h> 112 #include <sys/sysctl.h> 113 #include <sys/systm.h> 114 #include <sys/refcount.h> 115 #include <sys/sched.h> 116 #include <sys/queue.h> 117 #include <sys/smp.h> 118 #include <sys/counter.h> 119 #include <sys/time.h> 120 #include <sys/kthread.h> 121 #include <sys/kern_prefetch.h> 122 123 #include <vm/uma.h> 124 #include <vm/vm.h> 125 126 #include <net/route.h> 127 #include <net/vnet.h> 128 129 #ifdef RSS 130 #include <net/netisr.h> 131 #include <net/rss_config.h> 132 #endif 133 134 #define TCPSTATES /* for logging */ 135 136 #include <netinet/in.h> 137 #include <netinet/in_kdtrace.h> 138 #include <netinet/in_pcb.h> 139 #include <netinet/ip.h> 140 #include <netinet/ip_icmp.h> /* required for icmp_var.h */ 141 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */ 142 #include <netinet/ip_var.h> 143 #include <netinet/ip6.h> 144 #include <netinet6/in6_pcb.h> 145 #include <netinet6/ip6_var.h> 146 #include <netinet/tcp.h> 147 #include <netinet/tcp_fsm.h> 148 #include <netinet/tcp_seq.h> 149 #include <netinet/tcp_timer.h> 150 #include <netinet/tcp_var.h> 151 #include <netinet/tcpip.h> 152 #include <netinet/cc/cc.h> 153 #include <netinet/tcp_hpts.h> 154 #include <netinet/tcp_log_buf.h> 155 156 #ifdef tcp_offload 157 #include <netinet/tcp_offload.h> 158 #endif 159 160 /* 161 * The hpts uses a 102400 wheel. The wheel 162 * defines the time in 10 usec increments (102400 x 10). 163 * This gives a range of 10usec - 1024ms to place 164 * an entry within. If the user requests more than 165 * 1.024 second, a remaineder is attached and the hpts 166 * when seeing the remainder will re-insert the 167 * inpcb forward in time from where it is until 168 * the remainder is zero. 169 */ 170 171 #define NUM_OF_HPTSI_SLOTS 102400 172 173 /* Each hpts has its own p_mtx which is used for locking */ 174 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) 175 #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx) 176 #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx) 177 struct tcp_hpts_entry { 178 /* Cache line 0x00 */ 179 struct mtx p_mtx; /* Mutex for hpts */ 180 struct timeval p_mysleep; /* Our min sleep time */ 181 uint64_t syscall_cnt; 182 uint64_t sleeping; /* What the actual sleep was (if sleeping) */ 183 uint16_t p_hpts_active; /* Flag that says hpts is awake */ 184 uint8_t p_wheel_complete; /* have we completed the wheel arc walk? */ 185 uint32_t p_curtick; /* Tick in 10 us the hpts is going to */ 186 uint32_t p_runningslot; /* Current tick we are at if we are running */ 187 uint32_t p_prev_slot; /* Previous slot we were on */ 188 uint32_t p_cur_slot; /* Current slot in wheel hpts is draining */ 189 uint32_t p_nxt_slot; /* The next slot outside the current range of 190 * slots that the hpts is running on. */ 191 int32_t p_on_queue_cnt; /* Count on queue in this hpts */ 192 uint32_t p_lasttick; /* Last tick before the current one */ 193 uint8_t p_direct_wake :1, /* boolean */ 194 p_on_min_sleep:1, /* boolean */ 195 p_hpts_wake_scheduled:1, /* boolean */ 196 p_avail:5; 197 uint8_t p_fill[3]; /* Fill to 32 bits */ 198 /* Cache line 0x40 */ 199 struct hptsh { 200 TAILQ_HEAD(, tcpcb) head; 201 uint32_t count; 202 uint32_t gencnt; 203 } *p_hptss; /* Hptsi wheel */ 204 uint32_t p_hpts_sleep_time; /* Current sleep interval having a max 205 * of 255ms */ 206 uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */ 207 uint32_t saved_lasttick; /* for logging */ 208 uint32_t saved_curtick; /* for logging */ 209 uint32_t saved_curslot; /* for logging */ 210 uint32_t saved_prev_slot; /* for logging */ 211 uint32_t p_delayed_by; /* How much were we delayed by */ 212 /* Cache line 0x80 */ 213 struct sysctl_ctx_list hpts_ctx; 214 struct sysctl_oid *hpts_root; 215 struct intr_event *ie; 216 void *ie_cookie; 217 uint16_t p_num; /* The hpts number one per cpu */ 218 uint16_t p_cpu; /* The hpts CPU */ 219 /* There is extra space in here */ 220 /* Cache line 0x100 */ 221 struct callout co __aligned(CACHE_LINE_SIZE); 222 } __aligned(CACHE_LINE_SIZE); 223 224 static struct tcp_hptsi { 225 struct cpu_group **grps; 226 struct tcp_hpts_entry **rp_ent; /* Array of hptss */ 227 uint32_t *cts_last_ran; 228 uint32_t grp_cnt; 229 uint32_t rp_num_hptss; /* Number of hpts threads */ 230 } tcp_pace; 231 232 static MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts"); 233 #ifdef RSS 234 static int tcp_bind_threads = 1; 235 #else 236 static int tcp_bind_threads = 2; 237 #endif 238 static int tcp_use_irq_cpu = 0; 239 static int hpts_does_tp_logging = 0; 240 241 static int32_t tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout); 242 static void tcp_hpts_thread(void *ctx); 243 244 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; 245 static int conn_cnt_thresh = DEFAULT_CONNECTION_THESHOLD; 246 static int32_t dynamic_min_sleep = DYNAMIC_MIN_SLEEP; 247 static int32_t dynamic_max_sleep = DYNAMIC_MAX_SLEEP; 248 249 250 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 251 "TCP Hpts controls"); 252 SYSCTL_NODE(_net_inet_tcp_hpts, OID_AUTO, stats, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 253 "TCP Hpts statistics"); 254 255 #define timersub(tvp, uvp, vvp) \ 256 do { \ 257 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ 258 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ 259 if ((vvp)->tv_usec < 0) { \ 260 (vvp)->tv_sec--; \ 261 (vvp)->tv_usec += 1000000; \ 262 } \ 263 } while (0) 264 265 static int32_t tcp_hpts_precision = 120; 266 267 static struct hpts_domain_info { 268 int count; 269 int cpu[MAXCPU]; 270 } hpts_domains[MAXMEMDOM]; 271 272 counter_u64_t hpts_hopelessly_behind; 273 274 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, 275 &hpts_hopelessly_behind, 276 "Number of times hpts could not catch up and was behind hopelessly"); 277 278 counter_u64_t hpts_loops; 279 280 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, loops, CTLFLAG_RD, 281 &hpts_loops, "Number of times hpts had to loop to catch up"); 282 283 counter_u64_t back_tosleep; 284 285 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, no_tcbsfound, CTLFLAG_RD, 286 &back_tosleep, "Number of times hpts found no tcbs"); 287 288 counter_u64_t combined_wheel_wrap; 289 290 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, comb_wheel_wrap, CTLFLAG_RD, 291 &combined_wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap"); 292 293 counter_u64_t wheel_wrap; 294 295 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, wheel_wrap, CTLFLAG_RD, 296 &wheel_wrap, "Number of times the wheel lagged enough to have an insert see wrap"); 297 298 counter_u64_t hpts_direct_call; 299 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, direct_call, CTLFLAG_RD, 300 &hpts_direct_call, "Number of times hpts was called by syscall/trap or other entry"); 301 302 counter_u64_t hpts_wake_timeout; 303 304 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, timeout_wakeup, CTLFLAG_RD, 305 &hpts_wake_timeout, "Number of times hpts threads woke up via the callout expiring"); 306 307 counter_u64_t hpts_direct_awakening; 308 309 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, direct_awakening, CTLFLAG_RD, 310 &hpts_direct_awakening, "Number of times hpts threads woke up via the callout expiring"); 311 312 counter_u64_t hpts_back_tosleep; 313 314 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, back_tosleep, CTLFLAG_RD, 315 &hpts_back_tosleep, "Number of times hpts threads woke up via the callout expiring and went back to sleep no work"); 316 317 counter_u64_t cpu_uses_flowid; 318 counter_u64_t cpu_uses_random; 319 320 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_flowid, CTLFLAG_RD, 321 &cpu_uses_flowid, "Number of times when setting cpuid we used the flowid field"); 322 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, cpusel_random, CTLFLAG_RD, 323 &cpu_uses_random, "Number of times when setting cpuid we used the a random value"); 324 325 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); 326 TUNABLE_INT("net.inet.tcp.use_irq", &tcp_use_irq_cpu); 327 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, bind_hptss, CTLFLAG_RD, 328 &tcp_bind_threads, 2, 329 "Thread Binding tunable"); 330 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, use_irq, CTLFLAG_RD, 331 &tcp_use_irq_cpu, 0, 332 "Use of irq CPU tunable"); 333 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, 334 &tcp_hpts_precision, 120, 335 "Value for PRE() precision of callout"); 336 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, cnt_thresh, CTLFLAG_RW, 337 &conn_cnt_thresh, 0, 338 "How many connections (below) make us use the callout based mechanism"); 339 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, 340 &hpts_does_tp_logging, 0, 341 "Do we add to any tp that has logging on pacer logs"); 342 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, dyn_minsleep, CTLFLAG_RW, 343 &dynamic_min_sleep, 250, 344 "What is the dynamic minsleep value?"); 345 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, dyn_maxsleep, CTLFLAG_RW, 346 &dynamic_max_sleep, 5000, 347 "What is the dynamic maxsleep value?"); 348 349 static int32_t max_pacer_loops = 10; 350 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, loopmax, CTLFLAG_RW, 351 &max_pacer_loops, 10, 352 "What is the maximum number of times the pacer will loop trying to catch up"); 353 354 #define HPTS_MAX_SLEEP_ALLOWED (NUM_OF_HPTSI_SLOTS/2) 355 356 static uint32_t hpts_sleep_max = HPTS_MAX_SLEEP_ALLOWED; 357 358 static int 359 sysctl_net_inet_tcp_hpts_max_sleep(SYSCTL_HANDLER_ARGS) 360 { 361 int error; 362 uint32_t new; 363 364 new = hpts_sleep_max; 365 error = sysctl_handle_int(oidp, &new, 0, req); 366 if (error == 0 && req->newptr) { 367 if ((new < (dynamic_min_sleep/HPTS_TICKS_PER_SLOT)) || 368 (new > HPTS_MAX_SLEEP_ALLOWED)) 369 error = EINVAL; 370 else 371 hpts_sleep_max = new; 372 } 373 return (error); 374 } 375 376 static int 377 sysctl_net_inet_tcp_hpts_min_sleep(SYSCTL_HANDLER_ARGS) 378 { 379 int error; 380 uint32_t new; 381 382 new = tcp_min_hptsi_time; 383 error = sysctl_handle_int(oidp, &new, 0, req); 384 if (error == 0 && req->newptr) { 385 if (new < LOWEST_SLEEP_ALLOWED) 386 error = EINVAL; 387 else 388 tcp_min_hptsi_time = new; 389 } 390 return (error); 391 } 392 393 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, maxsleep, 394 CTLTYPE_UINT | CTLFLAG_RW, 395 &hpts_sleep_max, 0, 396 &sysctl_net_inet_tcp_hpts_max_sleep, "IU", 397 "Maximum time hpts will sleep in slots"); 398 399 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, minsleep, 400 CTLTYPE_UINT | CTLFLAG_RW, 401 &tcp_min_hptsi_time, 0, 402 &sysctl_net_inet_tcp_hpts_min_sleep, "IU", 403 "The minimum time the hpts must sleep before processing more slots"); 404 405 static int ticks_indicate_more_sleep = TICKS_INDICATE_MORE_SLEEP; 406 static int ticks_indicate_less_sleep = TICKS_INDICATE_LESS_SLEEP; 407 static int tcp_hpts_no_wake_over_thresh = 1; 408 409 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, more_sleep, CTLFLAG_RW, 410 &ticks_indicate_more_sleep, 0, 411 "If we only process this many or less on a timeout, we need longer sleep on the next callout"); 412 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, less_sleep, CTLFLAG_RW, 413 &ticks_indicate_less_sleep, 0, 414 "If we process this many or more on a timeout, we need less sleep on the next callout"); 415 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, nowake_over_thresh, CTLFLAG_RW, 416 &tcp_hpts_no_wake_over_thresh, 0, 417 "When we are over the threshold on the pacer do we prohibit wakeups?"); 418 419 static uint16_t 420 hpts_random_cpu(void) 421 { 422 uint16_t cpuid; 423 uint32_t ran; 424 425 ran = arc4random(); 426 cpuid = (((ran & 0xffff) % mp_ncpus) % tcp_pace.rp_num_hptss); 427 return (cpuid); 428 } 429 430 static void 431 tcp_hpts_log(struct tcp_hpts_entry *hpts, struct tcpcb *tp, struct timeval *tv, 432 int slots_to_run, int idx, int from_callout) 433 { 434 union tcp_log_stackspecific log; 435 /* 436 * Unused logs are 437 * 64 bit - delRate, rttProp, bw_inuse 438 * 16 bit - cwnd_gain 439 * 8 bit - bbr_state, bbr_substate, inhpts; 440 */ 441 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 442 log.u_bbr.flex1 = hpts->p_nxt_slot; 443 log.u_bbr.flex2 = hpts->p_cur_slot; 444 log.u_bbr.flex3 = hpts->p_prev_slot; 445 log.u_bbr.flex4 = idx; 446 log.u_bbr.flex5 = hpts->p_curtick; 447 log.u_bbr.flex6 = hpts->p_on_queue_cnt; 448 log.u_bbr.flex7 = hpts->p_cpu; 449 log.u_bbr.flex8 = (uint8_t)from_callout; 450 log.u_bbr.inflight = slots_to_run; 451 log.u_bbr.applimited = hpts->overidden_sleep; 452 log.u_bbr.delivered = hpts->saved_curtick; 453 log.u_bbr.timeStamp = tcp_tv_to_usectick(tv); 454 log.u_bbr.epoch = hpts->saved_curslot; 455 log.u_bbr.lt_epoch = hpts->saved_prev_slot; 456 log.u_bbr.pkts_out = hpts->p_delayed_by; 457 log.u_bbr.lost = hpts->p_hpts_sleep_time; 458 log.u_bbr.pacing_gain = hpts->p_cpu; 459 log.u_bbr.pkt_epoch = hpts->p_runningslot; 460 log.u_bbr.use_lt_bw = 1; 461 TCP_LOG_EVENTP(tp, NULL, 462 &tptosocket(tp)->so_rcv, 463 &tptosocket(tp)->so_snd, 464 BBR_LOG_HPTSDIAG, 0, 465 0, &log, false, tv); 466 } 467 468 static void 469 tcp_wakehpts(struct tcp_hpts_entry *hpts) 470 { 471 HPTS_MTX_ASSERT(hpts); 472 473 if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) { 474 hpts->p_direct_wake = 0; 475 return; 476 } 477 if (hpts->p_hpts_wake_scheduled == 0) { 478 hpts->p_hpts_wake_scheduled = 1; 479 swi_sched(hpts->ie_cookie, 0); 480 } 481 } 482 483 static void 484 hpts_timeout_swi(void *arg) 485 { 486 struct tcp_hpts_entry *hpts; 487 488 hpts = (struct tcp_hpts_entry *)arg; 489 swi_sched(hpts->ie_cookie, 0); 490 } 491 492 static void 493 tcp_hpts_insert_internal(struct tcpcb *tp, struct tcp_hpts_entry *hpts) 494 { 495 struct inpcb *inp = tptoinpcb(tp); 496 struct hptsh *hptsh; 497 498 INP_WLOCK_ASSERT(inp); 499 HPTS_MTX_ASSERT(hpts); 500 MPASS(hpts->p_cpu == tp->t_hpts_cpu); 501 MPASS(!(inp->inp_flags & INP_DROPPED)); 502 503 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; 504 505 if (tp->t_in_hpts == IHPTS_NONE) { 506 tp->t_in_hpts = IHPTS_ONQUEUE; 507 in_pcbref(inp); 508 } else if (tp->t_in_hpts == IHPTS_MOVING) { 509 tp->t_in_hpts = IHPTS_ONQUEUE; 510 } else 511 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); 512 tp->t_hpts_gencnt = hptsh->gencnt; 513 514 TAILQ_INSERT_TAIL(&hptsh->head, tp, t_hpts); 515 hptsh->count++; 516 hpts->p_on_queue_cnt++; 517 } 518 519 static struct tcp_hpts_entry * 520 tcp_hpts_lock(struct tcpcb *tp) 521 { 522 struct tcp_hpts_entry *hpts; 523 524 INP_LOCK_ASSERT(tptoinpcb(tp)); 525 526 hpts = tcp_pace.rp_ent[tp->t_hpts_cpu]; 527 HPTS_LOCK(hpts); 528 529 return (hpts); 530 } 531 532 static void 533 tcp_hpts_release(struct tcpcb *tp) 534 { 535 bool released __diagused; 536 537 tp->t_in_hpts = IHPTS_NONE; 538 released = in_pcbrele_wlocked(tptoinpcb(tp)); 539 MPASS(released == false); 540 } 541 542 /* 543 * Initialize tcpcb to get ready for use with HPTS. We will know which CPU 544 * is preferred on the first incoming packet. Before that avoid crowding 545 * a single CPU with newborn connections and use a random one. 546 * This initialization is normally called on a newborn tcpcb, but potentially 547 * can be called once again if stack is switched. In that case we inherit CPU 548 * that the previous stack has set, be it random or not. In extreme cases, 549 * e.g. syzkaller fuzzing, a tcpcb can already be in HPTS in IHPTS_MOVING state 550 * and has never received a first packet. 551 */ 552 void 553 tcp_hpts_init(struct tcpcb *tp) 554 { 555 556 if (__predict_true(tp->t_hpts_cpu == HPTS_CPU_NONE)) { 557 tp->t_hpts_cpu = hpts_random_cpu(); 558 MPASS(!(tp->t_flags2 & TF2_HPTS_CPU_SET)); 559 } 560 } 561 562 /* 563 * Called normally with the INP_LOCKED but it 564 * does not matter, the hpts lock is the key 565 * but the lock order allows us to hold the 566 * INP lock and then get the hpts lock. 567 */ 568 void 569 tcp_hpts_remove(struct tcpcb *tp) 570 { 571 struct tcp_hpts_entry *hpts; 572 struct hptsh *hptsh; 573 574 INP_WLOCK_ASSERT(tptoinpcb(tp)); 575 576 hpts = tcp_hpts_lock(tp); 577 if (tp->t_in_hpts == IHPTS_ONQUEUE) { 578 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; 579 tp->t_hpts_request = 0; 580 if (__predict_true(tp->t_hpts_gencnt == hptsh->gencnt)) { 581 TAILQ_REMOVE(&hptsh->head, tp, t_hpts); 582 MPASS(hptsh->count > 0); 583 hptsh->count--; 584 MPASS(hpts->p_on_queue_cnt > 0); 585 hpts->p_on_queue_cnt--; 586 tcp_hpts_release(tp); 587 } else { 588 /* 589 * tcp_hptsi() now owns the TAILQ head of this inp. 590 * Can't TAILQ_REMOVE, just mark it. 591 */ 592 #ifdef INVARIANTS 593 struct tcpcb *tmp; 594 595 TAILQ_FOREACH(tmp, &hptsh->head, t_hpts) 596 MPASS(tmp != tp); 597 #endif 598 tp->t_in_hpts = IHPTS_MOVING; 599 tp->t_hpts_slot = -1; 600 } 601 } else if (tp->t_in_hpts == IHPTS_MOVING) { 602 /* 603 * Handle a special race condition: 604 * tcp_hptsi() moves inpcb to detached tailq 605 * tcp_hpts_remove() marks as IHPTS_MOVING, slot = -1 606 * tcp_hpts_insert() sets slot to a meaningful value 607 * tcp_hpts_remove() again (we are here!), then in_pcbdrop() 608 * tcp_hptsi() finds pcb with meaningful slot and INP_DROPPED 609 */ 610 tp->t_hpts_slot = -1; 611 } 612 HPTS_UNLOCK(hpts); 613 } 614 615 static inline int 616 hpts_slot(uint32_t wheel_slot, uint32_t plus) 617 { 618 /* 619 * Given a slot on the wheel, what slot 620 * is that plus ticks out? 621 */ 622 KASSERT(wheel_slot < NUM_OF_HPTSI_SLOTS, ("Invalid tick %u not on wheel", wheel_slot)); 623 return ((wheel_slot + plus) % NUM_OF_HPTSI_SLOTS); 624 } 625 626 static inline int 627 tick_to_wheel(uint32_t cts_in_wticks) 628 { 629 /* 630 * Given a timestamp in ticks (so by 631 * default to get it to a real time one 632 * would multiply by 10.. i.e the number 633 * of ticks in a slot) map it to our limited 634 * space wheel. 635 */ 636 return (cts_in_wticks % NUM_OF_HPTSI_SLOTS); 637 } 638 639 static inline int 640 hpts_slots_diff(int prev_slot, int slot_now) 641 { 642 /* 643 * Given two slots that are someplace 644 * on our wheel. How far are they apart? 645 */ 646 if (slot_now > prev_slot) 647 return (slot_now - prev_slot); 648 else if (slot_now == prev_slot) 649 /* 650 * Special case, same means we can go all of our 651 * wheel less one slot. 652 */ 653 return (NUM_OF_HPTSI_SLOTS - 1); 654 else 655 return ((NUM_OF_HPTSI_SLOTS - prev_slot) + slot_now); 656 } 657 658 /* 659 * Given a slot on the wheel that is the current time 660 * mapped to the wheel (wheel_slot), what is the maximum 661 * distance forward that can be obtained without 662 * wrapping past either prev_slot or running_slot 663 * depending on the htps state? Also if passed 664 * a uint32_t *, fill it with the slot location. 665 * 666 * Note if you do not give this function the current 667 * time (that you think it is) mapped to the wheel slot 668 * then the results will not be what you expect and 669 * could lead to invalid inserts. 670 */ 671 static inline int32_t 672 max_slots_available(struct tcp_hpts_entry *hpts, uint32_t wheel_slot, uint32_t *target_slot) 673 { 674 uint32_t dis_to_travel, end_slot, pacer_to_now, avail_on_wheel; 675 676 if ((hpts->p_hpts_active == 1) && 677 (hpts->p_wheel_complete == 0)) { 678 end_slot = hpts->p_runningslot; 679 /* Back up one tick */ 680 if (end_slot == 0) 681 end_slot = NUM_OF_HPTSI_SLOTS - 1; 682 else 683 end_slot--; 684 if (target_slot) 685 *target_slot = end_slot; 686 } else { 687 /* 688 * For the case where we are 689 * not active, or we have 690 * completed the pass over 691 * the wheel, we can use the 692 * prev tick and subtract one from it. This puts us 693 * as far out as possible on the wheel. 694 */ 695 end_slot = hpts->p_prev_slot; 696 if (end_slot == 0) 697 end_slot = NUM_OF_HPTSI_SLOTS - 1; 698 else 699 end_slot--; 700 if (target_slot) 701 *target_slot = end_slot; 702 /* 703 * Now we have close to the full wheel left minus the 704 * time it has been since the pacer went to sleep. Note 705 * that wheel_tick, passed in, should be the current time 706 * from the perspective of the caller, mapped to the wheel. 707 */ 708 if (hpts->p_prev_slot != wheel_slot) 709 dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); 710 else 711 dis_to_travel = 1; 712 /* 713 * dis_to_travel in this case is the space from when the 714 * pacer stopped (p_prev_slot) and where our wheel_slot 715 * is now. To know how many slots we can put it in we 716 * subtract from the wheel size. We would not want 717 * to place something after p_prev_slot or it will 718 * get ran too soon. 719 */ 720 return (NUM_OF_HPTSI_SLOTS - dis_to_travel); 721 } 722 /* 723 * So how many slots are open between p_runningslot -> p_cur_slot 724 * that is what is currently un-available for insertion. Special 725 * case when we are at the last slot, this gets 1, so that 726 * the answer to how many slots are available is all but 1. 727 */ 728 if (hpts->p_runningslot == hpts->p_cur_slot) 729 dis_to_travel = 1; 730 else 731 dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); 732 /* 733 * How long has the pacer been running? 734 */ 735 if (hpts->p_cur_slot != wheel_slot) { 736 /* The pacer is a bit late */ 737 pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot); 738 } else { 739 /* The pacer is right on time, now == pacers start time */ 740 pacer_to_now = 0; 741 } 742 /* 743 * To get the number left we can insert into we simply 744 * subtract the distance the pacer has to run from how 745 * many slots there are. 746 */ 747 avail_on_wheel = NUM_OF_HPTSI_SLOTS - dis_to_travel; 748 /* 749 * Now how many of those we will eat due to the pacer's 750 * time (p_cur_slot) of start being behind the 751 * real time (wheel_slot)? 752 */ 753 if (avail_on_wheel <= pacer_to_now) { 754 /* 755 * Wheel wrap, we can't fit on the wheel, that 756 * is unusual the system must be way overloaded! 757 * Insert into the assured slot, and return special 758 * "0". 759 */ 760 counter_u64_add(combined_wheel_wrap, 1); 761 *target_slot = hpts->p_nxt_slot; 762 return (0); 763 } else { 764 /* 765 * We know how many slots are open 766 * on the wheel (the reverse of what 767 * is left to run. Take away the time 768 * the pacer started to now (wheel_slot) 769 * and that tells you how many slots are 770 * open that can be inserted into that won't 771 * be touched by the pacer until later. 772 */ 773 return (avail_on_wheel - pacer_to_now); 774 } 775 } 776 777 778 #ifdef INVARIANTS 779 static void 780 check_if_slot_would_be_wrong(struct tcp_hpts_entry *hpts, struct tcpcb *tp, 781 uint32_t hptsslot, int line) 782 { 783 /* 784 * Sanity checks for the pacer with invariants 785 * on insert. 786 */ 787 KASSERT(hptsslot < NUM_OF_HPTSI_SLOTS, 788 ("hpts:%p tp:%p slot:%d > max", hpts, tp, hptsslot)); 789 if ((hpts->p_hpts_active) && 790 (hpts->p_wheel_complete == 0)) { 791 /* 792 * If the pacer is processing a arc 793 * of the wheel, we need to make 794 * sure we are not inserting within 795 * that arc. 796 */ 797 int distance, yet_to_run; 798 799 distance = hpts_slots_diff(hpts->p_runningslot, hptsslot); 800 if (hpts->p_runningslot != hpts->p_cur_slot) 801 yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); 802 else 803 yet_to_run = 0; /* processing last slot */ 804 KASSERT(yet_to_run <= distance, ("hpts:%p tp:%p slot:%d " 805 "distance:%d yet_to_run:%d rs:%d cs:%d", hpts, tp, 806 hptsslot, distance, yet_to_run, hpts->p_runningslot, 807 hpts->p_cur_slot)); 808 } 809 } 810 #endif 811 812 uint32_t 813 tcp_hpts_insert_diag(struct tcpcb *tp, uint32_t slot, int32_t line, struct hpts_diag *diag) 814 { 815 struct tcp_hpts_entry *hpts; 816 struct timeval tv; 817 uint32_t slot_on, wheel_cts, last_slot, need_new_to = 0; 818 int32_t wheel_slot, maxslots; 819 bool need_wakeup = false; 820 821 INP_WLOCK_ASSERT(tptoinpcb(tp)); 822 MPASS(!(tptoinpcb(tp)->inp_flags & INP_DROPPED)); 823 MPASS(!tcp_in_hpts(tp)); 824 825 /* 826 * We now return the next-slot the hpts will be on, beyond its 827 * current run (if up) or where it was when it stopped if it is 828 * sleeping. 829 */ 830 hpts = tcp_hpts_lock(tp); 831 microuptime(&tv); 832 if (diag) { 833 memset(diag, 0, sizeof(struct hpts_diag)); 834 diag->p_hpts_active = hpts->p_hpts_active; 835 diag->p_prev_slot = hpts->p_prev_slot; 836 diag->p_runningslot = hpts->p_runningslot; 837 diag->p_nxt_slot = hpts->p_nxt_slot; 838 diag->p_cur_slot = hpts->p_cur_slot; 839 diag->p_curtick = hpts->p_curtick; 840 diag->p_lasttick = hpts->p_lasttick; 841 diag->slot_req = slot; 842 diag->p_on_min_sleep = hpts->p_on_min_sleep; 843 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; 844 } 845 if (slot == 0) { 846 /* Ok we need to set it on the hpts in the current slot */ 847 tp->t_hpts_request = 0; 848 if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) { 849 /* 850 * A sleeping hpts we want in next slot to run 851 * note that in this state p_prev_slot == p_cur_slot 852 */ 853 tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1); 854 if ((hpts->p_on_min_sleep == 0) && 855 (hpts->p_hpts_active == 0)) 856 need_wakeup = true; 857 } else 858 tp->t_hpts_slot = hpts->p_runningslot; 859 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) 860 tcp_hpts_insert_internal(tp, hpts); 861 if (need_wakeup) { 862 /* 863 * Activate the hpts if it is sleeping and its 864 * timeout is not 1. 865 */ 866 hpts->p_direct_wake = 1; 867 tcp_wakehpts(hpts); 868 } 869 slot_on = hpts->p_nxt_slot; 870 HPTS_UNLOCK(hpts); 871 872 return (slot_on); 873 } 874 /* Get the current time relative to the wheel */ 875 wheel_cts = tcp_tv_to_hptstick(&tv); 876 /* Map it onto the wheel */ 877 wheel_slot = tick_to_wheel(wheel_cts); 878 /* Now what's the max we can place it at? */ 879 maxslots = max_slots_available(hpts, wheel_slot, &last_slot); 880 if (diag) { 881 diag->wheel_slot = wheel_slot; 882 diag->maxslots = maxslots; 883 diag->wheel_cts = wheel_cts; 884 } 885 if (maxslots == 0) { 886 /* The pacer is in a wheel wrap behind, yikes! */ 887 if (slot > 1) { 888 /* 889 * Reduce by 1 to prevent a forever loop in 890 * case something else is wrong. Note this 891 * probably does not hurt because the pacer 892 * if its true is so far behind we will be 893 * > 1second late calling anyway. 894 */ 895 slot--; 896 } 897 tp->t_hpts_slot = last_slot; 898 tp->t_hpts_request = slot; 899 } else if (maxslots >= slot) { 900 /* It all fits on the wheel */ 901 tp->t_hpts_request = 0; 902 tp->t_hpts_slot = hpts_slot(wheel_slot, slot); 903 } else { 904 /* It does not fit */ 905 tp->t_hpts_request = slot - maxslots; 906 tp->t_hpts_slot = last_slot; 907 } 908 if (diag) { 909 diag->slot_remaining = tp->t_hpts_request; 910 diag->inp_hptsslot = tp->t_hpts_slot; 911 } 912 #ifdef INVARIANTS 913 check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line); 914 #endif 915 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) 916 tcp_hpts_insert_internal(tp, hpts); 917 if ((hpts->p_hpts_active == 0) && 918 (tp->t_hpts_request == 0) && 919 (hpts->p_on_min_sleep == 0)) { 920 /* 921 * The hpts is sleeping and NOT on a minimum 922 * sleep time, we need to figure out where 923 * it will wake up at and if we need to reschedule 924 * its time-out. 925 */ 926 uint32_t have_slept, yet_to_sleep; 927 928 /* Now do we need to restart the hpts's timer? */ 929 have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); 930 if (have_slept < hpts->p_hpts_sleep_time) 931 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; 932 else { 933 /* We are over-due */ 934 yet_to_sleep = 0; 935 need_wakeup = 1; 936 } 937 if (diag) { 938 diag->have_slept = have_slept; 939 diag->yet_to_sleep = yet_to_sleep; 940 } 941 if (yet_to_sleep && 942 (yet_to_sleep > slot)) { 943 /* 944 * We need to reschedule the hpts's time-out. 945 */ 946 hpts->p_hpts_sleep_time = slot; 947 need_new_to = slot * HPTS_TICKS_PER_SLOT; 948 } 949 } 950 /* 951 * Now how far is the hpts sleeping to? if active is 1, its 952 * up and ticking we do nothing, otherwise we may need to 953 * reschedule its callout if need_new_to is set from above. 954 */ 955 if (need_wakeup) { 956 hpts->p_direct_wake = 1; 957 tcp_wakehpts(hpts); 958 if (diag) { 959 diag->need_new_to = 0; 960 diag->co_ret = 0xffff0000; 961 } 962 } else if (need_new_to) { 963 int32_t co_ret; 964 struct timeval tv; 965 sbintime_t sb; 966 967 tv.tv_sec = 0; 968 tv.tv_usec = 0; 969 while (need_new_to > HPTS_USEC_IN_SEC) { 970 tv.tv_sec++; 971 need_new_to -= HPTS_USEC_IN_SEC; 972 } 973 tv.tv_usec = need_new_to; 974 sb = tvtosbt(tv); 975 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, 976 hpts_timeout_swi, hpts, hpts->p_cpu, 977 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 978 if (diag) { 979 diag->need_new_to = need_new_to; 980 diag->co_ret = co_ret; 981 } 982 } 983 slot_on = hpts->p_nxt_slot; 984 HPTS_UNLOCK(hpts); 985 986 return (slot_on); 987 } 988 989 static uint16_t 990 hpts_cpuid(struct tcpcb *tp, int *failed) 991 { 992 struct inpcb *inp = tptoinpcb(tp); 993 u_int cpuid; 994 #ifdef NUMA 995 struct hpts_domain_info *di; 996 #endif 997 998 *failed = 0; 999 if (tp->t_flags2 & TF2_HPTS_CPU_SET) { 1000 return (tp->t_hpts_cpu); 1001 } 1002 /* 1003 * If we are using the irq cpu set by LRO or 1004 * the driver then it overrides all other domains. 1005 */ 1006 if (tcp_use_irq_cpu) { 1007 if (tp->t_lro_cpu == HPTS_CPU_NONE) { 1008 *failed = 1; 1009 return (0); 1010 } 1011 return (tp->t_lro_cpu); 1012 } 1013 /* If one is set the other must be the same */ 1014 #ifdef RSS 1015 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); 1016 if (cpuid == NETISR_CPUID_NONE) 1017 return (hpts_random_cpu()); 1018 else 1019 return (cpuid); 1020 #endif 1021 /* 1022 * We don't have a flowid -> cpuid mapping, so cheat and just map 1023 * unknown cpuids to curcpu. Not the best, but apparently better 1024 * than defaulting to swi 0. 1025 */ 1026 if (inp->inp_flowtype == M_HASHTYPE_NONE) { 1027 counter_u64_add(cpu_uses_random, 1); 1028 return (hpts_random_cpu()); 1029 } 1030 /* 1031 * Hash to a thread based on the flowid. If we are using numa, 1032 * then restrict the hash to the numa domain where the inp lives. 1033 */ 1034 1035 #ifdef NUMA 1036 if ((vm_ndomains == 1) || 1037 (inp->inp_numa_domain == M_NODOM)) { 1038 #endif 1039 cpuid = inp->inp_flowid % mp_ncpus; 1040 #ifdef NUMA 1041 } else { 1042 /* Hash into the cpu's that use that domain */ 1043 di = &hpts_domains[inp->inp_numa_domain]; 1044 cpuid = di->cpu[inp->inp_flowid % di->count]; 1045 } 1046 #endif 1047 counter_u64_add(cpu_uses_flowid, 1); 1048 return (cpuid); 1049 } 1050 1051 static void 1052 tcp_hpts_set_max_sleep(struct tcp_hpts_entry *hpts, int wrap_loop_cnt) 1053 { 1054 uint32_t t = 0, i; 1055 1056 if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) { 1057 /* 1058 * Find next slot that is occupied and use that to 1059 * be the sleep time. 1060 */ 1061 for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) { 1062 if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) { 1063 break; 1064 } 1065 t = (t + 1) % NUM_OF_HPTSI_SLOTS; 1066 } 1067 KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt)); 1068 hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max); 1069 } else { 1070 /* No one on the wheel sleep for all but 400 slots or sleep max */ 1071 hpts->p_hpts_sleep_time = hpts_sleep_max; 1072 } 1073 } 1074 1075 static int32_t 1076 tcp_hptsi(struct tcp_hpts_entry *hpts, int from_callout) 1077 { 1078 struct tcpcb *tp; 1079 struct timeval tv; 1080 int32_t slots_to_run, i, error; 1081 int32_t loop_cnt = 0; 1082 int32_t did_prefetch = 0; 1083 int32_t prefetch_tp = 0; 1084 int32_t wrap_loop_cnt = 0; 1085 int32_t slot_pos_of_endpoint = 0; 1086 int32_t orig_exit_slot; 1087 int8_t completed_measure = 0, seen_endpoint = 0; 1088 1089 HPTS_MTX_ASSERT(hpts); 1090 NET_EPOCH_ASSERT(); 1091 /* record previous info for any logging */ 1092 hpts->saved_lasttick = hpts->p_lasttick; 1093 hpts->saved_curtick = hpts->p_curtick; 1094 hpts->saved_curslot = hpts->p_cur_slot; 1095 hpts->saved_prev_slot = hpts->p_prev_slot; 1096 1097 hpts->p_lasttick = hpts->p_curtick; 1098 hpts->p_curtick = tcp_gethptstick(&tv); 1099 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); 1100 orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); 1101 if ((hpts->p_on_queue_cnt == 0) || 1102 (hpts->p_lasttick == hpts->p_curtick)) { 1103 /* 1104 * No time has yet passed, 1105 * or nothing to do. 1106 */ 1107 hpts->p_prev_slot = hpts->p_cur_slot; 1108 hpts->p_lasttick = hpts->p_curtick; 1109 goto no_run; 1110 } 1111 again: 1112 hpts->p_wheel_complete = 0; 1113 HPTS_MTX_ASSERT(hpts); 1114 slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot); 1115 if (((hpts->p_curtick - hpts->p_lasttick) > 1116 ((NUM_OF_HPTSI_SLOTS-1) * HPTS_TICKS_PER_SLOT)) && 1117 (hpts->p_on_queue_cnt != 0)) { 1118 /* 1119 * Wheel wrap is occuring, basically we 1120 * are behind and the distance between 1121 * run's has spread so much it has exceeded 1122 * the time on the wheel (1.024 seconds). This 1123 * is ugly and should NOT be happening. We 1124 * need to run the entire wheel. We last processed 1125 * p_prev_slot, so that needs to be the last slot 1126 * we run. The next slot after that should be our 1127 * reserved first slot for new, and then starts 1128 * the running position. Now the problem is the 1129 * reserved "not to yet" place does not exist 1130 * and there may be inp's in there that need 1131 * running. We can merge those into the 1132 * first slot at the head. 1133 */ 1134 wrap_loop_cnt++; 1135 hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1); 1136 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2); 1137 /* 1138 * Adjust p_cur_slot to be where we are starting from 1139 * hopefully we will catch up (fat chance if something 1140 * is broken this bad :( ) 1141 */ 1142 hpts->p_cur_slot = hpts->p_prev_slot; 1143 /* 1144 * The next slot has guys to run too, and that would 1145 * be where we would normally start, lets move them into 1146 * the next slot (p_prev_slot + 2) so that we will 1147 * run them, the extra 10usecs of late (by being 1148 * put behind) does not really matter in this situation. 1149 */ 1150 TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head, 1151 t_hpts) { 1152 MPASS(tp->t_hpts_slot == hpts->p_nxt_slot); 1153 MPASS(tp->t_hpts_gencnt == 1154 hpts->p_hptss[hpts->p_nxt_slot].gencnt); 1155 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); 1156 1157 /* 1158 * Update gencnt and nextslot accordingly to match 1159 * the new location. This is safe since it takes both 1160 * the INP lock and the pacer mutex to change the 1161 * t_hptsslot and t_hpts_gencnt. 1162 */ 1163 tp->t_hpts_gencnt = 1164 hpts->p_hptss[hpts->p_runningslot].gencnt; 1165 tp->t_hpts_slot = hpts->p_runningslot; 1166 } 1167 TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head, 1168 &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts); 1169 hpts->p_hptss[hpts->p_runningslot].count += 1170 hpts->p_hptss[hpts->p_nxt_slot].count; 1171 hpts->p_hptss[hpts->p_nxt_slot].count = 0; 1172 hpts->p_hptss[hpts->p_nxt_slot].gencnt++; 1173 slots_to_run = NUM_OF_HPTSI_SLOTS - 1; 1174 counter_u64_add(wheel_wrap, 1); 1175 } else { 1176 /* 1177 * Nxt slot is always one after p_runningslot though 1178 * its not used usually unless we are doing wheel wrap. 1179 */ 1180 hpts->p_nxt_slot = hpts->p_prev_slot; 1181 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1); 1182 } 1183 if (hpts->p_on_queue_cnt == 0) { 1184 goto no_one; 1185 } 1186 for (i = 0; i < slots_to_run; i++) { 1187 struct tcpcb *tp, *ntp; 1188 TAILQ_HEAD(, tcpcb) head = TAILQ_HEAD_INITIALIZER(head); 1189 struct hptsh *hptsh; 1190 uint32_t runningslot; 1191 1192 /* 1193 * Calculate our delay, if there are no extra ticks there 1194 * was not any (i.e. if slots_to_run == 1, no delay). 1195 */ 1196 hpts->p_delayed_by = (slots_to_run - (i + 1)) * 1197 HPTS_TICKS_PER_SLOT; 1198 1199 runningslot = hpts->p_runningslot; 1200 hptsh = &hpts->p_hptss[runningslot]; 1201 TAILQ_SWAP(&head, &hptsh->head, tcpcb, t_hpts); 1202 hpts->p_on_queue_cnt -= hptsh->count; 1203 hptsh->count = 0; 1204 hptsh->gencnt++; 1205 1206 HPTS_UNLOCK(hpts); 1207 1208 TAILQ_FOREACH_SAFE(tp, &head, t_hpts, ntp) { 1209 struct inpcb *inp = tptoinpcb(tp); 1210 bool set_cpu; 1211 1212 if (ntp != NULL) { 1213 /* 1214 * If we have a next tcpcb, see if we can 1215 * prefetch it. Note this may seem 1216 * "risky" since we have no locks (other 1217 * than the previous inp) and there no 1218 * assurance that ntp was not pulled while 1219 * we were processing tp and freed. If this 1220 * occurred it could mean that either: 1221 * 1222 * a) Its NULL (which is fine we won't go 1223 * here) <or> b) Its valid (which is cool we 1224 * will prefetch it) <or> c) The inp got 1225 * freed back to the slab which was 1226 * reallocated. Then the piece of memory was 1227 * re-used and something else (not an 1228 * address) is in inp_ppcb. If that occurs 1229 * we don't crash, but take a TLB shootdown 1230 * performance hit (same as if it was NULL 1231 * and we tried to pre-fetch it). 1232 * 1233 * Considering that the likelyhood of <c> is 1234 * quite rare we will take a risk on doing 1235 * this. If performance drops after testing 1236 * we can always take this out. NB: the 1237 * kern_prefetch on amd64 actually has 1238 * protection against a bad address now via 1239 * the DMAP_() tests. This will prevent the 1240 * TLB hit, and instead if <c> occurs just 1241 * cause us to load cache with a useless 1242 * address (to us). 1243 * 1244 * XXXGL: this comment and the prefetch action 1245 * could be outdated after tp == inp change. 1246 */ 1247 kern_prefetch(ntp, &prefetch_tp); 1248 prefetch_tp = 1; 1249 } 1250 1251 /* For debugging */ 1252 if (seen_endpoint == 0) { 1253 seen_endpoint = 1; 1254 orig_exit_slot = slot_pos_of_endpoint = 1255 runningslot; 1256 } else if (completed_measure == 0) { 1257 /* Record the new position */ 1258 orig_exit_slot = runningslot; 1259 } 1260 1261 INP_WLOCK(inp); 1262 if ((tp->t_flags2 & TF2_HPTS_CPU_SET) == 0) { 1263 set_cpu = true; 1264 } else { 1265 set_cpu = false; 1266 } 1267 1268 if (__predict_false(tp->t_in_hpts == IHPTS_MOVING)) { 1269 if (tp->t_hpts_slot == -1) { 1270 tp->t_in_hpts = IHPTS_NONE; 1271 if (in_pcbrele_wlocked(inp) == false) 1272 INP_WUNLOCK(inp); 1273 } else { 1274 HPTS_LOCK(hpts); 1275 tcp_hpts_insert_internal(tp, hpts); 1276 HPTS_UNLOCK(hpts); 1277 INP_WUNLOCK(inp); 1278 } 1279 continue; 1280 } 1281 1282 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); 1283 MPASS(!(inp->inp_flags & INP_DROPPED)); 1284 KASSERT(runningslot == tp->t_hpts_slot, 1285 ("Hpts:%p inp:%p slot mis-aligned %u vs %u", 1286 hpts, inp, runningslot, tp->t_hpts_slot)); 1287 1288 if (tp->t_hpts_request) { 1289 /* 1290 * This guy is deferred out further in time 1291 * then our wheel had available on it. 1292 * Push him back on the wheel or run it 1293 * depending. 1294 */ 1295 uint32_t maxslots, last_slot, remaining_slots; 1296 1297 remaining_slots = slots_to_run - (i + 1); 1298 if (tp->t_hpts_request > remaining_slots) { 1299 HPTS_LOCK(hpts); 1300 /* 1301 * How far out can we go? 1302 */ 1303 maxslots = max_slots_available(hpts, 1304 hpts->p_cur_slot, &last_slot); 1305 if (maxslots >= tp->t_hpts_request) { 1306 /* We can place it finally to 1307 * be processed. */ 1308 tp->t_hpts_slot = hpts_slot( 1309 hpts->p_runningslot, 1310 tp->t_hpts_request); 1311 tp->t_hpts_request = 0; 1312 } else { 1313 /* Work off some more time */ 1314 tp->t_hpts_slot = last_slot; 1315 tp->t_hpts_request -= 1316 maxslots; 1317 } 1318 tcp_hpts_insert_internal(tp, hpts); 1319 HPTS_UNLOCK(hpts); 1320 INP_WUNLOCK(inp); 1321 continue; 1322 } 1323 tp->t_hpts_request = 0; 1324 /* Fall through we will so do it now */ 1325 } 1326 1327 tcp_hpts_release(tp); 1328 if (set_cpu) { 1329 /* 1330 * Setup so the next time we will move to 1331 * the right CPU. This should be a rare 1332 * event. It will sometimes happens when we 1333 * are the client side (usually not the 1334 * server). Somehow tcp_output() gets called 1335 * before the tcp_do_segment() sets the 1336 * intial state. This means the r_cpu and 1337 * r_hpts_cpu is 0. We get on the hpts, and 1338 * then tcp_input() gets called setting up 1339 * the r_cpu to the correct value. The hpts 1340 * goes off and sees the mis-match. We 1341 * simply correct it here and the CPU will 1342 * switch to the new hpts nextime the tcb 1343 * gets added to the hpts (not this one) 1344 * :-) 1345 */ 1346 tcp_set_hpts(tp); 1347 } 1348 CURVNET_SET(inp->inp_vnet); 1349 /* Lets do any logging that we might want to */ 1350 if (hpts_does_tp_logging && tcp_bblogging_on(tp)) { 1351 tcp_hpts_log(hpts, tp, &tv, slots_to_run, i, from_callout); 1352 } 1353 1354 if (tp->t_fb_ptr != NULL) { 1355 kern_prefetch(tp->t_fb_ptr, &did_prefetch); 1356 did_prefetch = 1; 1357 } 1358 /* 1359 * We set TF2_HPTS_CALLS before any possible output. 1360 * The contract with the transport is that if it cares 1361 * about hpts calling it should clear the flag. That 1362 * way next time it is called it will know it is hpts. 1363 * 1364 * We also only call tfb_do_queued_segments() <or> 1365 * tcp_output(). It is expected that if segments are 1366 * queued and come in that the final input mbuf will 1367 * cause a call to output if it is needed. 1368 */ 1369 tp->t_flags2 |= TF2_HPTS_CALLS; 1370 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) && 1371 !STAILQ_EMPTY(&tp->t_inqueue)) { 1372 error = (*tp->t_fb->tfb_do_queued_segments)(tp, 0); 1373 if (error) { 1374 /* The input killed the connection */ 1375 goto skip_pacing; 1376 } 1377 } 1378 error = tcp_output(tp); 1379 if (error < 0) 1380 goto skip_pacing; 1381 INP_WUNLOCK(inp); 1382 skip_pacing: 1383 CURVNET_RESTORE(); 1384 } 1385 if (seen_endpoint) { 1386 /* 1387 * We now have a accurate distance between 1388 * slot_pos_of_endpoint <-> orig_exit_slot 1389 * to tell us how late we were, orig_exit_slot 1390 * is where we calculated the end of our cycle to 1391 * be when we first entered. 1392 */ 1393 completed_measure = 1; 1394 } 1395 HPTS_LOCK(hpts); 1396 hpts->p_runningslot++; 1397 if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) { 1398 hpts->p_runningslot = 0; 1399 } 1400 } 1401 no_one: 1402 HPTS_MTX_ASSERT(hpts); 1403 hpts->p_delayed_by = 0; 1404 /* 1405 * Check to see if we took an excess amount of time and need to run 1406 * more ticks (if we did not hit eno-bufs). 1407 */ 1408 hpts->p_prev_slot = hpts->p_cur_slot; 1409 hpts->p_lasttick = hpts->p_curtick; 1410 if ((from_callout == 0) || (loop_cnt > max_pacer_loops)) { 1411 /* 1412 * Something is serious slow we have 1413 * looped through processing the wheel 1414 * and by the time we cleared the 1415 * needs to run max_pacer_loops time 1416 * we still needed to run. That means 1417 * the system is hopelessly behind and 1418 * can never catch up :( 1419 * 1420 * We will just lie to this thread 1421 * and let it thing p_curtick is 1422 * correct. When it next awakens 1423 * it will find itself further behind. 1424 */ 1425 if (from_callout) 1426 counter_u64_add(hpts_hopelessly_behind, 1); 1427 goto no_run; 1428 } 1429 hpts->p_curtick = tcp_gethptstick(&tv); 1430 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); 1431 if (seen_endpoint == 0) { 1432 /* We saw no endpoint but we may be looping */ 1433 orig_exit_slot = hpts->p_cur_slot; 1434 } 1435 if ((wrap_loop_cnt < 2) && 1436 (hpts->p_lasttick != hpts->p_curtick)) { 1437 counter_u64_add(hpts_loops, 1); 1438 loop_cnt++; 1439 goto again; 1440 } 1441 no_run: 1442 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); 1443 /* 1444 * Set flag to tell that we are done for 1445 * any slot input that happens during 1446 * input. 1447 */ 1448 hpts->p_wheel_complete = 1; 1449 /* 1450 * Now did we spend too long running input and need to run more ticks? 1451 * Note that if wrap_loop_cnt < 2 then we should have the conditions 1452 * in the KASSERT's true. But if the wheel is behind i.e. wrap_loop_cnt 1453 * is greater than 2, then the condtion most likely are *not* true. 1454 * Also if we are called not from the callout, we don't run the wheel 1455 * multiple times so the slots may not align either. 1456 */ 1457 KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) || 1458 (wrap_loop_cnt >= 2) || (from_callout == 0)), 1459 ("H:%p p_prev_slot:%u not equal to p_cur_slot:%u", hpts, 1460 hpts->p_prev_slot, hpts->p_cur_slot)); 1461 KASSERT(((hpts->p_lasttick == hpts->p_curtick) 1462 || (wrap_loop_cnt >= 2) || (from_callout == 0)), 1463 ("H:%p p_lasttick:%u not equal to p_curtick:%u", hpts, 1464 hpts->p_lasttick, hpts->p_curtick)); 1465 if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) { 1466 hpts->p_curtick = tcp_gethptstick(&tv); 1467 counter_u64_add(hpts_loops, 1); 1468 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); 1469 goto again; 1470 } 1471 1472 if (from_callout){ 1473 tcp_hpts_set_max_sleep(hpts, wrap_loop_cnt); 1474 } 1475 if (seen_endpoint) 1476 return(hpts_slots_diff(slot_pos_of_endpoint, orig_exit_slot)); 1477 else 1478 return (0); 1479 } 1480 1481 void 1482 __tcp_set_hpts(struct tcpcb *tp, int32_t line) 1483 { 1484 struct tcp_hpts_entry *hpts; 1485 int failed; 1486 1487 INP_WLOCK_ASSERT(tptoinpcb(tp)); 1488 1489 hpts = tcp_hpts_lock(tp); 1490 if (tp->t_in_hpts == IHPTS_NONE && !(tp->t_flags2 & TF2_HPTS_CPU_SET)) { 1491 tp->t_hpts_cpu = hpts_cpuid(tp, &failed); 1492 if (failed == 0) 1493 tp->t_flags2 |= TF2_HPTS_CPU_SET; 1494 } 1495 mtx_unlock(&hpts->p_mtx); 1496 } 1497 1498 static struct tcp_hpts_entry * 1499 tcp_choose_hpts_to_run(void) 1500 { 1501 int i, oldest_idx, start, end; 1502 uint32_t cts, time_since_ran, calc; 1503 1504 cts = tcp_get_usecs(NULL); 1505 time_since_ran = 0; 1506 /* Default is all one group */ 1507 start = 0; 1508 end = tcp_pace.rp_num_hptss; 1509 /* 1510 * If we have more than one L3 group figure out which one 1511 * this CPU is in. 1512 */ 1513 if (tcp_pace.grp_cnt > 1) { 1514 for (i = 0; i < tcp_pace.grp_cnt; i++) { 1515 if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) { 1516 start = tcp_pace.grps[i]->cg_first; 1517 end = (tcp_pace.grps[i]->cg_last + 1); 1518 break; 1519 } 1520 } 1521 } 1522 oldest_idx = -1; 1523 for (i = start; i < end; i++) { 1524 if (TSTMP_GT(cts, tcp_pace.cts_last_ran[i])) 1525 calc = cts - tcp_pace.cts_last_ran[i]; 1526 else 1527 calc = 0; 1528 if (calc > time_since_ran) { 1529 oldest_idx = i; 1530 time_since_ran = calc; 1531 } 1532 } 1533 if (oldest_idx >= 0) 1534 return(tcp_pace.rp_ent[oldest_idx]); 1535 else 1536 return(tcp_pace.rp_ent[(curcpu % tcp_pace.rp_num_hptss)]); 1537 } 1538 1539 static void 1540 __tcp_run_hpts(void) 1541 { 1542 struct epoch_tracker et; 1543 struct tcp_hpts_entry *hpts; 1544 int ticks_ran; 1545 1546 hpts = tcp_choose_hpts_to_run(); 1547 1548 if (hpts->p_hpts_active) { 1549 /* Already active */ 1550 return; 1551 } 1552 if (mtx_trylock(&hpts->p_mtx) == 0) { 1553 /* Someone else got the lock */ 1554 return; 1555 } 1556 NET_EPOCH_ENTER(et); 1557 if (hpts->p_hpts_active) 1558 goto out_with_mtx; 1559 hpts->syscall_cnt++; 1560 counter_u64_add(hpts_direct_call, 1); 1561 hpts->p_hpts_active = 1; 1562 ticks_ran = tcp_hptsi(hpts, 0); 1563 /* We may want to adjust the sleep values here */ 1564 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { 1565 if (ticks_ran > ticks_indicate_less_sleep) { 1566 struct timeval tv; 1567 sbintime_t sb; 1568 1569 hpts->p_mysleep.tv_usec /= 2; 1570 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) 1571 hpts->p_mysleep.tv_usec = dynamic_min_sleep; 1572 /* Reschedule with new to value */ 1573 tcp_hpts_set_max_sleep(hpts, 0); 1574 tv.tv_sec = 0; 1575 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; 1576 /* Validate its in the right ranges */ 1577 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { 1578 hpts->overidden_sleep = tv.tv_usec; 1579 tv.tv_usec = hpts->p_mysleep.tv_usec; 1580 } else if (tv.tv_usec > dynamic_max_sleep) { 1581 /* Lets not let sleep get above this value */ 1582 hpts->overidden_sleep = tv.tv_usec; 1583 tv.tv_usec = dynamic_max_sleep; 1584 } 1585 /* 1586 * In this mode the timer is a backstop to 1587 * all the userret/lro_flushes so we use 1588 * the dynamic value and set the on_min_sleep 1589 * flag so we will not be awoken. 1590 */ 1591 sb = tvtosbt(tv); 1592 /* Store off to make visible the actual sleep time */ 1593 hpts->sleeping = tv.tv_usec; 1594 callout_reset_sbt_on(&hpts->co, sb, 0, 1595 hpts_timeout_swi, hpts, hpts->p_cpu, 1596 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1597 } else if (ticks_ran < ticks_indicate_more_sleep) { 1598 /* For the further sleep, don't reschedule hpts */ 1599 hpts->p_mysleep.tv_usec *= 2; 1600 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) 1601 hpts->p_mysleep.tv_usec = dynamic_max_sleep; 1602 } 1603 hpts->p_on_min_sleep = 1; 1604 } 1605 hpts->p_hpts_active = 0; 1606 out_with_mtx: 1607 HPTS_MTX_ASSERT(hpts); 1608 mtx_unlock(&hpts->p_mtx); 1609 NET_EPOCH_EXIT(et); 1610 } 1611 1612 static void 1613 tcp_hpts_thread(void *ctx) 1614 { 1615 struct tcp_hpts_entry *hpts; 1616 struct epoch_tracker et; 1617 struct timeval tv; 1618 sbintime_t sb; 1619 int ticks_ran; 1620 1621 hpts = (struct tcp_hpts_entry *)ctx; 1622 mtx_lock(&hpts->p_mtx); 1623 if (hpts->p_direct_wake) { 1624 /* Signaled by input or output with low occupancy count. */ 1625 callout_stop(&hpts->co); 1626 counter_u64_add(hpts_direct_awakening, 1); 1627 } else { 1628 /* Timed out, the normal case. */ 1629 counter_u64_add(hpts_wake_timeout, 1); 1630 if (callout_pending(&hpts->co) || 1631 !callout_active(&hpts->co)) { 1632 mtx_unlock(&hpts->p_mtx); 1633 return; 1634 } 1635 } 1636 callout_deactivate(&hpts->co); 1637 hpts->p_hpts_wake_scheduled = 0; 1638 NET_EPOCH_ENTER(et); 1639 if (hpts->p_hpts_active) { 1640 /* 1641 * We are active already. This means that a syscall 1642 * trap or LRO is running in behalf of hpts. In that case 1643 * we need to double our timeout since there seems to be 1644 * enough activity in the system that we don't need to 1645 * run as often (if we were not directly woken). 1646 */ 1647 if (hpts->p_direct_wake == 0) { 1648 counter_u64_add(hpts_back_tosleep, 1); 1649 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { 1650 hpts->p_mysleep.tv_usec *= 2; 1651 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) 1652 hpts->p_mysleep.tv_usec = dynamic_max_sleep; 1653 tv.tv_usec = hpts->p_mysleep.tv_usec; 1654 hpts->p_on_min_sleep = 1; 1655 } else { 1656 /* 1657 * Here we have low count on the wheel, but 1658 * somehow we still collided with one of the 1659 * connections. Lets go back to sleep for a 1660 * min sleep time, but clear the flag so we 1661 * can be awoken by insert. 1662 */ 1663 hpts->p_on_min_sleep = 0; 1664 tv.tv_usec = tcp_min_hptsi_time; 1665 } 1666 } else { 1667 /* 1668 * Directly woken most likely to reset the 1669 * callout time. 1670 */ 1671 tv.tv_sec = 0; 1672 tv.tv_usec = hpts->p_mysleep.tv_usec; 1673 } 1674 goto back_to_sleep; 1675 } 1676 hpts->sleeping = 0; 1677 hpts->p_hpts_active = 1; 1678 ticks_ran = tcp_hptsi(hpts, 1); 1679 tv.tv_sec = 0; 1680 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; 1681 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { 1682 if(hpts->p_direct_wake == 0) { 1683 /* 1684 * Only adjust sleep time if we were 1685 * called from the callout i.e. direct_wake == 0. 1686 */ 1687 if (ticks_ran < ticks_indicate_more_sleep) { 1688 hpts->p_mysleep.tv_usec *= 2; 1689 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) 1690 hpts->p_mysleep.tv_usec = dynamic_max_sleep; 1691 } else if (ticks_ran > ticks_indicate_less_sleep) { 1692 hpts->p_mysleep.tv_usec /= 2; 1693 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) 1694 hpts->p_mysleep.tv_usec = dynamic_min_sleep; 1695 } 1696 } 1697 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { 1698 hpts->overidden_sleep = tv.tv_usec; 1699 tv.tv_usec = hpts->p_mysleep.tv_usec; 1700 } else if (tv.tv_usec > dynamic_max_sleep) { 1701 /* Lets not let sleep get above this value */ 1702 hpts->overidden_sleep = tv.tv_usec; 1703 tv.tv_usec = dynamic_max_sleep; 1704 } 1705 /* 1706 * In this mode the timer is a backstop to 1707 * all the userret/lro_flushes so we use 1708 * the dynamic value and set the on_min_sleep 1709 * flag so we will not be awoken. 1710 */ 1711 hpts->p_on_min_sleep = 1; 1712 } else if (hpts->p_on_queue_cnt == 0) { 1713 /* 1714 * No one on the wheel, please wake us up 1715 * if you insert on the wheel. 1716 */ 1717 hpts->p_on_min_sleep = 0; 1718 hpts->overidden_sleep = 0; 1719 } else { 1720 /* 1721 * We hit here when we have a low number of 1722 * clients on the wheel (our else clause). 1723 * We may need to go on min sleep, if we set 1724 * the flag we will not be awoken if someone 1725 * is inserted ahead of us. Clearing the flag 1726 * means we can be awoken. This is "old mode" 1727 * where the timer is what runs hpts mainly. 1728 */ 1729 if (tv.tv_usec < tcp_min_hptsi_time) { 1730 /* 1731 * Yes on min sleep, which means 1732 * we cannot be awoken. 1733 */ 1734 hpts->overidden_sleep = tv.tv_usec; 1735 tv.tv_usec = tcp_min_hptsi_time; 1736 hpts->p_on_min_sleep = 1; 1737 } else { 1738 /* Clear the min sleep flag */ 1739 hpts->overidden_sleep = 0; 1740 hpts->p_on_min_sleep = 0; 1741 } 1742 } 1743 HPTS_MTX_ASSERT(hpts); 1744 hpts->p_hpts_active = 0; 1745 back_to_sleep: 1746 hpts->p_direct_wake = 0; 1747 sb = tvtosbt(tv); 1748 /* Store off to make visible the actual sleep time */ 1749 hpts->sleeping = tv.tv_usec; 1750 callout_reset_sbt_on(&hpts->co, sb, 0, 1751 hpts_timeout_swi, hpts, hpts->p_cpu, 1752 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1753 NET_EPOCH_EXIT(et); 1754 mtx_unlock(&hpts->p_mtx); 1755 } 1756 1757 #undef timersub 1758 1759 static int32_t 1760 hpts_count_level(struct cpu_group *cg) 1761 { 1762 int32_t count_l3, i; 1763 1764 count_l3 = 0; 1765 if (cg->cg_level == CG_SHARE_L3) 1766 count_l3++; 1767 /* Walk all the children looking for L3 */ 1768 for (i = 0; i < cg->cg_children; i++) { 1769 count_l3 += hpts_count_level(&cg->cg_child[i]); 1770 } 1771 return (count_l3); 1772 } 1773 1774 static void 1775 hpts_gather_grps(struct cpu_group **grps, int32_t *at, int32_t max, struct cpu_group *cg) 1776 { 1777 int32_t idx, i; 1778 1779 idx = *at; 1780 if (cg->cg_level == CG_SHARE_L3) { 1781 grps[idx] = cg; 1782 idx++; 1783 if (idx == max) { 1784 *at = idx; 1785 return; 1786 } 1787 } 1788 *at = idx; 1789 /* Walk all the children looking for L3 */ 1790 for (i = 0; i < cg->cg_children; i++) { 1791 hpts_gather_grps(grps, at, max, &cg->cg_child[i]); 1792 } 1793 } 1794 1795 static void 1796 tcp_hpts_mod_load(void) 1797 { 1798 struct cpu_group *cpu_top; 1799 int32_t error __diagused; 1800 int32_t i, j, bound = 0, created = 0; 1801 size_t sz, asz; 1802 struct timeval tv; 1803 sbintime_t sb; 1804 struct tcp_hpts_entry *hpts; 1805 struct pcpu *pc; 1806 char unit[16]; 1807 uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU; 1808 int count, domain; 1809 1810 #ifdef SMP 1811 cpu_top = smp_topo(); 1812 #else 1813 cpu_top = NULL; 1814 #endif 1815 tcp_pace.rp_num_hptss = ncpus; 1816 hpts_hopelessly_behind = counter_u64_alloc(M_WAITOK); 1817 hpts_loops = counter_u64_alloc(M_WAITOK); 1818 back_tosleep = counter_u64_alloc(M_WAITOK); 1819 combined_wheel_wrap = counter_u64_alloc(M_WAITOK); 1820 wheel_wrap = counter_u64_alloc(M_WAITOK); 1821 hpts_wake_timeout = counter_u64_alloc(M_WAITOK); 1822 hpts_direct_awakening = counter_u64_alloc(M_WAITOK); 1823 hpts_back_tosleep = counter_u64_alloc(M_WAITOK); 1824 hpts_direct_call = counter_u64_alloc(M_WAITOK); 1825 cpu_uses_flowid = counter_u64_alloc(M_WAITOK); 1826 cpu_uses_random = counter_u64_alloc(M_WAITOK); 1827 1828 sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *)); 1829 tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); 1830 sz = (sizeof(uint32_t) * tcp_pace.rp_num_hptss); 1831 tcp_pace.cts_last_ran = malloc(sz, M_TCPHPTS, M_WAITOK); 1832 tcp_pace.grp_cnt = 0; 1833 if (cpu_top == NULL) { 1834 tcp_pace.grp_cnt = 1; 1835 } else { 1836 /* Find out how many cache level 3 domains we have */ 1837 count = 0; 1838 tcp_pace.grp_cnt = hpts_count_level(cpu_top); 1839 if (tcp_pace.grp_cnt == 0) { 1840 tcp_pace.grp_cnt = 1; 1841 } 1842 sz = (tcp_pace.grp_cnt * sizeof(struct cpu_group *)); 1843 tcp_pace.grps = malloc(sz, M_TCPHPTS, M_WAITOK); 1844 /* Now populate the groups */ 1845 if (tcp_pace.grp_cnt == 1) { 1846 /* 1847 * All we need is the top level all cpu's are in 1848 * the same cache so when we use grp[0]->cg_mask 1849 * with the cg_first <-> cg_last it will include 1850 * all cpu's in it. The level here is probably 1851 * zero which is ok. 1852 */ 1853 tcp_pace.grps[0] = cpu_top; 1854 } else { 1855 /* 1856 * Here we must find all the level three cache domains 1857 * and setup our pointers to them. 1858 */ 1859 count = 0; 1860 hpts_gather_grps(tcp_pace.grps, &count, tcp_pace.grp_cnt, cpu_top); 1861 } 1862 } 1863 asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS; 1864 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 1865 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry), 1866 M_TCPHPTS, M_WAITOK | M_ZERO); 1867 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, M_TCPHPTS, M_WAITOK); 1868 hpts = tcp_pace.rp_ent[i]; 1869 /* 1870 * Init all the hpts structures that are not specifically 1871 * zero'd by the allocations. Also lets attach them to the 1872 * appropriate sysctl block as well. 1873 */ 1874 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", 1875 "hpts", MTX_DEF | MTX_DUPOK); 1876 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) { 1877 TAILQ_INIT(&hpts->p_hptss[j].head); 1878 hpts->p_hptss[j].count = 0; 1879 hpts->p_hptss[j].gencnt = 0; 1880 } 1881 sysctl_ctx_init(&hpts->hpts_ctx); 1882 sprintf(unit, "%d", i); 1883 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, 1884 SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts), 1885 OID_AUTO, 1886 unit, 1887 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1888 ""); 1889 SYSCTL_ADD_INT(&hpts->hpts_ctx, 1890 SYSCTL_CHILDREN(hpts->hpts_root), 1891 OID_AUTO, "out_qcnt", CTLFLAG_RD, 1892 &hpts->p_on_queue_cnt, 0, 1893 "Count TCB's awaiting output processing"); 1894 SYSCTL_ADD_U16(&hpts->hpts_ctx, 1895 SYSCTL_CHILDREN(hpts->hpts_root), 1896 OID_AUTO, "active", CTLFLAG_RD, 1897 &hpts->p_hpts_active, 0, 1898 "Is the hpts active"); 1899 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1900 SYSCTL_CHILDREN(hpts->hpts_root), 1901 OID_AUTO, "curslot", CTLFLAG_RD, 1902 &hpts->p_cur_slot, 0, 1903 "What the current running pacers goal"); 1904 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1905 SYSCTL_CHILDREN(hpts->hpts_root), 1906 OID_AUTO, "runtick", CTLFLAG_RD, 1907 &hpts->p_runningslot, 0, 1908 "What the running pacers current slot is"); 1909 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1910 SYSCTL_CHILDREN(hpts->hpts_root), 1911 OID_AUTO, "curtick", CTLFLAG_RD, 1912 &hpts->p_curtick, 0, 1913 "What the running pacers last tick mapped to the wheel was"); 1914 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1915 SYSCTL_CHILDREN(hpts->hpts_root), 1916 OID_AUTO, "lastran", CTLFLAG_RD, 1917 &tcp_pace.cts_last_ran[i], 0, 1918 "The last usec tick that this hpts ran"); 1919 SYSCTL_ADD_LONG(&hpts->hpts_ctx, 1920 SYSCTL_CHILDREN(hpts->hpts_root), 1921 OID_AUTO, "cur_min_sleep", CTLFLAG_RD, 1922 &hpts->p_mysleep.tv_usec, 1923 "What the running pacers is using for p_mysleep.tv_usec"); 1924 SYSCTL_ADD_U64(&hpts->hpts_ctx, 1925 SYSCTL_CHILDREN(hpts->hpts_root), 1926 OID_AUTO, "now_sleeping", CTLFLAG_RD, 1927 &hpts->sleeping, 0, 1928 "What the running pacers is actually sleeping for"); 1929 SYSCTL_ADD_U64(&hpts->hpts_ctx, 1930 SYSCTL_CHILDREN(hpts->hpts_root), 1931 OID_AUTO, "syscall_cnt", CTLFLAG_RD, 1932 &hpts->syscall_cnt, 0, 1933 "How many times we had syscalls on this hpts"); 1934 1935 hpts->p_hpts_sleep_time = hpts_sleep_max; 1936 hpts->p_num = i; 1937 hpts->p_curtick = tcp_gethptstick(&tv); 1938 tcp_pace.cts_last_ran[i] = tcp_tv_to_usectick(&tv); 1939 hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); 1940 hpts->p_cpu = 0xffff; 1941 hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1); 1942 callout_init(&hpts->co, 1); 1943 } 1944 /* Don't try to bind to NUMA domains if we don't have any */ 1945 if (vm_ndomains == 1 && tcp_bind_threads == 2) 1946 tcp_bind_threads = 0; 1947 1948 /* 1949 * Now lets start ithreads to handle the hptss. 1950 */ 1951 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 1952 hpts = tcp_pace.rp_ent[i]; 1953 hpts->p_cpu = i; 1954 1955 error = swi_add(&hpts->ie, "hpts", 1956 tcp_hpts_thread, (void *)hpts, 1957 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); 1958 KASSERT(error == 0, 1959 ("Can't add hpts:%p i:%d err:%d", 1960 hpts, i, error)); 1961 created++; 1962 hpts->p_mysleep.tv_sec = 0; 1963 hpts->p_mysleep.tv_usec = tcp_min_hptsi_time; 1964 if (tcp_bind_threads == 1) { 1965 if (intr_event_bind(hpts->ie, i) == 0) 1966 bound++; 1967 } else if (tcp_bind_threads == 2) { 1968 /* Find the group for this CPU (i) and bind into it */ 1969 for (j = 0; j < tcp_pace.grp_cnt; j++) { 1970 if (CPU_ISSET(i, &tcp_pace.grps[j]->cg_mask)) { 1971 if (intr_event_bind_ithread_cpuset(hpts->ie, 1972 &tcp_pace.grps[j]->cg_mask) == 0) { 1973 bound++; 1974 pc = pcpu_find(i); 1975 domain = pc->pc_domain; 1976 count = hpts_domains[domain].count; 1977 hpts_domains[domain].cpu[count] = i; 1978 hpts_domains[domain].count++; 1979 break; 1980 } 1981 } 1982 } 1983 } 1984 tv.tv_sec = 0; 1985 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; 1986 hpts->sleeping = tv.tv_usec; 1987 sb = tvtosbt(tv); 1988 callout_reset_sbt_on(&hpts->co, sb, 0, 1989 hpts_timeout_swi, hpts, hpts->p_cpu, 1990 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1991 } 1992 /* 1993 * If we somehow have an empty domain, fall back to choosing 1994 * among all htps threads. 1995 */ 1996 for (i = 0; i < vm_ndomains; i++) { 1997 if (hpts_domains[i].count == 0) { 1998 tcp_bind_threads = 0; 1999 break; 2000 } 2001 } 2002 tcp_hpts_softclock = __tcp_run_hpts; 2003 tcp_lro_hpts_init(); 2004 printf("TCP Hpts created %d swi interrupt threads and bound %d to %s\n", 2005 created, bound, 2006 tcp_bind_threads == 2 ? "NUMA domains" : "cpus"); 2007 } 2008 2009 static void 2010 tcp_hpts_mod_unload(void) 2011 { 2012 int rv __diagused; 2013 2014 tcp_lro_hpts_uninit(); 2015 atomic_store_ptr(&tcp_hpts_softclock, NULL); 2016 2017 for (int i = 0; i < tcp_pace.rp_num_hptss; i++) { 2018 struct tcp_hpts_entry *hpts = tcp_pace.rp_ent[i]; 2019 2020 rv = callout_drain(&hpts->co); 2021 MPASS(rv != 0); 2022 2023 rv = swi_remove(hpts->ie_cookie); 2024 MPASS(rv == 0); 2025 2026 rv = sysctl_ctx_free(&hpts->hpts_ctx); 2027 MPASS(rv == 0); 2028 2029 mtx_destroy(&hpts->p_mtx); 2030 free(hpts->p_hptss, M_TCPHPTS); 2031 free(hpts, M_TCPHPTS); 2032 } 2033 2034 free(tcp_pace.rp_ent, M_TCPHPTS); 2035 free(tcp_pace.cts_last_ran, M_TCPHPTS); 2036 #ifdef SMP 2037 free(tcp_pace.grps, M_TCPHPTS); 2038 #endif 2039 2040 counter_u64_free(hpts_hopelessly_behind); 2041 counter_u64_free(hpts_loops); 2042 counter_u64_free(back_tosleep); 2043 counter_u64_free(combined_wheel_wrap); 2044 counter_u64_free(wheel_wrap); 2045 counter_u64_free(hpts_wake_timeout); 2046 counter_u64_free(hpts_direct_awakening); 2047 counter_u64_free(hpts_back_tosleep); 2048 counter_u64_free(hpts_direct_call); 2049 counter_u64_free(cpu_uses_flowid); 2050 counter_u64_free(cpu_uses_random); 2051 } 2052 2053 static int 2054 tcp_hpts_modevent(module_t mod, int what, void *arg) 2055 { 2056 2057 switch (what) { 2058 case MOD_LOAD: 2059 tcp_hpts_mod_load(); 2060 return (0); 2061 case MOD_QUIESCE: 2062 /* 2063 * Since we are a dependency of TCP stack modules, they should 2064 * already be unloaded, and the HPTS ring is empty. However, 2065 * function pointer manipulations aren't 100% safe. Although, 2066 * tcp_hpts_mod_unload() use atomic(9) the userret() doesn't. 2067 * Thus, allow only forced unload of HPTS. 2068 */ 2069 return (EBUSY); 2070 case MOD_UNLOAD: 2071 tcp_hpts_mod_unload(); 2072 return (0); 2073 default: 2074 return (EINVAL); 2075 }; 2076 } 2077 2078 static moduledata_t tcp_hpts_module = { 2079 .name = "tcphpts", 2080 .evhand = tcp_hpts_modevent, 2081 }; 2082 2083 DECLARE_MODULE(tcphpts, tcp_hpts_module, SI_SUB_SOFTINTR, SI_ORDER_ANY); 2084 MODULE_VERSION(tcphpts, 1); 2085