1 /*- 2 * Copyright (c) 2016-2018 Netflix Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include "opt_inet.h" 30 #include "opt_inet6.h" 31 #include "opt_ipsec.h" 32 #include "opt_tcpdebug.h" 33 /** 34 * Some notes about usage. 35 * 36 * The tcp_hpts system is designed to provide a high precision timer 37 * system for tcp. Its main purpose is to provide a mechanism for 38 * pacing packets out onto the wire. It can be used in two ways 39 * by a given TCP stack (and those two methods can be used simultaneously). 40 * 41 * First, and probably the main thing its used by Rack and BBR for, it can 42 * be used to call tcp_output() of a transport stack at some time in the future. 43 * The normal way this is done is that tcp_output() of the stack schedules 44 * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The 45 * slot is the time from now that the stack wants to be called but it 46 * must be converted to tcp_hpts's notion of slot. This is done with 47 * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical 48 * call from the tcp_output() routine might look like: 49 * 50 * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550)); 51 * 52 * The above would schedule tcp_ouput() to be called in 550 useconds. 53 * Note that if using this mechanism the stack will want to add near 54 * its top a check to prevent unwanted calls (from user land or the 55 * arrival of incoming ack's). So it would add something like: 56 * 57 * if (inp->inp_in_hpts) 58 * return; 59 * 60 * to prevent output processing until the time alotted has gone by. 61 * Of course this is a bare bones example and the stack will probably 62 * have more consideration then just the above. 63 * 64 * Now the tcp_hpts system will call tcp_output in one of two forms, 65 * it will first check to see if the stack as defined a 66 * tfb_tcp_output_wtime() function, if so that is the routine it 67 * will call, if that function is not defined then it will call the 68 * tfb_tcp_output() function. The only difference between these 69 * two calls is that the former passes the time in to the function 70 * so the function does not have to access the time (which tcp_hpts 71 * already has). What these functions do is of course totally up 72 * to the individual tcp stack. 73 * 74 * Now the second function (actually two functions I guess :D) 75 * the tcp_hpts system provides is the ability to either abort 76 * a connection (later) or process input on a connection. 77 * Why would you want to do this? To keep processor locality. 78 * 79 * So in order to use the input redirection function the 80 * stack changes its tcp_do_segment() routine to instead 81 * of process the data call the function: 82 * 83 * tcp_queue_pkt_to_input() 84 * 85 * You will note that the arguments to this function look 86 * a lot like tcp_do_segments's arguments. This function 87 * will assure that the tcp_hpts system will 88 * call the functions tfb_tcp_hpts_do_segment() from the 89 * correct CPU. Note that multiple calls can get pushed 90 * into the tcp_hpts system this will be indicated by 91 * the next to last argument to tfb_tcp_hpts_do_segment() 92 * (nxt_pkt). If nxt_pkt is a 1 then another packet is 93 * coming. If nxt_pkt is a 0 then this is the last call 94 * that the tcp_hpts system has available for the tcp stack. 95 * 96 * The other point of the input system is to be able to safely 97 * drop a tcp connection without worrying about the recursive 98 * locking that may be occuring on the INP_WLOCK. So if 99 * a stack wants to drop a connection it calls: 100 * 101 * tcp_set_inp_to_drop(tp, ETIMEDOUT) 102 * 103 * To schedule the tcp_hpts system to call 104 * 105 * tcp_drop(tp, drop_reason) 106 * 107 * at a future point. This is quite handy to prevent locking 108 * issues when dropping connections. 109 * 110 */ 111 112 #include <sys/param.h> 113 #include <sys/bus.h> 114 #include <sys/interrupt.h> 115 #include <sys/module.h> 116 #include <sys/kernel.h> 117 #include <sys/hhook.h> 118 #include <sys/malloc.h> 119 #include <sys/mbuf.h> 120 #include <sys/proc.h> /* for proc0 declaration */ 121 #include <sys/socket.h> 122 #include <sys/socketvar.h> 123 #include <sys/sysctl.h> 124 #include <sys/systm.h> 125 #include <sys/refcount.h> 126 #include <sys/sched.h> 127 #include <sys/queue.h> 128 #include <sys/smp.h> 129 #include <sys/counter.h> 130 #include <sys/time.h> 131 #include <sys/kthread.h> 132 #include <sys/kern_prefetch.h> 133 134 #include <vm/uma.h> 135 136 #include <net/route.h> 137 #include <net/vnet.h> 138 139 #define TCPSTATES /* for logging */ 140 141 #include <netinet/in.h> 142 #include <netinet/in_kdtrace.h> 143 #include <netinet/in_pcb.h> 144 #include <netinet/ip.h> 145 #include <netinet/ip_icmp.h> /* required for icmp_var.h */ 146 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */ 147 #include <netinet/ip_var.h> 148 #include <netinet/ip6.h> 149 #include <netinet6/in6_pcb.h> 150 #include <netinet6/ip6_var.h> 151 #define TCPOUTFLAGS 152 #include <netinet/tcp.h> 153 #include <netinet/tcp_fsm.h> 154 #include <netinet/tcp_seq.h> 155 #include <netinet/tcp_timer.h> 156 #include <netinet/tcp_var.h> 157 #include <netinet/tcpip.h> 158 #include <netinet/cc/cc.h> 159 #include <netinet/tcp_hpts.h> 160 161 #ifdef tcpdebug 162 #include <netinet/tcp_debug.h> 163 #endif /* tcpdebug */ 164 #ifdef tcp_offload 165 #include <netinet/tcp_offload.h> 166 #endif 167 168 #ifdef ipsec 169 #include <netipsec/ipsec.h> 170 #include <netipsec/ipsec6.h> 171 #endif /* ipsec */ 172 #include "opt_rss.h" 173 174 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts"); 175 #ifdef RSS 176 static int tcp_bind_threads = 1; 177 #else 178 static int tcp_bind_threads = 0; 179 #endif 180 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); 181 182 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG; 183 184 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size); 185 186 static struct tcp_hptsi tcp_pace; 187 188 static int 189 tcp_hptsi_lock_inpinfo(struct inpcb *inp, 190 struct tcpcb **tp); 191 static void tcp_wakehpts(struct tcp_hpts_entry *p); 192 static void tcp_wakeinput(struct tcp_hpts_entry *p); 193 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv); 194 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick); 195 static void tcp_hpts_thread(void *ctx); 196 static void tcp_init_hptsi(void *st); 197 198 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; 199 static int32_t tcp_hpts_callout_skip_swi = 0; 200 201 SYSCTL_DECL(_net_inet_tcp); 202 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls"); 203 204 #define timersub(tvp, uvp, vvp) \ 205 do { \ 206 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ 207 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ 208 if ((vvp)->tv_usec < 0) { \ 209 (vvp)->tv_sec--; \ 210 (vvp)->tv_usec += 1000000; \ 211 } \ 212 } while (0) 213 214 static int32_t logging_on = 0; 215 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2); 216 static int32_t tcp_hpts_precision = 120; 217 218 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, 219 &tcp_hpts_precision, 120, 220 "Value for PRE() precision of callout"); 221 222 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, 223 &logging_on, 0, 224 "Turn on logging if compiled in"); 225 226 counter_u64_t hpts_loops; 227 228 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD, 229 &hpts_loops, "Number of times hpts had to loop to catch up"); 230 231 counter_u64_t back_tosleep; 232 233 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD, 234 &back_tosleep, "Number of times hpts found no tcbs"); 235 236 static int32_t in_newts_every_tcb = 0; 237 238 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW, 239 &in_newts_every_tcb, 0, 240 "Do we have a new cts every tcb we process for input"); 241 static int32_t in_ts_percision = 0; 242 243 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW, 244 &in_ts_percision, 0, 245 "Do we use percise timestamp for clients on input"); 246 static int32_t out_newts_every_tcb = 0; 247 248 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW, 249 &out_newts_every_tcb, 0, 250 "Do we have a new cts every tcb we process for output"); 251 static int32_t out_ts_percision = 0; 252 253 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW, 254 &out_ts_percision, 0, 255 "Do we use a percise timestamp for every output cts"); 256 257 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW, 258 &hpts_sleep_max, 0, 259 "The maximum time the hpts will sleep <1 - 254>"); 260 261 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW, 262 &tcp_min_hptsi_time, 0, 263 "The minimum time the hpts must sleep before processing more slots"); 264 265 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW, 266 &tcp_hpts_callout_skip_swi, 0, 267 "Do we have the callout call directly to the hpts?"); 268 269 static void 270 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot, 271 uint32_t ticknow, int32_t line) 272 { 273 struct hpts_log *pl; 274 275 HPTS_MTX_ASSERT(hpts); 276 if (hpts->p_log == NULL) 277 return; 278 pl = &hpts->p_log[hpts->p_log_at]; 279 hpts->p_log_at++; 280 if (hpts->p_log_at >= hpts->p_logsize) { 281 hpts->p_log_at = 0; 282 hpts->p_log_wrapped = 1; 283 } 284 pl->inp = inp; 285 if (inp) { 286 pl->t_paceslot = inp->inp_hptsslot; 287 pl->t_hptsreq = inp->inp_hpts_request; 288 pl->p_onhpts = inp->inp_in_hpts; 289 pl->p_oninput = inp->inp_in_input; 290 } else { 291 pl->t_paceslot = 0; 292 pl->t_hptsreq = 0; 293 pl->p_onhpts = 0; 294 pl->p_oninput = 0; 295 } 296 pl->is_notempty = 1; 297 pl->event = event; 298 pl->line = line; 299 pl->cts = tcp_get_usecs(NULL); 300 pl->p_curtick = hpts->p_curtick; 301 pl->p_prevtick = hpts->p_prevtick; 302 pl->p_on_queue_cnt = hpts->p_on_queue_cnt; 303 pl->ticknow = ticknow; 304 pl->slot_req = slot; 305 pl->p_nxt_slot = hpts->p_nxt_slot; 306 pl->p_cur_slot = hpts->p_cur_slot; 307 pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time; 308 pl->p_flags = (hpts->p_cpu & 0x7f); 309 pl->p_flags <<= 7; 310 pl->p_flags |= (hpts->p_num & 0x7f); 311 pl->p_flags <<= 2; 312 if (hpts->p_hpts_active) { 313 pl->p_flags |= HPTS_HPTS_ACTIVE; 314 } 315 } 316 317 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__) 318 319 static void 320 hpts_timeout_swi(void *arg) 321 { 322 struct tcp_hpts_entry *hpts; 323 324 hpts = (struct tcp_hpts_entry *)arg; 325 swi_sched(hpts->ie_cookie, 0); 326 } 327 328 static void 329 hpts_timeout_dir(void *arg) 330 { 331 tcp_hpts_thread(arg); 332 } 333 334 static inline void 335 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear) 336 { 337 #ifdef INVARIANTS 338 if (mtx_owned(&hpts->p_mtx) == 0) { 339 /* We don't own the mutex? */ 340 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 341 } 342 if (hpts->p_cpu != inp->inp_hpts_cpu) { 343 /* It is not the right cpu/mutex? */ 344 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 345 } 346 if (inp->inp_in_hpts == 0) { 347 /* We are not on the hpts? */ 348 panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp); 349 } 350 if (TAILQ_EMPTY(head) && 351 (hpts->p_on_queue_cnt != 0)) { 352 /* We should not be empty with a queue count */ 353 panic("%s hpts:%p hpts bucket empty but cnt:%d", 354 __FUNCTION__, hpts, hpts->p_on_queue_cnt); 355 } 356 #endif 357 TAILQ_REMOVE(head, inp, inp_hpts); 358 hpts->p_on_queue_cnt--; 359 if (hpts->p_on_queue_cnt < 0) { 360 /* Count should not go negative .. */ 361 #ifdef INVARIANTS 362 panic("Hpts goes negative inp:%p hpts:%p", 363 inp, hpts); 364 #endif 365 hpts->p_on_queue_cnt = 0; 366 } 367 if (clear) { 368 inp->inp_hpts_request = 0; 369 inp->inp_in_hpts = 0; 370 } 371 } 372 373 static inline void 374 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref) 375 { 376 #ifdef INVARIANTS 377 if (mtx_owned(&hpts->p_mtx) == 0) { 378 /* We don't own the mutex? */ 379 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 380 } 381 if (hpts->p_cpu != inp->inp_hpts_cpu) { 382 /* It is not the right cpu/mutex? */ 383 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 384 } 385 if ((noref == 0) && (inp->inp_in_hpts == 1)) { 386 /* We are already on the hpts? */ 387 panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp); 388 } 389 #endif 390 TAILQ_INSERT_TAIL(head, inp, inp_hpts); 391 inp->inp_in_hpts = 1; 392 hpts->p_on_queue_cnt++; 393 if (noref == 0) { 394 in_pcbref(inp); 395 } 396 } 397 398 static inline void 399 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear) 400 { 401 #ifdef INVARIANTS 402 if (mtx_owned(&hpts->p_mtx) == 0) { 403 /* We don't own the mutex? */ 404 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 405 } 406 if (hpts->p_cpu != inp->inp_input_cpu) { 407 /* It is not the right cpu/mutex? */ 408 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 409 } 410 if (inp->inp_in_input == 0) { 411 /* We are not on the input hpts? */ 412 panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp); 413 } 414 #endif 415 TAILQ_REMOVE(&hpts->p_input, inp, inp_input); 416 hpts->p_on_inqueue_cnt--; 417 if (hpts->p_on_inqueue_cnt < 0) { 418 #ifdef INVARIANTS 419 panic("Hpts in goes negative inp:%p hpts:%p", 420 inp, hpts); 421 #endif 422 hpts->p_on_inqueue_cnt = 0; 423 } 424 #ifdef INVARIANTS 425 if (TAILQ_EMPTY(&hpts->p_input) && 426 (hpts->p_on_inqueue_cnt != 0)) { 427 /* We should not be empty with a queue count */ 428 panic("%s hpts:%p in_hpts input empty but cnt:%d", 429 __FUNCTION__, hpts, hpts->p_on_inqueue_cnt); 430 } 431 #endif 432 if (clear) 433 inp->inp_in_input = 0; 434 } 435 436 static inline void 437 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line) 438 { 439 #ifdef INVARIANTS 440 if (mtx_owned(&hpts->p_mtx) == 0) { 441 /* We don't own the mutex? */ 442 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 443 } 444 if (hpts->p_cpu != inp->inp_input_cpu) { 445 /* It is not the right cpu/mutex? */ 446 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 447 } 448 if (inp->inp_in_input == 1) { 449 /* We are already on the input hpts? */ 450 panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp); 451 } 452 #endif 453 TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input); 454 inp->inp_in_input = 1; 455 hpts->p_on_inqueue_cnt++; 456 in_pcbref(inp); 457 } 458 459 static int 460 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS) 461 { 462 struct tcp_hpts_entry *hpts; 463 size_t sz; 464 int32_t logging_was, i; 465 int32_t error = 0; 466 467 /* 468 * HACK: Turn off logging so no locks are required this really needs 469 * a memory barrier :) 470 */ 471 logging_was = logging_on; 472 logging_on = 0; 473 if (!req->oldptr) { 474 /* How much? */ 475 sz = 0; 476 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 477 hpts = tcp_pace.rp_ent[i]; 478 if (hpts->p_log == NULL) 479 continue; 480 sz += (sizeof(struct hpts_log) * hpts->p_logsize); 481 } 482 error = SYSCTL_OUT(req, 0, sz); 483 } else { 484 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 485 hpts = tcp_pace.rp_ent[i]; 486 if (hpts->p_log == NULL) 487 continue; 488 if (hpts->p_log_wrapped) 489 sz = (sizeof(struct hpts_log) * hpts->p_logsize); 490 else 491 sz = (sizeof(struct hpts_log) * hpts->p_log_at); 492 error = SYSCTL_OUT(req, hpts->p_log, sz); 493 } 494 } 495 logging_on = logging_was; 496 return error; 497 } 498 499 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 500 0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log"); 501 502 503 /* 504 * Try to get the INP_INFO lock. 505 * 506 * This function always succeeds in getting the lock. It will clear 507 * *tpp and return (1) if something critical changed while the inpcb 508 * was unlocked. Otherwise, it will leave *tpp unchanged and return (0). 509 * 510 * This function relies on the fact that the hpts always holds a 511 * reference on the inpcb while the segment is on the hptsi wheel and 512 * in the input queue. 513 * 514 */ 515 static int 516 tcp_hptsi_lock_inpinfo(struct inpcb *inp, struct tcpcb **tpp) 517 { 518 struct tcp_function_block *tfb; 519 struct tcpcb *tp; 520 void *ptr; 521 522 /* Try the easy way. */ 523 if (INP_INFO_TRY_RLOCK(&V_tcbinfo)) 524 return (0); 525 526 /* 527 * OK, let's try the hard way. We'll save the function pointer block 528 * to make sure that doesn't change while we aren't holding the 529 * lock. 530 */ 531 tp = *tpp; 532 tfb = tp->t_fb; 533 ptr = tp->t_fb_ptr; 534 INP_WUNLOCK(inp); 535 INP_INFO_RLOCK(&V_tcbinfo); 536 INP_WLOCK(inp); 537 /* If the session went away, return an error. */ 538 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 539 (inp->inp_flags2 & INP_FREED)) { 540 *tpp = NULL; 541 return (1); 542 } 543 /* 544 * If the function block or stack-specific data block changed, 545 * report an error. 546 */ 547 tp = intotcpcb(inp); 548 if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) { 549 *tpp = NULL; 550 return (1); 551 } 552 return (0); 553 } 554 555 556 static void 557 tcp_wakehpts(struct tcp_hpts_entry *hpts) 558 { 559 HPTS_MTX_ASSERT(hpts); 560 swi_sched(hpts->ie_cookie, 0); 561 if (hpts->p_hpts_active == 2) { 562 /* Rare sleeping on a ENOBUF */ 563 wakeup_one(hpts); 564 } 565 } 566 567 static void 568 tcp_wakeinput(struct tcp_hpts_entry *hpts) 569 { 570 HPTS_MTX_ASSERT(hpts); 571 swi_sched(hpts->ie_cookie, 0); 572 if (hpts->p_hpts_active == 2) { 573 /* Rare sleeping on a ENOBUF */ 574 wakeup_one(hpts); 575 } 576 } 577 578 struct tcp_hpts_entry * 579 tcp_cur_hpts(struct inpcb *inp) 580 { 581 int32_t hpts_num; 582 struct tcp_hpts_entry *hpts; 583 584 hpts_num = inp->inp_hpts_cpu; 585 hpts = tcp_pace.rp_ent[hpts_num]; 586 return (hpts); 587 } 588 589 struct tcp_hpts_entry * 590 tcp_hpts_lock(struct inpcb *inp) 591 { 592 struct tcp_hpts_entry *hpts; 593 int32_t hpts_num; 594 595 again: 596 hpts_num = inp->inp_hpts_cpu; 597 hpts = tcp_pace.rp_ent[hpts_num]; 598 #ifdef INVARIANTS 599 if (mtx_owned(&hpts->p_mtx)) { 600 panic("Hpts:%p owns mtx prior-to lock line:%d", 601 hpts, __LINE__); 602 } 603 #endif 604 mtx_lock(&hpts->p_mtx); 605 if (hpts_num != inp->inp_hpts_cpu) { 606 mtx_unlock(&hpts->p_mtx); 607 goto again; 608 } 609 return (hpts); 610 } 611 612 struct tcp_hpts_entry * 613 tcp_input_lock(struct inpcb *inp) 614 { 615 struct tcp_hpts_entry *hpts; 616 int32_t hpts_num; 617 618 again: 619 hpts_num = inp->inp_input_cpu; 620 hpts = tcp_pace.rp_ent[hpts_num]; 621 #ifdef INVARIANTS 622 if (mtx_owned(&hpts->p_mtx)) { 623 panic("Hpts:%p owns mtx prior-to lock line:%d", 624 hpts, __LINE__); 625 } 626 #endif 627 mtx_lock(&hpts->p_mtx); 628 if (hpts_num != inp->inp_input_cpu) { 629 mtx_unlock(&hpts->p_mtx); 630 goto again; 631 } 632 return (hpts); 633 } 634 635 static void 636 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line) 637 { 638 int32_t add_freed; 639 640 if (inp->inp_flags2 & INP_FREED) { 641 /* 642 * Need to play a special trick so that in_pcbrele_wlocked 643 * does not return 1 when it really should have returned 0. 644 */ 645 add_freed = 1; 646 inp->inp_flags2 &= ~INP_FREED; 647 } else { 648 add_freed = 0; 649 } 650 #ifndef INP_REF_DEBUG 651 if (in_pcbrele_wlocked(inp)) { 652 /* 653 * This should not happen. We have the inpcb referred to by 654 * the main socket (why we are called) and the hpts. It 655 * should always return 0. 656 */ 657 panic("inpcb:%p release ret 1", 658 inp); 659 } 660 #else 661 if (__in_pcbrele_wlocked(inp, line)) { 662 /* 663 * This should not happen. We have the inpcb referred to by 664 * the main socket (why we are called) and the hpts. It 665 * should always return 0. 666 */ 667 panic("inpcb:%p release ret 1", 668 inp); 669 } 670 #endif 671 if (add_freed) { 672 inp->inp_flags2 |= INP_FREED; 673 } 674 } 675 676 static void 677 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line) 678 { 679 if (inp->inp_in_hpts) { 680 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1); 681 tcp_remove_hpts_ref(inp, hpts, line); 682 } 683 } 684 685 static void 686 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line) 687 { 688 HPTS_MTX_ASSERT(hpts); 689 if (inp->inp_in_input) { 690 hpts_sane_input_remove(hpts, inp, 1); 691 tcp_remove_hpts_ref(inp, hpts, line); 692 } 693 } 694 695 /* 696 * Called normally with the INP_LOCKED but it 697 * does not matter, the hpts lock is the key 698 * but the lock order allows us to hold the 699 * INP lock and then get the hpts lock. 700 * 701 * Valid values in the flags are 702 * HPTS_REMOVE_OUTPUT - remove from the output of the hpts. 703 * HPTS_REMOVE_INPUT - remove from the input of the hpts. 704 * Note that you can or both values together and get two 705 * actions. 706 */ 707 void 708 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line) 709 { 710 struct tcp_hpts_entry *hpts; 711 712 INP_WLOCK_ASSERT(inp); 713 if (flags & HPTS_REMOVE_OUTPUT) { 714 hpts = tcp_hpts_lock(inp); 715 tcp_hpts_remove_locked_output(hpts, inp, flags, line); 716 mtx_unlock(&hpts->p_mtx); 717 } 718 if (flags & HPTS_REMOVE_INPUT) { 719 hpts = tcp_input_lock(inp); 720 tcp_hpts_remove_locked_input(hpts, inp, flags, line); 721 mtx_unlock(&hpts->p_mtx); 722 } 723 } 724 725 static inline int 726 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus) 727 { 728 return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS); 729 } 730 731 static int 732 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref) 733 { 734 int32_t need_wake = 0; 735 uint32_t ticknow = 0; 736 737 HPTS_MTX_ASSERT(hpts); 738 if (inp->inp_in_hpts == 0) { 739 /* Ok we need to set it on the hpts in the current slot */ 740 if (hpts->p_hpts_active == 0) { 741 /* A sleeping hpts we want in next slot to run */ 742 if (logging_on) { 743 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0, 744 hpts_tick(hpts, 1)); 745 } 746 inp->inp_hptsslot = hpts_tick(hpts, 1); 747 inp->inp_hpts_request = 0; 748 if (logging_on) { 749 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow); 750 } 751 need_wake = 1; 752 } else if ((void *)inp == hpts->p_inp) { 753 /* 754 * We can't allow you to go into the same slot we 755 * are in. We must put you out. 756 */ 757 inp->inp_hptsslot = hpts->p_nxt_slot; 758 } else 759 inp->inp_hptsslot = hpts->p_cur_slot; 760 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref); 761 inp->inp_hpts_request = 0; 762 if (logging_on) { 763 tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0); 764 } 765 if (need_wake) { 766 /* 767 * Activate the hpts if it is sleeping and its 768 * timeout is not 1. 769 */ 770 if (logging_on) { 771 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow); 772 } 773 hpts->p_direct_wake = 1; 774 tcp_wakehpts(hpts); 775 } 776 } 777 return (need_wake); 778 } 779 780 int 781 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line) 782 { 783 int32_t ret; 784 struct tcp_hpts_entry *hpts; 785 786 INP_WLOCK_ASSERT(inp); 787 hpts = tcp_hpts_lock(inp); 788 ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0); 789 mtx_unlock(&hpts->p_mtx); 790 return (ret); 791 } 792 793 static void 794 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line, 795 struct hpts_diag *diag, int32_t noref) 796 { 797 int32_t need_new_to = 0; 798 int32_t need_wakeup = 0; 799 uint32_t largest_slot; 800 uint32_t ticknow = 0; 801 uint32_t slot_calc; 802 803 HPTS_MTX_ASSERT(hpts); 804 if (diag) { 805 memset(diag, 0, sizeof(struct hpts_diag)); 806 diag->p_hpts_active = hpts->p_hpts_active; 807 diag->p_nxt_slot = hpts->p_nxt_slot; 808 diag->p_cur_slot = hpts->p_cur_slot; 809 diag->slot_req = slot; 810 } 811 if ((inp->inp_in_hpts == 0) || noref) { 812 inp->inp_hpts_request = slot; 813 if (slot == 0) { 814 /* Immediate */ 815 tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref); 816 return; 817 } 818 if (hpts->p_hpts_active) { 819 /* 820 * Its slot - 1 since nxt_slot is the next tick that 821 * will go off since the hpts is awake 822 */ 823 if (logging_on) { 824 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0); 825 } 826 /* 827 * We want to make sure that we don't place a inp in 828 * the range of p_cur_slot <-> p_nxt_slot. If we 829 * take from p_nxt_slot to the end, plus p_cur_slot 830 * and then take away 2, we will know how many is 831 * the max slots we can use. 832 */ 833 if (hpts->p_nxt_slot > hpts->p_cur_slot) { 834 /* 835 * Non-wrap case nxt_slot <-> cur_slot we 836 * don't want to land in. So the diff gives 837 * us what is taken away from the number of 838 * slots. 839 */ 840 largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot); 841 } else if (hpts->p_nxt_slot == hpts->p_cur_slot) { 842 largest_slot = NUM_OF_HPTSI_SLOTS - 2; 843 } else { 844 /* 845 * Wrap case so the diff gives us the number 846 * of slots that we can land in. 847 */ 848 largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot; 849 } 850 /* 851 * We take away two so we never have a problem (20 852 * usec's) out of 1024000 usecs 853 */ 854 largest_slot -= 2; 855 if (inp->inp_hpts_request > largest_slot) { 856 /* 857 * Restrict max jump of slots and remember 858 * leftover 859 */ 860 slot = largest_slot; 861 inp->inp_hpts_request -= largest_slot; 862 } else { 863 /* This one will run when we hit it */ 864 inp->inp_hpts_request = 0; 865 } 866 if (hpts->p_nxt_slot == hpts->p_cur_slot) 867 slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS; 868 else 869 slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS; 870 if (slot_calc == hpts->p_cur_slot) { 871 #ifdef INVARIANTS 872 /* TSNH */ 873 panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n", 874 hpts, slot_calc, slot, largest_slot); 875 #endif 876 if (slot_calc) 877 slot_calc--; 878 else 879 slot_calc = NUM_OF_HPTSI_SLOTS - 1; 880 } 881 inp->inp_hptsslot = slot_calc; 882 if (diag) { 883 diag->inp_hptsslot = inp->inp_hptsslot; 884 } 885 } else { 886 /* 887 * The hpts is sleeping, we need to figure out where 888 * it will wake up at and if we need to reschedule 889 * its time-out. 890 */ 891 uint32_t have_slept, yet_to_sleep; 892 uint32_t slot_now; 893 struct timeval tv; 894 895 ticknow = tcp_gethptstick(&tv); 896 slot_now = ticknow % NUM_OF_HPTSI_SLOTS; 897 /* 898 * The user wants to be inserted at (slot_now + 899 * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up. 900 */ 901 largest_slot = NUM_OF_HPTSI_SLOTS - 2; 902 if (inp->inp_hpts_request > largest_slot) { 903 /* Adjust the residual in inp_hpts_request */ 904 slot = largest_slot; 905 inp->inp_hpts_request -= largest_slot; 906 } else { 907 /* No residual it all fits */ 908 inp->inp_hpts_request = 0; 909 } 910 inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS; 911 if (diag) { 912 diag->slot_now = slot_now; 913 diag->inp_hptsslot = inp->inp_hptsslot; 914 diag->p_on_min_sleep = hpts->p_on_min_sleep; 915 } 916 if (logging_on) { 917 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow); 918 } 919 /* Now do we need to restart the hpts's timer? */ 920 if (TSTMP_GT(ticknow, hpts->p_curtick)) 921 have_slept = ticknow - hpts->p_curtick; 922 else 923 have_slept = 0; 924 if (have_slept < hpts->p_hpts_sleep_time) { 925 /* This should be what happens */ 926 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; 927 } else { 928 /* We are over-due */ 929 yet_to_sleep = 0; 930 need_wakeup = 1; 931 } 932 if (diag) { 933 diag->have_slept = have_slept; 934 diag->yet_to_sleep = yet_to_sleep; 935 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; 936 } 937 if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) { 938 /* 939 * We need to reschedule the hptss time-out. 940 */ 941 hpts->p_hpts_sleep_time = slot; 942 need_new_to = slot * HPTS_TICKS_PER_USEC; 943 } 944 } 945 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref); 946 if (logging_on) { 947 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow); 948 } 949 /* 950 * Now how far is the hpts sleeping to? if active is 1, its 951 * up and ticking we do nothing, otherwise we may need to 952 * reschedule its callout if need_new_to is set from above. 953 */ 954 if (need_wakeup) { 955 if (logging_on) { 956 tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0); 957 } 958 hpts->p_direct_wake = 1; 959 tcp_wakehpts(hpts); 960 if (diag) { 961 diag->need_new_to = 0; 962 diag->co_ret = 0xffff0000; 963 } 964 } else if (need_new_to) { 965 int32_t co_ret; 966 struct timeval tv; 967 sbintime_t sb; 968 969 tv.tv_sec = 0; 970 tv.tv_usec = 0; 971 while (need_new_to > HPTS_USEC_IN_SEC) { 972 tv.tv_sec++; 973 need_new_to -= HPTS_USEC_IN_SEC; 974 } 975 tv.tv_usec = need_new_to; 976 sb = tvtosbt(tv); 977 if (tcp_hpts_callout_skip_swi == 0) { 978 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, 979 hpts_timeout_swi, hpts, hpts->p_cpu, 980 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 981 } else { 982 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, 983 hpts_timeout_dir, hpts, 984 hpts->p_cpu, 985 C_PREL(tcp_hpts_precision)); 986 } 987 if (diag) { 988 diag->need_new_to = need_new_to; 989 diag->co_ret = co_ret; 990 } 991 } 992 } else { 993 #ifdef INVARIANTS 994 panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp); 995 #endif 996 } 997 } 998 999 uint32_t 1000 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){ 1001 struct tcp_hpts_entry *hpts; 1002 uint32_t slot_on, cts; 1003 struct timeval tv; 1004 1005 /* 1006 * We now return the next-slot the hpts will be on, beyond its 1007 * current run (if up) or where it was when it stopped if it is 1008 * sleeping. 1009 */ 1010 INP_WLOCK_ASSERT(inp); 1011 hpts = tcp_hpts_lock(inp); 1012 if (in_ts_percision) 1013 microuptime(&tv); 1014 else 1015 getmicrouptime(&tv); 1016 cts = tcp_tv_to_usectick(&tv); 1017 tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0); 1018 slot_on = hpts->p_nxt_slot; 1019 mtx_unlock(&hpts->p_mtx); 1020 return (slot_on); 1021 } 1022 1023 uint32_t 1024 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){ 1025 return (tcp_hpts_insert_diag(inp, slot, line, NULL)); 1026 } 1027 1028 int 1029 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line) 1030 { 1031 int32_t retval = 0; 1032 1033 HPTS_MTX_ASSERT(hpts); 1034 if (inp->inp_in_input == 0) { 1035 /* Ok we need to set it on the hpts in the current slot */ 1036 hpts_sane_input_insert(hpts, inp, line); 1037 retval = 1; 1038 if (hpts->p_hpts_active == 0) { 1039 /* 1040 * Activate the hpts if it is sleeping. 1041 */ 1042 if (logging_on) { 1043 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0); 1044 } 1045 retval = 2; 1046 hpts->p_direct_wake = 1; 1047 tcp_wakeinput(hpts); 1048 } 1049 } else if (hpts->p_hpts_active == 0) { 1050 retval = 4; 1051 hpts->p_direct_wake = 1; 1052 tcp_wakeinput(hpts); 1053 } 1054 return (retval); 1055 } 1056 1057 void 1058 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, 1059 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked) 1060 { 1061 /* Setup packet for input first */ 1062 INP_WLOCK_ASSERT(tp->t_inpcb); 1063 m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t)); 1064 m->m_pkthdr.pace_tlen = (uint16_t) tlen; 1065 m->m_pkthdr.pace_drphdrlen = drop_hdrlen; 1066 m->m_pkthdr.pace_tos = iptos; 1067 m->m_pkthdr.pace_lock = (uint8_t) ti_locked; 1068 if (tp->t_in_pkt == NULL) { 1069 tp->t_in_pkt = m; 1070 tp->t_tail_pkt = m; 1071 } else { 1072 tp->t_tail_pkt->m_nextpkt = m; 1073 tp->t_tail_pkt = m; 1074 } 1075 } 1076 1077 1078 int32_t 1079 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, 1080 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line){ 1081 struct tcp_hpts_entry *hpts; 1082 int32_t ret; 1083 1084 tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos, ti_locked); 1085 hpts = tcp_input_lock(tp->t_inpcb); 1086 ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line); 1087 mtx_unlock(&hpts->p_mtx); 1088 return (ret); 1089 } 1090 1091 void 1092 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line) 1093 { 1094 struct tcp_hpts_entry *hpts; 1095 struct tcpcb *tp; 1096 1097 tp = intotcpcb(inp); 1098 hpts = tcp_input_lock(tp->t_inpcb); 1099 if (inp->inp_in_input == 0) { 1100 /* Ok we need to set it on the hpts in the current slot */ 1101 hpts_sane_input_insert(hpts, inp, line); 1102 if (hpts->p_hpts_active == 0) { 1103 /* 1104 * Activate the hpts if it is sleeping. 1105 */ 1106 hpts->p_direct_wake = 1; 1107 tcp_wakeinput(hpts); 1108 } 1109 } else if (hpts->p_hpts_active == 0) { 1110 hpts->p_direct_wake = 1; 1111 tcp_wakeinput(hpts); 1112 } 1113 inp->inp_hpts_drop_reas = reason; 1114 mtx_unlock(&hpts->p_mtx); 1115 } 1116 1117 static uint16_t 1118 hpts_random_cpu(struct inpcb *inp){ 1119 /* 1120 * No flow type set distribute the load randomly. 1121 */ 1122 uint16_t cpuid; 1123 uint32_t ran; 1124 1125 /* 1126 * If one has been set use it i.e. we want both in and out on the 1127 * same hpts. 1128 */ 1129 if (inp->inp_input_cpu_set) { 1130 return (inp->inp_input_cpu); 1131 } else if (inp->inp_hpts_cpu_set) { 1132 return (inp->inp_hpts_cpu); 1133 } 1134 /* Nothing set use a random number */ 1135 ran = arc4random(); 1136 cpuid = (ran & 0xffff) % mp_ncpus; 1137 return (cpuid); 1138 } 1139 1140 static uint16_t 1141 hpts_cpuid(struct inpcb *inp){ 1142 uint16_t cpuid; 1143 1144 1145 /* 1146 * If one has been set use it i.e. we want both in and out on the 1147 * same hpts. 1148 */ 1149 if (inp->inp_input_cpu_set) { 1150 return (inp->inp_input_cpu); 1151 } else if (inp->inp_hpts_cpu_set) { 1152 return (inp->inp_hpts_cpu); 1153 } 1154 /* If one is set the other must be the same */ 1155 #ifdef RSS 1156 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); 1157 if (cpuid == NETISR_CPUID_NONE) 1158 return (hpts_random_cpu(inp)); 1159 else 1160 return (cpuid); 1161 #else 1162 /* 1163 * We don't have a flowid -> cpuid mapping, so cheat and just map 1164 * unknown cpuids to curcpu. Not the best, but apparently better 1165 * than defaulting to swi 0. 1166 */ 1167 if (inp->inp_flowtype != M_HASHTYPE_NONE) { 1168 cpuid = inp->inp_flowid % mp_ncpus; 1169 return (cpuid); 1170 } 1171 cpuid = hpts_random_cpu(inp); 1172 return (cpuid); 1173 #endif 1174 } 1175 1176 /* 1177 * Do NOT try to optimize the processing of inp's 1178 * by first pulling off all the inp's into a temporary 1179 * list (e.g. TAILQ_CONCAT). If you do that the subtle 1180 * interactions of switching CPU's will kill because of 1181 * problems in the linked list manipulation. Basically 1182 * you would switch cpu's with the hpts mutex locked 1183 * but then while you were processing one of the inp's 1184 * some other one that you switch will get a new 1185 * packet on the different CPU. It will insert it 1186 * on the new hptss input list. Creating a temporary 1187 * link in the inp will not fix it either, since 1188 * the other hpts will be doing the same thing and 1189 * you will both end up using the temporary link. 1190 * 1191 * You will die in an ASSERT for tailq corruption if you 1192 * run INVARIANTS or you will die horribly without 1193 * INVARIANTS in some unknown way with a corrupt linked 1194 * list. 1195 */ 1196 static void 1197 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv) 1198 { 1199 struct mbuf *m, *n; 1200 struct tcpcb *tp; 1201 struct inpcb *inp; 1202 uint16_t drop_reason; 1203 int16_t set_cpu; 1204 uint32_t did_prefetch = 0; 1205 int32_t ti_locked = TI_UNLOCKED; 1206 1207 HPTS_MTX_ASSERT(hpts); 1208 while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) { 1209 HPTS_MTX_ASSERT(hpts); 1210 hpts_sane_input_remove(hpts, inp, 0); 1211 if (inp->inp_input_cpu_set == 0) { 1212 set_cpu = 1; 1213 } else { 1214 set_cpu = 0; 1215 } 1216 hpts->p_inp = inp; 1217 drop_reason = inp->inp_hpts_drop_reas; 1218 inp->inp_in_input = 0; 1219 mtx_unlock(&hpts->p_mtx); 1220 if (drop_reason) { 1221 INP_INFO_RLOCK(&V_tcbinfo); 1222 ti_locked = TI_RLOCKED; 1223 } else { 1224 ti_locked = TI_UNLOCKED; 1225 } 1226 INP_WLOCK(inp); 1227 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 1228 (inp->inp_flags2 & INP_FREED)) { 1229 out: 1230 hpts->p_inp = NULL; 1231 if (ti_locked == TI_RLOCKED) { 1232 INP_INFO_RUNLOCK(&V_tcbinfo); 1233 } 1234 if (in_pcbrele_wlocked(inp) == 0) { 1235 INP_WUNLOCK(inp); 1236 } 1237 ti_locked = TI_UNLOCKED; 1238 mtx_lock(&hpts->p_mtx); 1239 continue; 1240 } 1241 tp = intotcpcb(inp); 1242 if ((tp == NULL) || (tp->t_inpcb == NULL)) { 1243 goto out; 1244 } 1245 if (drop_reason) { 1246 /* This tcb is being destroyed for drop_reason */ 1247 m = tp->t_in_pkt; 1248 if (m) 1249 n = m->m_nextpkt; 1250 else 1251 n = NULL; 1252 tp->t_in_pkt = NULL; 1253 while (m) { 1254 m_freem(m); 1255 m = n; 1256 if (m) 1257 n = m->m_nextpkt; 1258 } 1259 tp = tcp_drop(tp, drop_reason); 1260 INP_INFO_RUNLOCK(&V_tcbinfo); 1261 if (tp == NULL) { 1262 INP_WLOCK(inp); 1263 } 1264 if (in_pcbrele_wlocked(inp) == 0) 1265 INP_WUNLOCK(inp); 1266 mtx_lock(&hpts->p_mtx); 1267 continue; 1268 } 1269 if (set_cpu) { 1270 /* 1271 * Setup so the next time we will move to the right 1272 * CPU. This should be a rare event. It will 1273 * sometimes happens when we are the client side 1274 * (usually not the server). Somehow tcp_output() 1275 * gets called before the tcp_do_segment() sets the 1276 * intial state. This means the r_cpu and r_hpts_cpu 1277 * is 0. We get on the hpts, and then tcp_input() 1278 * gets called setting up the r_cpu to the correct 1279 * value. The hpts goes off and sees the mis-match. 1280 * We simply correct it here and the CPU will switch 1281 * to the new hpts nextime the tcb gets added to the 1282 * the hpts (not this time) :-) 1283 */ 1284 tcp_set_hpts(inp); 1285 } 1286 CURVNET_SET(tp->t_vnet); 1287 m = tp->t_in_pkt; 1288 n = NULL; 1289 if (m != NULL && 1290 (m->m_pkthdr.pace_lock == TI_RLOCKED || 1291 tp->t_state != TCPS_ESTABLISHED)) { 1292 ti_locked = TI_RLOCKED; 1293 if (tcp_hptsi_lock_inpinfo(inp, &tp)) { 1294 CURVNET_RESTORE(); 1295 goto out; 1296 } 1297 m = tp->t_in_pkt; 1298 } 1299 if (in_newts_every_tcb) { 1300 if (in_ts_percision) 1301 microuptime(tv); 1302 else 1303 getmicrouptime(tv); 1304 } 1305 if (tp->t_fb_ptr != NULL) { 1306 kern_prefetch(tp->t_fb_ptr, &did_prefetch); 1307 did_prefetch = 1; 1308 } 1309 /* Any input work to do, if so do it first */ 1310 if ((m != NULL) && (m == tp->t_in_pkt)) { 1311 struct tcphdr *th; 1312 int32_t tlen, drop_hdrlen, nxt_pkt; 1313 uint8_t iptos; 1314 1315 n = m->m_nextpkt; 1316 tp->t_in_pkt = tp->t_tail_pkt = NULL; 1317 while (m) { 1318 th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff); 1319 tlen = m->m_pkthdr.pace_tlen; 1320 drop_hdrlen = m->m_pkthdr.pace_drphdrlen; 1321 iptos = m->m_pkthdr.pace_tos; 1322 m->m_nextpkt = NULL; 1323 if (n) 1324 nxt_pkt = 1; 1325 else 1326 nxt_pkt = 0; 1327 inp->inp_input_calls = 1; 1328 if (tp->t_fb->tfb_tcp_hpts_do_segment) { 1329 /* Use the hpts specific do_segment */ 1330 (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket, 1331 tp, drop_hdrlen, 1332 tlen, iptos, ti_locked, nxt_pkt, tv); 1333 } else { 1334 /* Use the default do_segment */ 1335 (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket, 1336 tp, drop_hdrlen, 1337 tlen, iptos, ti_locked); 1338 } 1339 /* 1340 * Do segment returns unlocked we need the 1341 * lock again but we also need some kasserts 1342 * here. 1343 */ 1344 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 1345 INP_UNLOCK_ASSERT(inp); 1346 m = n; 1347 if (m) 1348 n = m->m_nextpkt; 1349 if (m != NULL && 1350 m->m_pkthdr.pace_lock == TI_RLOCKED) { 1351 INP_INFO_RLOCK(&V_tcbinfo); 1352 ti_locked = TI_RLOCKED; 1353 } else 1354 ti_locked = TI_UNLOCKED; 1355 INP_WLOCK(inp); 1356 /* 1357 * Since we have an opening here we must 1358 * re-check if the tcb went away while we 1359 * were getting the lock(s). 1360 */ 1361 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 1362 (inp->inp_flags2 & INP_FREED)) { 1363 out_free: 1364 while (m) { 1365 m_freem(m); 1366 m = n; 1367 if (m) 1368 n = m->m_nextpkt; 1369 } 1370 CURVNET_RESTORE(); 1371 goto out; 1372 } 1373 /* 1374 * Now that we hold the INP lock, check if 1375 * we need to upgrade our lock. 1376 */ 1377 if (ti_locked == TI_UNLOCKED && 1378 (tp->t_state != TCPS_ESTABLISHED)) { 1379 ti_locked = TI_RLOCKED; 1380 if (tcp_hptsi_lock_inpinfo(inp, &tp)) 1381 goto out_free; 1382 } 1383 } /** end while(m) */ 1384 } /** end if ((m != NULL) && (m == tp->t_in_pkt)) */ 1385 if (in_pcbrele_wlocked(inp) == 0) 1386 INP_WUNLOCK(inp); 1387 if (ti_locked == TI_RLOCKED) 1388 INP_INFO_RUNLOCK(&V_tcbinfo); 1389 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 1390 INP_UNLOCK_ASSERT(inp); 1391 ti_locked = TI_UNLOCKED; 1392 mtx_lock(&hpts->p_mtx); 1393 hpts->p_inp = NULL; 1394 CURVNET_RESTORE(); 1395 } 1396 } 1397 1398 static int 1399 tcp_hpts_est_run(struct tcp_hpts_entry *hpts) 1400 { 1401 int32_t ticks_to_run; 1402 1403 if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) { 1404 ticks_to_run = hpts->p_curtick - hpts->p_prevtick; 1405 if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) { 1406 ticks_to_run = NUM_OF_HPTSI_SLOTS - 2; 1407 } 1408 } else { 1409 if (hpts->p_prevtick == hpts->p_curtick) { 1410 /* This happens when we get woken up right away */ 1411 return (-1); 1412 } 1413 ticks_to_run = 1; 1414 } 1415 /* Set in where we will be when we catch up */ 1416 hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS; 1417 if (hpts->p_nxt_slot == hpts->p_cur_slot) { 1418 panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d", 1419 hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run); 1420 } 1421 return (ticks_to_run); 1422 } 1423 1424 static void 1425 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick) 1426 { 1427 struct tcpcb *tp; 1428 struct inpcb *inp = NULL, *ninp; 1429 struct timeval tv; 1430 int32_t ticks_to_run, i, error, tick_now, interum_tick; 1431 int32_t paced_cnt = 0; 1432 int32_t did_prefetch = 0; 1433 int32_t prefetch_ninp = 0; 1434 int32_t prefetch_tp = 0; 1435 uint32_t cts; 1436 int16_t set_cpu; 1437 1438 HPTS_MTX_ASSERT(hpts); 1439 hpts->p_curtick = tcp_tv_to_hptstick(ctick); 1440 cts = tcp_tv_to_usectick(ctick); 1441 memcpy(&tv, ctick, sizeof(struct timeval)); 1442 hpts->p_cur_slot = hpts_tick(hpts, 1); 1443 1444 /* Figure out if we had missed ticks */ 1445 again: 1446 HPTS_MTX_ASSERT(hpts); 1447 ticks_to_run = tcp_hpts_est_run(hpts); 1448 if (!TAILQ_EMPTY(&hpts->p_input)) { 1449 tcp_input_data(hpts, &tv); 1450 } 1451 #ifdef INVARIANTS 1452 if (TAILQ_EMPTY(&hpts->p_input) && 1453 (hpts->p_on_inqueue_cnt != 0)) { 1454 panic("tp:%p in_hpts input empty but cnt:%d", 1455 hpts, hpts->p_on_inqueue_cnt); 1456 } 1457 #endif 1458 HPTS_MTX_ASSERT(hpts); 1459 /* Reset the ticks to run and time if we need too */ 1460 interum_tick = tcp_gethptstick(&tv); 1461 if (interum_tick != hpts->p_curtick) { 1462 /* Save off the new time we execute to */ 1463 *ctick = tv; 1464 hpts->p_curtick = interum_tick; 1465 cts = tcp_tv_to_usectick(&tv); 1466 hpts->p_cur_slot = hpts_tick(hpts, 1); 1467 ticks_to_run = tcp_hpts_est_run(hpts); 1468 } 1469 if (ticks_to_run == -1) { 1470 goto no_run; 1471 } 1472 if (logging_on) { 1473 tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0); 1474 } 1475 if (hpts->p_on_queue_cnt == 0) { 1476 goto no_one; 1477 } 1478 HPTS_MTX_ASSERT(hpts); 1479 for (i = 0; i < ticks_to_run; i++) { 1480 /* 1481 * Calculate our delay, if there are no extra ticks there 1482 * was not any 1483 */ 1484 hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC; 1485 HPTS_MTX_ASSERT(hpts); 1486 while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) { 1487 /* For debugging */ 1488 if (logging_on) { 1489 tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i); 1490 } 1491 hpts->p_inp = inp; 1492 paced_cnt++; 1493 if (hpts->p_cur_slot != inp->inp_hptsslot) { 1494 panic("Hpts:%p inp:%p slot mis-aligned %u vs %u", 1495 hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot); 1496 } 1497 /* Now pull it */ 1498 if (inp->inp_hpts_cpu_set == 0) { 1499 set_cpu = 1; 1500 } else { 1501 set_cpu = 0; 1502 } 1503 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0); 1504 if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) { 1505 /* We prefetch the next inp if possible */ 1506 kern_prefetch(ninp, &prefetch_ninp); 1507 prefetch_ninp = 1; 1508 } 1509 if (inp->inp_hpts_request) { 1510 /* 1511 * This guy is deferred out further in time 1512 * then our wheel had on it. Push him back 1513 * on the wheel. 1514 */ 1515 int32_t remaining_slots; 1516 1517 remaining_slots = ticks_to_run - (i + 1); 1518 if (inp->inp_hpts_request > remaining_slots) { 1519 /* 1520 * Keep INVARIANTS happy by clearing 1521 * the flag 1522 */ 1523 tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1); 1524 hpts->p_inp = NULL; 1525 continue; 1526 } 1527 inp->inp_hpts_request = 0; 1528 } 1529 /* 1530 * We clear the hpts flag here after dealing with 1531 * remaining slots. This way anyone looking with the 1532 * TCB lock will see its on the hpts until just 1533 * before we unlock. 1534 */ 1535 inp->inp_in_hpts = 0; 1536 mtx_unlock(&hpts->p_mtx); 1537 INP_WLOCK(inp); 1538 if (in_pcbrele_wlocked(inp)) { 1539 mtx_lock(&hpts->p_mtx); 1540 if (logging_on) 1541 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1); 1542 hpts->p_inp = NULL; 1543 continue; 1544 } 1545 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 1546 out_now: 1547 #ifdef INVARIANTS 1548 if (mtx_owned(&hpts->p_mtx)) { 1549 panic("Hpts:%p owns mtx prior-to lock line:%d", 1550 hpts, __LINE__); 1551 } 1552 #endif 1553 INP_WUNLOCK(inp); 1554 mtx_lock(&hpts->p_mtx); 1555 if (logging_on) 1556 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3); 1557 hpts->p_inp = NULL; 1558 continue; 1559 } 1560 tp = intotcpcb(inp); 1561 if ((tp == NULL) || (tp->t_inpcb == NULL)) { 1562 goto out_now; 1563 } 1564 if (set_cpu) { 1565 /* 1566 * Setup so the next time we will move to 1567 * the right CPU. This should be a rare 1568 * event. It will sometimes happens when we 1569 * are the client side (usually not the 1570 * server). Somehow tcp_output() gets called 1571 * before the tcp_do_segment() sets the 1572 * intial state. This means the r_cpu and 1573 * r_hpts_cpu is 0. We get on the hpts, and 1574 * then tcp_input() gets called setting up 1575 * the r_cpu to the correct value. The hpts 1576 * goes off and sees the mis-match. We 1577 * simply correct it here and the CPU will 1578 * switch to the new hpts nextime the tcb 1579 * gets added to the the hpts (not this one) 1580 * :-) 1581 */ 1582 tcp_set_hpts(inp); 1583 } 1584 if (out_newts_every_tcb) { 1585 struct timeval sv; 1586 1587 if (out_ts_percision) 1588 microuptime(&sv); 1589 else 1590 getmicrouptime(&sv); 1591 cts = tcp_tv_to_usectick(&sv); 1592 } 1593 CURVNET_SET(tp->t_vnet); 1594 /* 1595 * There is a hole here, we get the refcnt on the 1596 * inp so it will still be preserved but to make 1597 * sure we can get the INP we need to hold the p_mtx 1598 * above while we pull out the tp/inp, as long as 1599 * fini gets the lock first we are assured of having 1600 * a sane INP we can lock and test. 1601 */ 1602 #ifdef INVARIANTS 1603 if (mtx_owned(&hpts->p_mtx)) { 1604 panic("Hpts:%p owns mtx before tcp-output:%d", 1605 hpts, __LINE__); 1606 } 1607 #endif 1608 if (tp->t_fb_ptr != NULL) { 1609 kern_prefetch(tp->t_fb_ptr, &did_prefetch); 1610 did_prefetch = 1; 1611 } 1612 inp->inp_hpts_calls = 1; 1613 if (tp->t_fb->tfb_tcp_output_wtime != NULL) { 1614 error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv); 1615 } else { 1616 error = tp->t_fb->tfb_tcp_output(tp); 1617 } 1618 if (ninp && ninp->inp_ppcb) { 1619 /* 1620 * If we have a nxt inp, see if we can 1621 * prefetch its ppcb. Note this may seem 1622 * "risky" since we have no locks (other 1623 * than the previous inp) and there no 1624 * assurance that ninp was not pulled while 1625 * we were processing inp and freed. If this 1626 * occured it could mean that either: 1627 * 1628 * a) Its NULL (which is fine we won't go 1629 * here) <or> b) Its valid (which is cool we 1630 * will prefetch it) <or> c) The inp got 1631 * freed back to the slab which was 1632 * reallocated. Then the piece of memory was 1633 * re-used and something else (not an 1634 * address) is in inp_ppcb. If that occurs 1635 * we don't crash, but take a TLB shootdown 1636 * performance hit (same as if it was NULL 1637 * and we tried to pre-fetch it). 1638 * 1639 * Considering that the likelyhood of <c> is 1640 * quite rare we will take a risk on doing 1641 * this. If performance drops after testing 1642 * we can always take this out. NB: the 1643 * kern_prefetch on amd64 actually has 1644 * protection against a bad address now via 1645 * the DMAP_() tests. This will prevent the 1646 * TLB hit, and instead if <c> occurs just 1647 * cause us to load cache with a useless 1648 * address (to us). 1649 */ 1650 kern_prefetch(ninp->inp_ppcb, &prefetch_tp); 1651 prefetch_tp = 1; 1652 } 1653 INP_WUNLOCK(inp); 1654 INP_UNLOCK_ASSERT(inp); 1655 CURVNET_RESTORE(); 1656 #ifdef INVARIANTS 1657 if (mtx_owned(&hpts->p_mtx)) { 1658 panic("Hpts:%p owns mtx prior-to lock line:%d", 1659 hpts, __LINE__); 1660 } 1661 #endif 1662 mtx_lock(&hpts->p_mtx); 1663 if (logging_on) 1664 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4); 1665 hpts->p_inp = NULL; 1666 } 1667 HPTS_MTX_ASSERT(hpts); 1668 hpts->p_inp = NULL; 1669 hpts->p_cur_slot++; 1670 if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) { 1671 hpts->p_cur_slot = 0; 1672 } 1673 } 1674 no_one: 1675 HPTS_MTX_ASSERT(hpts); 1676 hpts->p_prevtick = hpts->p_curtick; 1677 hpts->p_delayed_by = 0; 1678 /* 1679 * Check to see if we took an excess amount of time and need to run 1680 * more ticks (if we did not hit eno-bufs). 1681 */ 1682 /* Re-run any input that may be there */ 1683 (void)tcp_gethptstick(&tv); 1684 if (!TAILQ_EMPTY(&hpts->p_input)) { 1685 tcp_input_data(hpts, &tv); 1686 } 1687 #ifdef INVARIANTS 1688 if (TAILQ_EMPTY(&hpts->p_input) && 1689 (hpts->p_on_inqueue_cnt != 0)) { 1690 panic("tp:%p in_hpts input empty but cnt:%d", 1691 hpts, hpts->p_on_inqueue_cnt); 1692 } 1693 #endif 1694 tick_now = tcp_gethptstick(&tv); 1695 if (SEQ_GT(tick_now, hpts->p_prevtick)) { 1696 struct timeval res; 1697 1698 /* Did we really spend a full tick or more in here? */ 1699 timersub(&tv, ctick, &res); 1700 if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) { 1701 counter_u64_add(hpts_loops, 1); 1702 if (logging_on) { 1703 tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now); 1704 } 1705 *ctick = res; 1706 hpts->p_curtick = tick_now; 1707 goto again; 1708 } 1709 } 1710 no_run: 1711 { 1712 uint32_t t = 0, i, fnd = 0; 1713 1714 if (hpts->p_on_queue_cnt) { 1715 1716 1717 /* 1718 * Find next slot that is occupied and use that to 1719 * be the sleep time. 1720 */ 1721 for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) { 1722 if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) { 1723 fnd = 1; 1724 break; 1725 } 1726 t = (t + 1) % NUM_OF_HPTSI_SLOTS; 1727 } 1728 if (fnd) { 1729 hpts->p_hpts_sleep_time = i; 1730 } else { 1731 counter_u64_add(back_tosleep, 1); 1732 #ifdef INVARIANTS 1733 panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt); 1734 #endif 1735 hpts->p_on_queue_cnt = 0; 1736 goto non_found; 1737 } 1738 t++; 1739 } else { 1740 /* No one on the wheel sleep for all but 2 slots */ 1741 non_found: 1742 if (hpts_sleep_max == 0) 1743 hpts_sleep_max = 1; 1744 hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max); 1745 t = 0; 1746 } 1747 if (logging_on) { 1748 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC)); 1749 } 1750 } 1751 } 1752 1753 void 1754 __tcp_set_hpts(struct inpcb *inp, int32_t line) 1755 { 1756 struct tcp_hpts_entry *hpts; 1757 1758 INP_WLOCK_ASSERT(inp); 1759 hpts = tcp_hpts_lock(inp); 1760 if ((inp->inp_in_hpts == 0) && 1761 (inp->inp_hpts_cpu_set == 0)) { 1762 inp->inp_hpts_cpu = hpts_cpuid(inp); 1763 inp->inp_hpts_cpu_set = 1; 1764 } 1765 mtx_unlock(&hpts->p_mtx); 1766 hpts = tcp_input_lock(inp); 1767 if ((inp->inp_input_cpu_set == 0) && 1768 (inp->inp_in_input == 0)) { 1769 inp->inp_input_cpu = hpts_cpuid(inp); 1770 inp->inp_input_cpu_set = 1; 1771 } 1772 mtx_unlock(&hpts->p_mtx); 1773 } 1774 1775 uint16_t 1776 tcp_hpts_delayedby(struct inpcb *inp){ 1777 return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by); 1778 } 1779 1780 static void 1781 tcp_hpts_thread(void *ctx) 1782 { 1783 struct tcp_hpts_entry *hpts; 1784 struct timeval tv; 1785 sbintime_t sb; 1786 1787 hpts = (struct tcp_hpts_entry *)ctx; 1788 mtx_lock(&hpts->p_mtx); 1789 if (hpts->p_direct_wake) { 1790 /* Signaled by input */ 1791 if (logging_on) 1792 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1); 1793 callout_stop(&hpts->co); 1794 } else { 1795 /* Timed out */ 1796 if (callout_pending(&hpts->co) || 1797 !callout_active(&hpts->co)) { 1798 if (logging_on) 1799 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2); 1800 mtx_unlock(&hpts->p_mtx); 1801 return; 1802 } 1803 callout_deactivate(&hpts->co); 1804 if (logging_on) 1805 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3); 1806 } 1807 hpts->p_hpts_active = 1; 1808 (void)tcp_gethptstick(&tv); 1809 tcp_hptsi(hpts, &tv); 1810 HPTS_MTX_ASSERT(hpts); 1811 tv.tv_sec = 0; 1812 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC; 1813 if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) { 1814 tv.tv_usec = tcp_min_hptsi_time; 1815 hpts->p_on_min_sleep = 1; 1816 } else { 1817 /* Clear the min sleep flag */ 1818 hpts->p_on_min_sleep = 0; 1819 } 1820 hpts->p_hpts_active = 0; 1821 sb = tvtosbt(tv); 1822 if (tcp_hpts_callout_skip_swi == 0) { 1823 callout_reset_sbt_on(&hpts->co, sb, 0, 1824 hpts_timeout_swi, hpts, hpts->p_cpu, 1825 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1826 } else { 1827 callout_reset_sbt_on(&hpts->co, sb, 0, 1828 hpts_timeout_dir, hpts, 1829 hpts->p_cpu, 1830 C_PREL(tcp_hpts_precision)); 1831 } 1832 hpts->p_direct_wake = 0; 1833 mtx_unlock(&hpts->p_mtx); 1834 } 1835 1836 #undef timersub 1837 1838 static void 1839 tcp_init_hptsi(void *st) 1840 { 1841 int32_t i, j, error, bound = 0, created = 0; 1842 size_t sz, asz; 1843 struct timeval tv; 1844 sbintime_t sb; 1845 struct tcp_hpts_entry *hpts; 1846 char unit[16]; 1847 uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU; 1848 1849 tcp_pace.rp_proc = NULL; 1850 tcp_pace.rp_num_hptss = ncpus; 1851 hpts_loops = counter_u64_alloc(M_WAITOK); 1852 back_tosleep = counter_u64_alloc(M_WAITOK); 1853 1854 sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *)); 1855 tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); 1856 asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS; 1857 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 1858 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry), 1859 M_TCPHPTS, M_WAITOK | M_ZERO); 1860 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, 1861 M_TCPHPTS, M_WAITOK); 1862 hpts = tcp_pace.rp_ent[i]; 1863 /* 1864 * Init all the hpts structures that are not specifically 1865 * zero'd by the allocations. Also lets attach them to the 1866 * appropriate sysctl block as well. 1867 */ 1868 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", 1869 "hpts", MTX_DEF | MTX_DUPOK); 1870 TAILQ_INIT(&hpts->p_input); 1871 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) { 1872 TAILQ_INIT(&hpts->p_hptss[j]); 1873 } 1874 sysctl_ctx_init(&hpts->hpts_ctx); 1875 sprintf(unit, "%d", i); 1876 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, 1877 SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts), 1878 OID_AUTO, 1879 unit, 1880 CTLFLAG_RW, 0, 1881 ""); 1882 SYSCTL_ADD_INT(&hpts->hpts_ctx, 1883 SYSCTL_CHILDREN(hpts->hpts_root), 1884 OID_AUTO, "in_qcnt", CTLFLAG_RD, 1885 &hpts->p_on_inqueue_cnt, 0, 1886 "Count TCB's awaiting input processing"); 1887 SYSCTL_ADD_INT(&hpts->hpts_ctx, 1888 SYSCTL_CHILDREN(hpts->hpts_root), 1889 OID_AUTO, "out_qcnt", CTLFLAG_RD, 1890 &hpts->p_on_queue_cnt, 0, 1891 "Count TCB's awaiting output processing"); 1892 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1893 SYSCTL_CHILDREN(hpts->hpts_root), 1894 OID_AUTO, "active", CTLFLAG_RD, 1895 &hpts->p_hpts_active, 0, 1896 "Is the hpts active"); 1897 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1898 SYSCTL_CHILDREN(hpts->hpts_root), 1899 OID_AUTO, "curslot", CTLFLAG_RD, 1900 &hpts->p_cur_slot, 0, 1901 "What the current slot is if active"); 1902 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1903 SYSCTL_CHILDREN(hpts->hpts_root), 1904 OID_AUTO, "curtick", CTLFLAG_RD, 1905 &hpts->p_curtick, 0, 1906 "What the current tick on if active"); 1907 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1908 SYSCTL_CHILDREN(hpts->hpts_root), 1909 OID_AUTO, "logsize", CTLFLAG_RD, 1910 &hpts->p_logsize, 0, 1911 "Hpts logging buffer size"); 1912 hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2; 1913 hpts->p_num = i; 1914 hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv); 1915 hpts->p_prevtick -= 1; 1916 hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS; 1917 hpts->p_cpu = 0xffff; 1918 hpts->p_nxt_slot = 1; 1919 hpts->p_logsize = tcp_hpts_logging_size; 1920 if (hpts->p_logsize) { 1921 sz = (sizeof(struct hpts_log) * hpts->p_logsize); 1922 hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); 1923 } 1924 callout_init(&hpts->co, 1); 1925 } 1926 /* 1927 * Now lets start ithreads to handle the hptss. 1928 */ 1929 CPU_FOREACH(i) { 1930 hpts = tcp_pace.rp_ent[i]; 1931 hpts->p_cpu = i; 1932 error = swi_add(&hpts->ie, "hpts", 1933 tcp_hpts_thread, (void *)hpts, 1934 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); 1935 if (error) { 1936 panic("Can't add hpts:%p i:%d err:%d", 1937 hpts, i, error); 1938 } 1939 created++; 1940 if (tcp_bind_threads) { 1941 if (intr_event_bind(hpts->ie, i) == 0) 1942 bound++; 1943 } 1944 tv.tv_sec = 0; 1945 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC; 1946 sb = tvtosbt(tv); 1947 if (tcp_hpts_callout_skip_swi == 0) { 1948 callout_reset_sbt_on(&hpts->co, sb, 0, 1949 hpts_timeout_swi, hpts, hpts->p_cpu, 1950 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1951 } else { 1952 callout_reset_sbt_on(&hpts->co, sb, 0, 1953 hpts_timeout_dir, hpts, 1954 hpts->p_cpu, 1955 C_PREL(tcp_hpts_precision)); 1956 } 1957 } 1958 printf("TCP Hpts created %d swi interrupt thread and bound %d\n", 1959 created, bound); 1960 return; 1961 } 1962 1963 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL); 1964