1 /*- 2 * Copyright (c) 2016-8 3 * Netflix Inc. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_inet.h" 31 #include "opt_inet6.h" 32 #include "opt_ipsec.h" 33 #include "opt_tcpdebug.h" 34 /** 35 * Some notes about usage. 36 * 37 * The tcp_hpts system is designed to provide a high precision timer 38 * system for tcp. Its main purpose is to provide a mechanism for 39 * pacing packets out onto the wire. It can be used in two ways 40 * by a given TCP stack (and those two methods can be used simultaneously). 41 * 42 * First, and probably the main thing its used by Rack and BBR for, it can 43 * be used to call tcp_output() of a transport stack at some time in the future. 44 * The normal way this is done is that tcp_output() of the stack schedules 45 * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The 46 * slot is the time from now that the stack wants to be called but it 47 * must be converted to tcp_hpts's notion of slot. This is done with 48 * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical 49 * call from the tcp_output() routine might look like: 50 * 51 * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550)); 52 * 53 * The above would schedule tcp_ouput() to be called in 550 useconds. 54 * Note that if using this mechanism the stack will want to add near 55 * its top a check to prevent unwanted calls (from user land or the 56 * arrival of incoming ack's). So it would add something like: 57 * 58 * if (inp->inp_in_hpts) 59 * return; 60 * 61 * to prevent output processing until the time alotted has gone by. 62 * Of course this is a bare bones example and the stack will probably 63 * have more consideration then just the above. 64 * 65 * Now the tcp_hpts system will call tcp_output in one of two forms, 66 * it will first check to see if the stack as defined a 67 * tfb_tcp_output_wtime() function, if so that is the routine it 68 * will call, if that function is not defined then it will call the 69 * tfb_tcp_output() function. The only difference between these 70 * two calls is that the former passes the time in to the function 71 * so the function does not have to access the time (which tcp_hpts 72 * already has). What these functions do is of course totally up 73 * to the individual tcp stack. 74 * 75 * Now the second function (actually two functions I guess :D) 76 * the tcp_hpts system provides is the ability to either abort 77 * a connection (later) or process input on a connection. 78 * Why would you want to do this? To keep processor locality. 79 * 80 * So in order to use the input redirection function the 81 * stack changes its tcp_do_segment() routine to instead 82 * of process the data call the function: 83 * 84 * tcp_queue_pkt_to_input() 85 * 86 * You will note that the arguments to this function look 87 * a lot like tcp_do_segments's arguments. This function 88 * will assure that the tcp_hpts system will 89 * call the functions tfb_tcp_hpts_do_segment() from the 90 * correct CPU. Note that multiple calls can get pushed 91 * into the tcp_hpts system this will be indicated by 92 * the next to last argument to tfb_tcp_hpts_do_segment() 93 * (nxt_pkt). If nxt_pkt is a 1 then another packet is 94 * coming. If nxt_pkt is a 0 then this is the last call 95 * that the tcp_hpts system has available for the tcp stack. 96 * 97 * The other point of the input system is to be able to safely 98 * drop a tcp connection without worrying about the recursive 99 * locking that may be occuring on the INP_WLOCK. So if 100 * a stack wants to drop a connection it calls: 101 * 102 * tcp_set_inp_to_drop(tp, ETIMEDOUT) 103 * 104 * To schedule the tcp_hpts system to call 105 * 106 * tcp_drop(tp, drop_reason) 107 * 108 * at a future point. This is quite handy to prevent locking 109 * issues when dropping connections. 110 * 111 */ 112 113 #include <sys/param.h> 114 #include <sys/bus.h> 115 #include <sys/interrupt.h> 116 #include <sys/module.h> 117 #include <sys/kernel.h> 118 #include <sys/hhook.h> 119 #include <sys/malloc.h> 120 #include <sys/mbuf.h> 121 #include <sys/proc.h> /* for proc0 declaration */ 122 #include <sys/socket.h> 123 #include <sys/socketvar.h> 124 #include <sys/sysctl.h> 125 #include <sys/systm.h> 126 #include <sys/refcount.h> 127 #include <sys/sched.h> 128 #include <sys/queue.h> 129 #include <sys/smp.h> 130 #include <sys/counter.h> 131 #include <sys/time.h> 132 #include <sys/kthread.h> 133 #include <sys/kern_prefetch.h> 134 135 #include <vm/uma.h> 136 137 #include <net/route.h> 138 #include <net/vnet.h> 139 140 #define TCPSTATES /* for logging */ 141 142 #include <netinet/in.h> 143 #include <netinet/in_kdtrace.h> 144 #include <netinet/in_pcb.h> 145 #include <netinet/ip.h> 146 #include <netinet/ip_icmp.h> /* required for icmp_var.h */ 147 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */ 148 #include <netinet/ip_var.h> 149 #include <netinet/ip6.h> 150 #include <netinet6/in6_pcb.h> 151 #include <netinet6/ip6_var.h> 152 #define TCPOUTFLAGS 153 #include <netinet/tcp.h> 154 #include <netinet/tcp_fsm.h> 155 #include <netinet/tcp_seq.h> 156 #include <netinet/tcp_timer.h> 157 #include <netinet/tcp_var.h> 158 #include <netinet/tcpip.h> 159 #include <netinet/cc/cc.h> 160 #include <netinet/tcp_hpts.h> 161 162 #ifdef tcpdebug 163 #include <netinet/tcp_debug.h> 164 #endif /* tcpdebug */ 165 #ifdef tcp_offload 166 #include <netinet/tcp_offload.h> 167 #endif 168 169 #ifdef ipsec 170 #include <netipsec/ipsec.h> 171 #include <netipsec/ipsec6.h> 172 #endif /* ipsec */ 173 #include "opt_rss.h" 174 175 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts"); 176 #ifdef RSS 177 static int tcp_bind_threads = 1; 178 #else 179 static int tcp_bind_threads = 0; 180 #endif 181 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads); 182 183 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG; 184 185 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size); 186 187 static struct tcp_hptsi tcp_pace; 188 189 static int 190 tcp_hptsi_lock_inpinfo(struct inpcb *inp, 191 struct tcpcb **tp); 192 static void tcp_wakehpts(struct tcp_hpts_entry *p); 193 static void tcp_wakeinput(struct tcp_hpts_entry *p); 194 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv); 195 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick); 196 static void tcp_hpts_thread(void *ctx); 197 static void tcp_init_hptsi(void *st); 198 199 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP; 200 static int32_t tcp_hpts_callout_skip_swi = 0; 201 202 SYSCTL_DECL(_net_inet_tcp); 203 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls"); 204 205 #define timersub(tvp, uvp, vvp) \ 206 do { \ 207 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ 208 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ 209 if ((vvp)->tv_usec < 0) { \ 210 (vvp)->tv_sec--; \ 211 (vvp)->tv_usec += 1000000; \ 212 } \ 213 } while (0) 214 215 static int32_t logging_on = 0; 216 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2); 217 static int32_t tcp_hpts_precision = 120; 218 219 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW, 220 &tcp_hpts_precision, 120, 221 "Value for PRE() precision of callout"); 222 223 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW, 224 &logging_on, 0, 225 "Turn on logging if compiled in"); 226 227 counter_u64_t hpts_loops; 228 229 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD, 230 &hpts_loops, "Number of times hpts had to loop to catch up"); 231 232 counter_u64_t back_tosleep; 233 234 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD, 235 &back_tosleep, "Number of times hpts found no tcbs"); 236 237 static int32_t in_newts_every_tcb = 0; 238 239 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW, 240 &in_newts_every_tcb, 0, 241 "Do we have a new cts every tcb we process for input"); 242 static int32_t in_ts_percision = 0; 243 244 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW, 245 &in_ts_percision, 0, 246 "Do we use percise timestamp for clients on input"); 247 static int32_t out_newts_every_tcb = 0; 248 249 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW, 250 &out_newts_every_tcb, 0, 251 "Do we have a new cts every tcb we process for output"); 252 static int32_t out_ts_percision = 0; 253 254 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW, 255 &out_ts_percision, 0, 256 "Do we use a percise timestamp for every output cts"); 257 258 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW, 259 &hpts_sleep_max, 0, 260 "The maximum time the hpts will sleep <1 - 254>"); 261 262 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW, 263 &tcp_min_hptsi_time, 0, 264 "The minimum time the hpts must sleep before processing more slots"); 265 266 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW, 267 &tcp_hpts_callout_skip_swi, 0, 268 "Do we have the callout call directly to the hpts?"); 269 270 static void 271 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot, 272 uint32_t ticknow, int32_t line) 273 { 274 struct hpts_log *pl; 275 276 HPTS_MTX_ASSERT(hpts); 277 if (hpts->p_log == NULL) 278 return; 279 pl = &hpts->p_log[hpts->p_log_at]; 280 hpts->p_log_at++; 281 if (hpts->p_log_at >= hpts->p_logsize) { 282 hpts->p_log_at = 0; 283 hpts->p_log_wrapped = 1; 284 } 285 pl->inp = inp; 286 if (inp) { 287 pl->t_paceslot = inp->inp_hptsslot; 288 pl->t_hptsreq = inp->inp_hpts_request; 289 pl->p_onhpts = inp->inp_in_hpts; 290 pl->p_oninput = inp->inp_in_input; 291 } else { 292 pl->t_paceslot = 0; 293 pl->t_hptsreq = 0; 294 pl->p_onhpts = 0; 295 pl->p_oninput = 0; 296 } 297 pl->is_notempty = 1; 298 pl->event = event; 299 pl->line = line; 300 pl->cts = tcp_get_usecs(NULL); 301 pl->p_curtick = hpts->p_curtick; 302 pl->p_prevtick = hpts->p_prevtick; 303 pl->p_on_queue_cnt = hpts->p_on_queue_cnt; 304 pl->ticknow = ticknow; 305 pl->slot_req = slot; 306 pl->p_nxt_slot = hpts->p_nxt_slot; 307 pl->p_cur_slot = hpts->p_cur_slot; 308 pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time; 309 pl->p_flags = (hpts->p_cpu & 0x7f); 310 pl->p_flags <<= 7; 311 pl->p_flags |= (hpts->p_num & 0x7f); 312 pl->p_flags <<= 2; 313 if (hpts->p_hpts_active) { 314 pl->p_flags |= HPTS_HPTS_ACTIVE; 315 } 316 } 317 318 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__) 319 320 static void 321 hpts_timeout_swi(void *arg) 322 { 323 struct tcp_hpts_entry *hpts; 324 325 hpts = (struct tcp_hpts_entry *)arg; 326 swi_sched(hpts->ie_cookie, 0); 327 } 328 329 static void 330 hpts_timeout_dir(void *arg) 331 { 332 tcp_hpts_thread(arg); 333 } 334 335 static inline void 336 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear) 337 { 338 #ifdef INVARIANTS 339 if (mtx_owned(&hpts->p_mtx) == 0) { 340 /* We don't own the mutex? */ 341 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 342 } 343 if (hpts->p_cpu != inp->inp_hpts_cpu) { 344 /* It is not the right cpu/mutex? */ 345 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 346 } 347 if (inp->inp_in_hpts == 0) { 348 /* We are not on the hpts? */ 349 panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp); 350 } 351 if (TAILQ_EMPTY(head) && 352 (hpts->p_on_queue_cnt != 0)) { 353 /* We should not be empty with a queue count */ 354 panic("%s hpts:%p hpts bucket empty but cnt:%d", 355 __FUNCTION__, hpts, hpts->p_on_queue_cnt); 356 } 357 #endif 358 TAILQ_REMOVE(head, inp, inp_hpts); 359 hpts->p_on_queue_cnt--; 360 if (hpts->p_on_queue_cnt < 0) { 361 /* Count should not go negative .. */ 362 #ifdef INVARIANTS 363 panic("Hpts goes negative inp:%p hpts:%p", 364 inp, hpts); 365 #endif 366 hpts->p_on_queue_cnt = 0; 367 } 368 if (clear) { 369 inp->inp_hpts_request = 0; 370 inp->inp_in_hpts = 0; 371 } 372 } 373 374 static inline void 375 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref) 376 { 377 #ifdef INVARIANTS 378 if (mtx_owned(&hpts->p_mtx) == 0) { 379 /* We don't own the mutex? */ 380 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 381 } 382 if (hpts->p_cpu != inp->inp_hpts_cpu) { 383 /* It is not the right cpu/mutex? */ 384 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 385 } 386 if ((noref == 0) && (inp->inp_in_hpts == 1)) { 387 /* We are already on the hpts? */ 388 panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp); 389 } 390 #endif 391 TAILQ_INSERT_TAIL(head, inp, inp_hpts); 392 inp->inp_in_hpts = 1; 393 hpts->p_on_queue_cnt++; 394 if (noref == 0) { 395 in_pcbref(inp); 396 } 397 } 398 399 static inline void 400 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear) 401 { 402 #ifdef INVARIANTS 403 if (mtx_owned(&hpts->p_mtx) == 0) { 404 /* We don't own the mutex? */ 405 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 406 } 407 if (hpts->p_cpu != inp->inp_input_cpu) { 408 /* It is not the right cpu/mutex? */ 409 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 410 } 411 if (inp->inp_in_input == 0) { 412 /* We are not on the input hpts? */ 413 panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp); 414 } 415 #endif 416 TAILQ_REMOVE(&hpts->p_input, inp, inp_input); 417 hpts->p_on_inqueue_cnt--; 418 if (hpts->p_on_inqueue_cnt < 0) { 419 #ifdef INVARIANTS 420 panic("Hpts in goes negative inp:%p hpts:%p", 421 inp, hpts); 422 #endif 423 hpts->p_on_inqueue_cnt = 0; 424 } 425 #ifdef INVARIANTS 426 if (TAILQ_EMPTY(&hpts->p_input) && 427 (hpts->p_on_inqueue_cnt != 0)) { 428 /* We should not be empty with a queue count */ 429 panic("%s hpts:%p in_hpts input empty but cnt:%d", 430 __FUNCTION__, hpts, hpts->p_on_inqueue_cnt); 431 } 432 #endif 433 if (clear) 434 inp->inp_in_input = 0; 435 } 436 437 static inline void 438 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line) 439 { 440 #ifdef INVARIANTS 441 if (mtx_owned(&hpts->p_mtx) == 0) { 442 /* We don't own the mutex? */ 443 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp); 444 } 445 if (hpts->p_cpu != inp->inp_input_cpu) { 446 /* It is not the right cpu/mutex? */ 447 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp); 448 } 449 if (inp->inp_in_input == 1) { 450 /* We are already on the input hpts? */ 451 panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp); 452 } 453 #endif 454 TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input); 455 inp->inp_in_input = 1; 456 hpts->p_on_inqueue_cnt++; 457 in_pcbref(inp); 458 } 459 460 static int 461 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS) 462 { 463 struct tcp_hpts_entry *hpts; 464 size_t sz; 465 int32_t logging_was, i; 466 int32_t error = 0; 467 468 /* 469 * HACK: Turn off logging so no locks are required this really needs 470 * a memory barrier :) 471 */ 472 logging_was = logging_on; 473 logging_on = 0; 474 if (!req->oldptr) { 475 /* How much? */ 476 sz = 0; 477 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 478 hpts = tcp_pace.rp_ent[i]; 479 if (hpts->p_log == NULL) 480 continue; 481 sz += (sizeof(struct hpts_log) * hpts->p_logsize); 482 } 483 error = SYSCTL_OUT(req, 0, sz); 484 } else { 485 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 486 hpts = tcp_pace.rp_ent[i]; 487 if (hpts->p_log == NULL) 488 continue; 489 if (hpts->p_log_wrapped) 490 sz = (sizeof(struct hpts_log) * hpts->p_logsize); 491 else 492 sz = (sizeof(struct hpts_log) * hpts->p_log_at); 493 error = SYSCTL_OUT(req, hpts->p_log, sz); 494 } 495 } 496 logging_on = logging_was; 497 return error; 498 } 499 500 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, 501 0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log"); 502 503 504 /* 505 * Try to get the INP_INFO lock. 506 * 507 * This function always succeeds in getting the lock. It will clear 508 * *tpp and return (1) if something critical changed while the inpcb 509 * was unlocked. Otherwise, it will leave *tpp unchanged and return (0). 510 * 511 * This function relies on the fact that the hpts always holds a 512 * reference on the inpcb while the segment is on the hptsi wheel and 513 * in the input queue. 514 * 515 */ 516 static int 517 tcp_hptsi_lock_inpinfo(struct inpcb *inp, struct tcpcb **tpp) 518 { 519 struct tcp_function_block *tfb; 520 struct tcpcb *tp; 521 void *ptr; 522 523 /* Try the easy way. */ 524 if (INP_INFO_TRY_RLOCK(&V_tcbinfo)) 525 return (0); 526 527 /* 528 * OK, let's try the hard way. We'll save the function pointer block 529 * to make sure that doesn't change while we aren't holding the 530 * lock. 531 */ 532 tp = *tpp; 533 tfb = tp->t_fb; 534 ptr = tp->t_fb_ptr; 535 INP_WUNLOCK(inp); 536 INP_INFO_RLOCK(&V_tcbinfo); 537 INP_WLOCK(inp); 538 /* If the session went away, return an error. */ 539 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 540 (inp->inp_flags2 & INP_FREED)) { 541 *tpp = NULL; 542 return (1); 543 } 544 /* 545 * If the function block or stack-specific data block changed, 546 * report an error. 547 */ 548 tp = intotcpcb(inp); 549 if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) { 550 *tpp = NULL; 551 return (1); 552 } 553 return (0); 554 } 555 556 557 static void 558 tcp_wakehpts(struct tcp_hpts_entry *hpts) 559 { 560 HPTS_MTX_ASSERT(hpts); 561 swi_sched(hpts->ie_cookie, 0); 562 if (hpts->p_hpts_active == 2) { 563 /* Rare sleeping on a ENOBUF */ 564 wakeup_one(hpts); 565 } 566 } 567 568 static void 569 tcp_wakeinput(struct tcp_hpts_entry *hpts) 570 { 571 HPTS_MTX_ASSERT(hpts); 572 swi_sched(hpts->ie_cookie, 0); 573 if (hpts->p_hpts_active == 2) { 574 /* Rare sleeping on a ENOBUF */ 575 wakeup_one(hpts); 576 } 577 } 578 579 struct tcp_hpts_entry * 580 tcp_cur_hpts(struct inpcb *inp) 581 { 582 int32_t hpts_num; 583 struct tcp_hpts_entry *hpts; 584 585 hpts_num = inp->inp_hpts_cpu; 586 hpts = tcp_pace.rp_ent[hpts_num]; 587 return (hpts); 588 } 589 590 struct tcp_hpts_entry * 591 tcp_hpts_lock(struct inpcb *inp) 592 { 593 struct tcp_hpts_entry *hpts; 594 int32_t hpts_num; 595 596 again: 597 hpts_num = inp->inp_hpts_cpu; 598 hpts = tcp_pace.rp_ent[hpts_num]; 599 #ifdef INVARIANTS 600 if (mtx_owned(&hpts->p_mtx)) { 601 panic("Hpts:%p owns mtx prior-to lock line:%d", 602 hpts, __LINE__); 603 } 604 #endif 605 mtx_lock(&hpts->p_mtx); 606 if (hpts_num != inp->inp_hpts_cpu) { 607 mtx_unlock(&hpts->p_mtx); 608 goto again; 609 } 610 return (hpts); 611 } 612 613 struct tcp_hpts_entry * 614 tcp_input_lock(struct inpcb *inp) 615 { 616 struct tcp_hpts_entry *hpts; 617 int32_t hpts_num; 618 619 again: 620 hpts_num = inp->inp_input_cpu; 621 hpts = tcp_pace.rp_ent[hpts_num]; 622 #ifdef INVARIANTS 623 if (mtx_owned(&hpts->p_mtx)) { 624 panic("Hpts:%p owns mtx prior-to lock line:%d", 625 hpts, __LINE__); 626 } 627 #endif 628 mtx_lock(&hpts->p_mtx); 629 if (hpts_num != inp->inp_input_cpu) { 630 mtx_unlock(&hpts->p_mtx); 631 goto again; 632 } 633 return (hpts); 634 } 635 636 static void 637 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line) 638 { 639 int32_t add_freed; 640 641 if (inp->inp_flags2 & INP_FREED) { 642 /* 643 * Need to play a special trick so that in_pcbrele_wlocked 644 * does not return 1 when it really should have returned 0. 645 */ 646 add_freed = 1; 647 inp->inp_flags2 &= ~INP_FREED; 648 } else { 649 add_freed = 0; 650 } 651 #ifndef INP_REF_DEBUG 652 if (in_pcbrele_wlocked(inp)) { 653 /* 654 * This should not happen. We have the inpcb referred to by 655 * the main socket (why we are called) and the hpts. It 656 * should always return 0. 657 */ 658 panic("inpcb:%p release ret 1", 659 inp); 660 } 661 #else 662 if (__in_pcbrele_wlocked(inp, line)) { 663 /* 664 * This should not happen. We have the inpcb referred to by 665 * the main socket (why we are called) and the hpts. It 666 * should always return 0. 667 */ 668 panic("inpcb:%p release ret 1", 669 inp); 670 } 671 #endif 672 if (add_freed) { 673 inp->inp_flags2 |= INP_FREED; 674 } 675 } 676 677 static void 678 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line) 679 { 680 if (inp->inp_in_hpts) { 681 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1); 682 tcp_remove_hpts_ref(inp, hpts, line); 683 } 684 } 685 686 static void 687 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line) 688 { 689 HPTS_MTX_ASSERT(hpts); 690 if (inp->inp_in_input) { 691 hpts_sane_input_remove(hpts, inp, 1); 692 tcp_remove_hpts_ref(inp, hpts, line); 693 } 694 } 695 696 /* 697 * Called normally with the INP_LOCKED but it 698 * does not matter, the hpts lock is the key 699 * but the lock order allows us to hold the 700 * INP lock and then get the hpts lock. 701 * 702 * Valid values in the flags are 703 * HPTS_REMOVE_OUTPUT - remove from the output of the hpts. 704 * HPTS_REMOVE_INPUT - remove from the input of the hpts. 705 * Note that you can or both values together and get two 706 * actions. 707 */ 708 void 709 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line) 710 { 711 struct tcp_hpts_entry *hpts; 712 713 INP_WLOCK_ASSERT(inp); 714 if (flags & HPTS_REMOVE_OUTPUT) { 715 hpts = tcp_hpts_lock(inp); 716 tcp_hpts_remove_locked_output(hpts, inp, flags, line); 717 mtx_unlock(&hpts->p_mtx); 718 } 719 if (flags & HPTS_REMOVE_INPUT) { 720 hpts = tcp_input_lock(inp); 721 tcp_hpts_remove_locked_input(hpts, inp, flags, line); 722 mtx_unlock(&hpts->p_mtx); 723 } 724 } 725 726 static inline int 727 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus) 728 { 729 return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS); 730 } 731 732 static int 733 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref) 734 { 735 int32_t need_wake = 0; 736 uint32_t ticknow = 0; 737 738 HPTS_MTX_ASSERT(hpts); 739 if (inp->inp_in_hpts == 0) { 740 /* Ok we need to set it on the hpts in the current slot */ 741 if (hpts->p_hpts_active == 0) { 742 /* A sleeping hpts we want in next slot to run */ 743 if (logging_on) { 744 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0, 745 hpts_tick(hpts, 1)); 746 } 747 inp->inp_hptsslot = hpts_tick(hpts, 1); 748 inp->inp_hpts_request = 0; 749 if (logging_on) { 750 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow); 751 } 752 need_wake = 1; 753 } else if ((void *)inp == hpts->p_inp) { 754 /* 755 * We can't allow you to go into the same slot we 756 * are in. We must put you out. 757 */ 758 inp->inp_hptsslot = hpts->p_nxt_slot; 759 } else 760 inp->inp_hptsslot = hpts->p_cur_slot; 761 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref); 762 inp->inp_hpts_request = 0; 763 if (logging_on) { 764 tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0); 765 } 766 if (need_wake) { 767 /* 768 * Activate the hpts if it is sleeping and its 769 * timeout is not 1. 770 */ 771 if (logging_on) { 772 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow); 773 } 774 hpts->p_direct_wake = 1; 775 tcp_wakehpts(hpts); 776 } 777 } 778 return (need_wake); 779 } 780 781 int 782 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line) 783 { 784 int32_t ret; 785 struct tcp_hpts_entry *hpts; 786 787 INP_WLOCK_ASSERT(inp); 788 hpts = tcp_hpts_lock(inp); 789 ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0); 790 mtx_unlock(&hpts->p_mtx); 791 return (ret); 792 } 793 794 static void 795 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line, 796 struct hpts_diag *diag, int32_t noref) 797 { 798 int32_t need_new_to = 0; 799 int32_t need_wakeup = 0; 800 uint32_t largest_slot; 801 uint32_t ticknow = 0; 802 uint32_t slot_calc; 803 804 HPTS_MTX_ASSERT(hpts); 805 if (diag) { 806 memset(diag, 0, sizeof(struct hpts_diag)); 807 diag->p_hpts_active = hpts->p_hpts_active; 808 diag->p_nxt_slot = hpts->p_nxt_slot; 809 diag->p_cur_slot = hpts->p_cur_slot; 810 diag->slot_req = slot; 811 } 812 if ((inp->inp_in_hpts == 0) || noref) { 813 inp->inp_hpts_request = slot; 814 if (slot == 0) { 815 /* Immediate */ 816 tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref); 817 return; 818 } 819 if (hpts->p_hpts_active) { 820 /* 821 * Its slot - 1 since nxt_slot is the next tick that 822 * will go off since the hpts is awake 823 */ 824 if (logging_on) { 825 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0); 826 } 827 /* 828 * We want to make sure that we don't place a inp in 829 * the range of p_cur_slot <-> p_nxt_slot. If we 830 * take from p_nxt_slot to the end, plus p_cur_slot 831 * and then take away 2, we will know how many is 832 * the max slots we can use. 833 */ 834 if (hpts->p_nxt_slot > hpts->p_cur_slot) { 835 /* 836 * Non-wrap case nxt_slot <-> cur_slot we 837 * don't want to land in. So the diff gives 838 * us what is taken away from the number of 839 * slots. 840 */ 841 largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot); 842 } else if (hpts->p_nxt_slot == hpts->p_cur_slot) { 843 largest_slot = NUM_OF_HPTSI_SLOTS - 2; 844 } else { 845 /* 846 * Wrap case so the diff gives us the number 847 * of slots that we can land in. 848 */ 849 largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot; 850 } 851 /* 852 * We take away two so we never have a problem (20 853 * usec's) out of 1024000 usecs 854 */ 855 largest_slot -= 2; 856 if (inp->inp_hpts_request > largest_slot) { 857 /* 858 * Restrict max jump of slots and remember 859 * leftover 860 */ 861 slot = largest_slot; 862 inp->inp_hpts_request -= largest_slot; 863 } else { 864 /* This one will run when we hit it */ 865 inp->inp_hpts_request = 0; 866 } 867 if (hpts->p_nxt_slot == hpts->p_cur_slot) 868 slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS; 869 else 870 slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS; 871 if (slot_calc == hpts->p_cur_slot) { 872 #ifdef INVARIANTS 873 /* TSNH */ 874 panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n", 875 hpts, slot_calc, slot, largest_slot); 876 #endif 877 if (slot_calc) 878 slot_calc--; 879 else 880 slot_calc = NUM_OF_HPTSI_SLOTS - 1; 881 } 882 inp->inp_hptsslot = slot_calc; 883 if (diag) { 884 diag->inp_hptsslot = inp->inp_hptsslot; 885 } 886 } else { 887 /* 888 * The hpts is sleeping, we need to figure out where 889 * it will wake up at and if we need to reschedule 890 * its time-out. 891 */ 892 uint32_t have_slept, yet_to_sleep; 893 uint32_t slot_now; 894 struct timeval tv; 895 896 ticknow = tcp_gethptstick(&tv); 897 slot_now = ticknow % NUM_OF_HPTSI_SLOTS; 898 /* 899 * The user wants to be inserted at (slot_now + 900 * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up. 901 */ 902 largest_slot = NUM_OF_HPTSI_SLOTS - 2; 903 if (inp->inp_hpts_request > largest_slot) { 904 /* Adjust the residual in inp_hpts_request */ 905 slot = largest_slot; 906 inp->inp_hpts_request -= largest_slot; 907 } else { 908 /* No residual it all fits */ 909 inp->inp_hpts_request = 0; 910 } 911 inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS; 912 if (diag) { 913 diag->slot_now = slot_now; 914 diag->inp_hptsslot = inp->inp_hptsslot; 915 diag->p_on_min_sleep = hpts->p_on_min_sleep; 916 } 917 if (logging_on) { 918 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow); 919 } 920 /* Now do we need to restart the hpts's timer? */ 921 if (TSTMP_GT(ticknow, hpts->p_curtick)) 922 have_slept = ticknow - hpts->p_curtick; 923 else 924 have_slept = 0; 925 if (have_slept < hpts->p_hpts_sleep_time) { 926 /* This should be what happens */ 927 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; 928 } else { 929 /* We are over-due */ 930 yet_to_sleep = 0; 931 need_wakeup = 1; 932 } 933 if (diag) { 934 diag->have_slept = have_slept; 935 diag->yet_to_sleep = yet_to_sleep; 936 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; 937 } 938 if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) { 939 /* 940 * We need to reschedule the hptss time-out. 941 */ 942 hpts->p_hpts_sleep_time = slot; 943 need_new_to = slot * HPTS_TICKS_PER_USEC; 944 } 945 } 946 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref); 947 if (logging_on) { 948 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow); 949 } 950 /* 951 * Now how far is the hpts sleeping to? if active is 1, its 952 * up and ticking we do nothing, otherwise we may need to 953 * reschedule its callout if need_new_to is set from above. 954 */ 955 if (need_wakeup) { 956 if (logging_on) { 957 tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0); 958 } 959 hpts->p_direct_wake = 1; 960 tcp_wakehpts(hpts); 961 if (diag) { 962 diag->need_new_to = 0; 963 diag->co_ret = 0xffff0000; 964 } 965 } else if (need_new_to) { 966 int32_t co_ret; 967 struct timeval tv; 968 sbintime_t sb; 969 970 tv.tv_sec = 0; 971 tv.tv_usec = 0; 972 while (need_new_to > HPTS_USEC_IN_SEC) { 973 tv.tv_sec++; 974 need_new_to -= HPTS_USEC_IN_SEC; 975 } 976 tv.tv_usec = need_new_to; 977 sb = tvtosbt(tv); 978 if (tcp_hpts_callout_skip_swi == 0) { 979 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, 980 hpts_timeout_swi, hpts, hpts->p_cpu, 981 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 982 } else { 983 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, 984 hpts_timeout_dir, hpts, 985 hpts->p_cpu, 986 C_PREL(tcp_hpts_precision)); 987 } 988 if (diag) { 989 diag->need_new_to = need_new_to; 990 diag->co_ret = co_ret; 991 } 992 } 993 } else { 994 #ifdef INVARIANTS 995 panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp); 996 #endif 997 } 998 } 999 1000 uint32_t 1001 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){ 1002 struct tcp_hpts_entry *hpts; 1003 uint32_t slot_on, cts; 1004 struct timeval tv; 1005 1006 /* 1007 * We now return the next-slot the hpts will be on, beyond its 1008 * current run (if up) or where it was when it stopped if it is 1009 * sleeping. 1010 */ 1011 INP_WLOCK_ASSERT(inp); 1012 hpts = tcp_hpts_lock(inp); 1013 if (in_ts_percision) 1014 microuptime(&tv); 1015 else 1016 getmicrouptime(&tv); 1017 cts = tcp_tv_to_usectick(&tv); 1018 tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0); 1019 slot_on = hpts->p_nxt_slot; 1020 mtx_unlock(&hpts->p_mtx); 1021 return (slot_on); 1022 } 1023 1024 uint32_t 1025 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){ 1026 return (tcp_hpts_insert_diag(inp, slot, line, NULL)); 1027 } 1028 1029 int 1030 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line) 1031 { 1032 int32_t retval = 0; 1033 1034 HPTS_MTX_ASSERT(hpts); 1035 if (inp->inp_in_input == 0) { 1036 /* Ok we need to set it on the hpts in the current slot */ 1037 hpts_sane_input_insert(hpts, inp, line); 1038 retval = 1; 1039 if (hpts->p_hpts_active == 0) { 1040 /* 1041 * Activate the hpts if it is sleeping. 1042 */ 1043 if (logging_on) { 1044 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0); 1045 } 1046 retval = 2; 1047 hpts->p_direct_wake = 1; 1048 tcp_wakeinput(hpts); 1049 } 1050 } else if (hpts->p_hpts_active == 0) { 1051 retval = 4; 1052 hpts->p_direct_wake = 1; 1053 tcp_wakeinput(hpts); 1054 } 1055 return (retval); 1056 } 1057 1058 void 1059 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, 1060 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked) 1061 { 1062 /* Setup packet for input first */ 1063 INP_WLOCK_ASSERT(tp->t_inpcb); 1064 m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t)); 1065 m->m_pkthdr.pace_tlen = (uint16_t) tlen; 1066 m->m_pkthdr.pace_drphdrlen = drop_hdrlen; 1067 m->m_pkthdr.pace_tos = iptos; 1068 m->m_pkthdr.pace_lock = (uint8_t) ti_locked; 1069 if (tp->t_in_pkt == NULL) { 1070 tp->t_in_pkt = m; 1071 tp->t_tail_pkt = m; 1072 } else { 1073 tp->t_tail_pkt->m_nextpkt = m; 1074 tp->t_tail_pkt = m; 1075 } 1076 } 1077 1078 1079 int32_t 1080 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th, 1081 int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line){ 1082 struct tcp_hpts_entry *hpts; 1083 int32_t ret; 1084 1085 tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos, ti_locked); 1086 hpts = tcp_input_lock(tp->t_inpcb); 1087 ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line); 1088 mtx_unlock(&hpts->p_mtx); 1089 return (ret); 1090 } 1091 1092 void 1093 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line) 1094 { 1095 struct tcp_hpts_entry *hpts; 1096 struct tcpcb *tp; 1097 1098 tp = intotcpcb(inp); 1099 hpts = tcp_input_lock(tp->t_inpcb); 1100 if (inp->inp_in_input == 0) { 1101 /* Ok we need to set it on the hpts in the current slot */ 1102 hpts_sane_input_insert(hpts, inp, line); 1103 if (hpts->p_hpts_active == 0) { 1104 /* 1105 * Activate the hpts if it is sleeping. 1106 */ 1107 hpts->p_direct_wake = 1; 1108 tcp_wakeinput(hpts); 1109 } 1110 } else if (hpts->p_hpts_active == 0) { 1111 hpts->p_direct_wake = 1; 1112 tcp_wakeinput(hpts); 1113 } 1114 inp->inp_hpts_drop_reas = reason; 1115 mtx_unlock(&hpts->p_mtx); 1116 } 1117 1118 static uint16_t 1119 hpts_random_cpu(struct inpcb *inp){ 1120 /* 1121 * No flow type set distribute the load randomly. 1122 */ 1123 uint16_t cpuid; 1124 uint32_t ran; 1125 1126 /* 1127 * If one has been set use it i.e. we want both in and out on the 1128 * same hpts. 1129 */ 1130 if (inp->inp_input_cpu_set) { 1131 return (inp->inp_input_cpu); 1132 } else if (inp->inp_hpts_cpu_set) { 1133 return (inp->inp_hpts_cpu); 1134 } 1135 /* Nothing set use a random number */ 1136 ran = arc4random(); 1137 cpuid = (ran & 0xffff) % mp_ncpus; 1138 return (cpuid); 1139 } 1140 1141 static uint16_t 1142 hpts_cpuid(struct inpcb *inp){ 1143 uint16_t cpuid; 1144 1145 1146 /* 1147 * If one has been set use it i.e. we want both in and out on the 1148 * same hpts. 1149 */ 1150 if (inp->inp_input_cpu_set) { 1151 return (inp->inp_input_cpu); 1152 } else if (inp->inp_hpts_cpu_set) { 1153 return (inp->inp_hpts_cpu); 1154 } 1155 /* If one is set the other must be the same */ 1156 #ifdef RSS 1157 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); 1158 if (cpuid == NETISR_CPUID_NONE) 1159 return (hpts_random_cpu(inp)); 1160 else 1161 return (cpuid); 1162 #else 1163 /* 1164 * We don't have a flowid -> cpuid mapping, so cheat and just map 1165 * unknown cpuids to curcpu. Not the best, but apparently better 1166 * than defaulting to swi 0. 1167 */ 1168 if (inp->inp_flowtype != M_HASHTYPE_NONE) { 1169 cpuid = inp->inp_flowid % mp_ncpus; 1170 return (cpuid); 1171 } 1172 cpuid = hpts_random_cpu(inp); 1173 return (cpuid); 1174 #endif 1175 } 1176 1177 /* 1178 * Do NOT try to optimize the processing of inp's 1179 * by first pulling off all the inp's into a temporary 1180 * list (e.g. TAILQ_CONCAT). If you do that the subtle 1181 * interactions of switching CPU's will kill because of 1182 * problems in the linked list manipulation. Basically 1183 * you would switch cpu's with the hpts mutex locked 1184 * but then while you were processing one of the inp's 1185 * some other one that you switch will get a new 1186 * packet on the different CPU. It will insert it 1187 * on the new hptss input list. Creating a temporary 1188 * link in the inp will not fix it either, since 1189 * the other hpts will be doing the same thing and 1190 * you will both end up using the temporary link. 1191 * 1192 * You will die in an ASSERT for tailq corruption if you 1193 * run INVARIANTS or you will die horribly without 1194 * INVARIANTS in some unknown way with a corrupt linked 1195 * list. 1196 */ 1197 static void 1198 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv) 1199 { 1200 struct mbuf *m, *n; 1201 struct tcpcb *tp; 1202 struct inpcb *inp; 1203 uint16_t drop_reason; 1204 int16_t set_cpu; 1205 uint32_t did_prefetch = 0; 1206 int32_t ti_locked = TI_UNLOCKED; 1207 1208 HPTS_MTX_ASSERT(hpts); 1209 while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) { 1210 HPTS_MTX_ASSERT(hpts); 1211 hpts_sane_input_remove(hpts, inp, 0); 1212 if (inp->inp_input_cpu_set == 0) { 1213 set_cpu = 1; 1214 } else { 1215 set_cpu = 0; 1216 } 1217 hpts->p_inp = inp; 1218 drop_reason = inp->inp_hpts_drop_reas; 1219 inp->inp_in_input = 0; 1220 mtx_unlock(&hpts->p_mtx); 1221 if (drop_reason) { 1222 INP_INFO_RLOCK(&V_tcbinfo); 1223 ti_locked = TI_RLOCKED; 1224 } else { 1225 ti_locked = TI_UNLOCKED; 1226 } 1227 INP_WLOCK(inp); 1228 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 1229 (inp->inp_flags2 & INP_FREED)) { 1230 out: 1231 hpts->p_inp = NULL; 1232 if (ti_locked == TI_RLOCKED) { 1233 INP_INFO_RUNLOCK(&V_tcbinfo); 1234 } 1235 if (in_pcbrele_wlocked(inp) == 0) { 1236 INP_WUNLOCK(inp); 1237 } 1238 ti_locked = TI_UNLOCKED; 1239 mtx_lock(&hpts->p_mtx); 1240 continue; 1241 } 1242 tp = intotcpcb(inp); 1243 if ((tp == NULL) || (tp->t_inpcb == NULL)) { 1244 goto out; 1245 } 1246 if (drop_reason) { 1247 /* This tcb is being destroyed for drop_reason */ 1248 m = tp->t_in_pkt; 1249 if (m) 1250 n = m->m_nextpkt; 1251 else 1252 n = NULL; 1253 tp->t_in_pkt = NULL; 1254 while (m) { 1255 m_freem(m); 1256 m = n; 1257 if (m) 1258 n = m->m_nextpkt; 1259 } 1260 tp = tcp_drop(tp, drop_reason); 1261 INP_INFO_RUNLOCK(&V_tcbinfo); 1262 if (tp == NULL) { 1263 INP_WLOCK(inp); 1264 } 1265 if (in_pcbrele_wlocked(inp) == 0) 1266 INP_WUNLOCK(inp); 1267 mtx_lock(&hpts->p_mtx); 1268 continue; 1269 } 1270 if (set_cpu) { 1271 /* 1272 * Setup so the next time we will move to the right 1273 * CPU. This should be a rare event. It will 1274 * sometimes happens when we are the client side 1275 * (usually not the server). Somehow tcp_output() 1276 * gets called before the tcp_do_segment() sets the 1277 * intial state. This means the r_cpu and r_hpts_cpu 1278 * is 0. We get on the hpts, and then tcp_input() 1279 * gets called setting up the r_cpu to the correct 1280 * value. The hpts goes off and sees the mis-match. 1281 * We simply correct it here and the CPU will switch 1282 * to the new hpts nextime the tcb gets added to the 1283 * the hpts (not this time) :-) 1284 */ 1285 tcp_set_hpts(inp); 1286 } 1287 CURVNET_SET(tp->t_vnet); 1288 m = tp->t_in_pkt; 1289 n = NULL; 1290 if (m != NULL && 1291 (m->m_pkthdr.pace_lock == TI_RLOCKED || 1292 tp->t_state != TCPS_ESTABLISHED)) { 1293 ti_locked = TI_RLOCKED; 1294 if (tcp_hptsi_lock_inpinfo(inp, &tp)) { 1295 CURVNET_RESTORE(); 1296 goto out; 1297 } 1298 m = tp->t_in_pkt; 1299 } 1300 if (in_newts_every_tcb) { 1301 if (in_ts_percision) 1302 microuptime(tv); 1303 else 1304 getmicrouptime(tv); 1305 } 1306 if (tp->t_fb_ptr != NULL) { 1307 kern_prefetch(tp->t_fb_ptr, &did_prefetch); 1308 did_prefetch = 1; 1309 } 1310 /* Any input work to do, if so do it first */ 1311 if ((m != NULL) && (m == tp->t_in_pkt)) { 1312 struct tcphdr *th; 1313 int32_t tlen, drop_hdrlen, nxt_pkt; 1314 uint8_t iptos; 1315 1316 n = m->m_nextpkt; 1317 tp->t_in_pkt = tp->t_tail_pkt = NULL; 1318 while (m) { 1319 th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff); 1320 tlen = m->m_pkthdr.pace_tlen; 1321 drop_hdrlen = m->m_pkthdr.pace_drphdrlen; 1322 iptos = m->m_pkthdr.pace_tos; 1323 m->m_nextpkt = NULL; 1324 if (n) 1325 nxt_pkt = 1; 1326 else 1327 nxt_pkt = 0; 1328 inp->inp_input_calls = 1; 1329 if (tp->t_fb->tfb_tcp_hpts_do_segment) { 1330 /* Use the hpts specific do_segment */ 1331 (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket, 1332 tp, drop_hdrlen, 1333 tlen, iptos, ti_locked, nxt_pkt, tv); 1334 } else { 1335 /* Use the default do_segment */ 1336 (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket, 1337 tp, drop_hdrlen, 1338 tlen, iptos, ti_locked); 1339 } 1340 /* 1341 * Do segment returns unlocked we need the 1342 * lock again but we also need some kasserts 1343 * here. 1344 */ 1345 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 1346 INP_UNLOCK_ASSERT(inp); 1347 m = n; 1348 if (m) 1349 n = m->m_nextpkt; 1350 if (m != NULL && 1351 m->m_pkthdr.pace_lock == TI_RLOCKED) { 1352 INP_INFO_RLOCK(&V_tcbinfo); 1353 ti_locked = TI_RLOCKED; 1354 } else 1355 ti_locked = TI_UNLOCKED; 1356 INP_WLOCK(inp); 1357 /* 1358 * Since we have an opening here we must 1359 * re-check if the tcb went away while we 1360 * were getting the lock(s). 1361 */ 1362 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) || 1363 (inp->inp_flags2 & INP_FREED)) { 1364 out_free: 1365 while (m) { 1366 m_freem(m); 1367 m = n; 1368 if (m) 1369 n = m->m_nextpkt; 1370 } 1371 CURVNET_RESTORE(); 1372 goto out; 1373 } 1374 /* 1375 * Now that we hold the INP lock, check if 1376 * we need to upgrade our lock. 1377 */ 1378 if (ti_locked == TI_UNLOCKED && 1379 (tp->t_state != TCPS_ESTABLISHED)) { 1380 ti_locked = TI_RLOCKED; 1381 if (tcp_hptsi_lock_inpinfo(inp, &tp)) 1382 goto out_free; 1383 } 1384 } /** end while(m) */ 1385 } /** end if ((m != NULL) && (m == tp->t_in_pkt)) */ 1386 if (in_pcbrele_wlocked(inp) == 0) 1387 INP_WUNLOCK(inp); 1388 if (ti_locked == TI_RLOCKED) 1389 INP_INFO_RUNLOCK(&V_tcbinfo); 1390 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo); 1391 INP_UNLOCK_ASSERT(inp); 1392 ti_locked = TI_UNLOCKED; 1393 mtx_lock(&hpts->p_mtx); 1394 hpts->p_inp = NULL; 1395 CURVNET_RESTORE(); 1396 } 1397 } 1398 1399 static int 1400 tcp_hpts_est_run(struct tcp_hpts_entry *hpts) 1401 { 1402 int32_t ticks_to_run; 1403 1404 if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) { 1405 ticks_to_run = hpts->p_curtick - hpts->p_prevtick; 1406 if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) { 1407 ticks_to_run = NUM_OF_HPTSI_SLOTS - 2; 1408 } 1409 } else { 1410 if (hpts->p_prevtick == hpts->p_curtick) { 1411 /* This happens when we get woken up right away */ 1412 return (-1); 1413 } 1414 ticks_to_run = 1; 1415 } 1416 /* Set in where we will be when we catch up */ 1417 hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS; 1418 if (hpts->p_nxt_slot == hpts->p_cur_slot) { 1419 panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d", 1420 hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run); 1421 } 1422 return (ticks_to_run); 1423 } 1424 1425 static void 1426 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick) 1427 { 1428 struct tcpcb *tp; 1429 struct inpcb *inp = NULL, *ninp; 1430 struct timeval tv; 1431 int32_t ticks_to_run, i, error, tick_now, interum_tick; 1432 int32_t paced_cnt = 0; 1433 int32_t did_prefetch = 0; 1434 int32_t prefetch_ninp = 0; 1435 int32_t prefetch_tp = 0; 1436 uint32_t cts; 1437 int16_t set_cpu; 1438 1439 HPTS_MTX_ASSERT(hpts); 1440 hpts->p_curtick = tcp_tv_to_hptstick(ctick); 1441 cts = tcp_tv_to_usectick(ctick); 1442 memcpy(&tv, ctick, sizeof(struct timeval)); 1443 hpts->p_cur_slot = hpts_tick(hpts, 1); 1444 1445 /* Figure out if we had missed ticks */ 1446 again: 1447 HPTS_MTX_ASSERT(hpts); 1448 ticks_to_run = tcp_hpts_est_run(hpts); 1449 if (!TAILQ_EMPTY(&hpts->p_input)) { 1450 tcp_input_data(hpts, &tv); 1451 } 1452 #ifdef INVARIANTS 1453 if (TAILQ_EMPTY(&hpts->p_input) && 1454 (hpts->p_on_inqueue_cnt != 0)) { 1455 panic("tp:%p in_hpts input empty but cnt:%d", 1456 hpts, hpts->p_on_inqueue_cnt); 1457 } 1458 #endif 1459 HPTS_MTX_ASSERT(hpts); 1460 /* Reset the ticks to run and time if we need too */ 1461 interum_tick = tcp_gethptstick(&tv); 1462 if (interum_tick != hpts->p_curtick) { 1463 /* Save off the new time we execute to */ 1464 *ctick = tv; 1465 hpts->p_curtick = interum_tick; 1466 cts = tcp_tv_to_usectick(&tv); 1467 hpts->p_cur_slot = hpts_tick(hpts, 1); 1468 ticks_to_run = tcp_hpts_est_run(hpts); 1469 } 1470 if (ticks_to_run == -1) { 1471 goto no_run; 1472 } 1473 if (logging_on) { 1474 tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0); 1475 } 1476 if (hpts->p_on_queue_cnt == 0) { 1477 goto no_one; 1478 } 1479 HPTS_MTX_ASSERT(hpts); 1480 for (i = 0; i < ticks_to_run; i++) { 1481 /* 1482 * Calculate our delay, if there are no extra ticks there 1483 * was not any 1484 */ 1485 hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC; 1486 HPTS_MTX_ASSERT(hpts); 1487 while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) { 1488 /* For debugging */ 1489 if (logging_on) { 1490 tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i); 1491 } 1492 hpts->p_inp = inp; 1493 paced_cnt++; 1494 if (hpts->p_cur_slot != inp->inp_hptsslot) { 1495 panic("Hpts:%p inp:%p slot mis-aligned %u vs %u", 1496 hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot); 1497 } 1498 /* Now pull it */ 1499 if (inp->inp_hpts_cpu_set == 0) { 1500 set_cpu = 1; 1501 } else { 1502 set_cpu = 0; 1503 } 1504 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0); 1505 if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) { 1506 /* We prefetch the next inp if possible */ 1507 kern_prefetch(ninp, &prefetch_ninp); 1508 prefetch_ninp = 1; 1509 } 1510 if (inp->inp_hpts_request) { 1511 /* 1512 * This guy is deferred out further in time 1513 * then our wheel had on it. Push him back 1514 * on the wheel. 1515 */ 1516 int32_t remaining_slots; 1517 1518 remaining_slots = ticks_to_run - (i + 1); 1519 if (inp->inp_hpts_request > remaining_slots) { 1520 /* 1521 * Keep INVARIANTS happy by clearing 1522 * the flag 1523 */ 1524 tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1); 1525 hpts->p_inp = NULL; 1526 continue; 1527 } 1528 inp->inp_hpts_request = 0; 1529 } 1530 /* 1531 * We clear the hpts flag here after dealing with 1532 * remaining slots. This way anyone looking with the 1533 * TCB lock will see its on the hpts until just 1534 * before we unlock. 1535 */ 1536 inp->inp_in_hpts = 0; 1537 mtx_unlock(&hpts->p_mtx); 1538 INP_WLOCK(inp); 1539 if (in_pcbrele_wlocked(inp)) { 1540 mtx_lock(&hpts->p_mtx); 1541 if (logging_on) 1542 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1); 1543 hpts->p_inp = NULL; 1544 continue; 1545 } 1546 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 1547 out_now: 1548 #ifdef INVARIANTS 1549 if (mtx_owned(&hpts->p_mtx)) { 1550 panic("Hpts:%p owns mtx prior-to lock line:%d", 1551 hpts, __LINE__); 1552 } 1553 #endif 1554 INP_WUNLOCK(inp); 1555 mtx_lock(&hpts->p_mtx); 1556 if (logging_on) 1557 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3); 1558 hpts->p_inp = NULL; 1559 continue; 1560 } 1561 tp = intotcpcb(inp); 1562 if ((tp == NULL) || (tp->t_inpcb == NULL)) { 1563 goto out_now; 1564 } 1565 if (set_cpu) { 1566 /* 1567 * Setup so the next time we will move to 1568 * the right CPU. This should be a rare 1569 * event. It will sometimes happens when we 1570 * are the client side (usually not the 1571 * server). Somehow tcp_output() gets called 1572 * before the tcp_do_segment() sets the 1573 * intial state. This means the r_cpu and 1574 * r_hpts_cpu is 0. We get on the hpts, and 1575 * then tcp_input() gets called setting up 1576 * the r_cpu to the correct value. The hpts 1577 * goes off and sees the mis-match. We 1578 * simply correct it here and the CPU will 1579 * switch to the new hpts nextime the tcb 1580 * gets added to the the hpts (not this one) 1581 * :-) 1582 */ 1583 tcp_set_hpts(inp); 1584 } 1585 if (out_newts_every_tcb) { 1586 struct timeval sv; 1587 1588 if (out_ts_percision) 1589 microuptime(&sv); 1590 else 1591 getmicrouptime(&sv); 1592 cts = tcp_tv_to_usectick(&sv); 1593 } 1594 CURVNET_SET(tp->t_vnet); 1595 /* 1596 * There is a hole here, we get the refcnt on the 1597 * inp so it will still be preserved but to make 1598 * sure we can get the INP we need to hold the p_mtx 1599 * above while we pull out the tp/inp, as long as 1600 * fini gets the lock first we are assured of having 1601 * a sane INP we can lock and test. 1602 */ 1603 #ifdef INVARIANTS 1604 if (mtx_owned(&hpts->p_mtx)) { 1605 panic("Hpts:%p owns mtx before tcp-output:%d", 1606 hpts, __LINE__); 1607 } 1608 #endif 1609 if (tp->t_fb_ptr != NULL) { 1610 kern_prefetch(tp->t_fb_ptr, &did_prefetch); 1611 did_prefetch = 1; 1612 } 1613 inp->inp_hpts_calls = 1; 1614 if (tp->t_fb->tfb_tcp_output_wtime != NULL) { 1615 error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv); 1616 } else { 1617 error = tp->t_fb->tfb_tcp_output(tp); 1618 } 1619 if (ninp && ninp->inp_ppcb) { 1620 /* 1621 * If we have a nxt inp, see if we can 1622 * prefetch its ppcb. Note this may seem 1623 * "risky" since we have no locks (other 1624 * than the previous inp) and there no 1625 * assurance that ninp was not pulled while 1626 * we were processing inp and freed. If this 1627 * occured it could mean that either: 1628 * 1629 * a) Its NULL (which is fine we won't go 1630 * here) <or> b) Its valid (which is cool we 1631 * will prefetch it) <or> c) The inp got 1632 * freed back to the slab which was 1633 * reallocated. Then the piece of memory was 1634 * re-used and something else (not an 1635 * address) is in inp_ppcb. If that occurs 1636 * we don't crash, but take a TLB shootdown 1637 * performance hit (same as if it was NULL 1638 * and we tried to pre-fetch it). 1639 * 1640 * Considering that the likelyhood of <c> is 1641 * quite rare we will take a risk on doing 1642 * this. If performance drops after testing 1643 * we can always take this out. NB: the 1644 * kern_prefetch on amd64 actually has 1645 * protection against a bad address now via 1646 * the DMAP_() tests. This will prevent the 1647 * TLB hit, and instead if <c> occurs just 1648 * cause us to load cache with a useless 1649 * address (to us). 1650 */ 1651 kern_prefetch(ninp->inp_ppcb, &prefetch_tp); 1652 prefetch_tp = 1; 1653 } 1654 INP_WUNLOCK(inp); 1655 INP_UNLOCK_ASSERT(inp); 1656 CURVNET_RESTORE(); 1657 #ifdef INVARIANTS 1658 if (mtx_owned(&hpts->p_mtx)) { 1659 panic("Hpts:%p owns mtx prior-to lock line:%d", 1660 hpts, __LINE__); 1661 } 1662 #endif 1663 mtx_lock(&hpts->p_mtx); 1664 if (logging_on) 1665 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4); 1666 hpts->p_inp = NULL; 1667 } 1668 HPTS_MTX_ASSERT(hpts); 1669 hpts->p_inp = NULL; 1670 hpts->p_cur_slot++; 1671 if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) { 1672 hpts->p_cur_slot = 0; 1673 } 1674 } 1675 no_one: 1676 HPTS_MTX_ASSERT(hpts); 1677 hpts->p_prevtick = hpts->p_curtick; 1678 hpts->p_delayed_by = 0; 1679 /* 1680 * Check to see if we took an excess amount of time and need to run 1681 * more ticks (if we did not hit eno-bufs). 1682 */ 1683 /* Re-run any input that may be there */ 1684 (void)tcp_gethptstick(&tv); 1685 if (!TAILQ_EMPTY(&hpts->p_input)) { 1686 tcp_input_data(hpts, &tv); 1687 } 1688 #ifdef INVARIANTS 1689 if (TAILQ_EMPTY(&hpts->p_input) && 1690 (hpts->p_on_inqueue_cnt != 0)) { 1691 panic("tp:%p in_hpts input empty but cnt:%d", 1692 hpts, hpts->p_on_inqueue_cnt); 1693 } 1694 #endif 1695 tick_now = tcp_gethptstick(&tv); 1696 if (SEQ_GT(tick_now, hpts->p_prevtick)) { 1697 struct timeval res; 1698 1699 /* Did we really spend a full tick or more in here? */ 1700 timersub(&tv, ctick, &res); 1701 if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) { 1702 counter_u64_add(hpts_loops, 1); 1703 if (logging_on) { 1704 tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now); 1705 } 1706 *ctick = res; 1707 hpts->p_curtick = tick_now; 1708 goto again; 1709 } 1710 } 1711 no_run: 1712 { 1713 uint32_t t = 0, i, fnd = 0; 1714 1715 if (hpts->p_on_queue_cnt) { 1716 1717 1718 /* 1719 * Find next slot that is occupied and use that to 1720 * be the sleep time. 1721 */ 1722 for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) { 1723 if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) { 1724 fnd = 1; 1725 break; 1726 } 1727 t = (t + 1) % NUM_OF_HPTSI_SLOTS; 1728 } 1729 if (fnd) { 1730 hpts->p_hpts_sleep_time = i; 1731 } else { 1732 counter_u64_add(back_tosleep, 1); 1733 #ifdef INVARIANTS 1734 panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt); 1735 #endif 1736 hpts->p_on_queue_cnt = 0; 1737 goto non_found; 1738 } 1739 t++; 1740 } else { 1741 /* No one on the wheel sleep for all but 2 slots */ 1742 non_found: 1743 if (hpts_sleep_max == 0) 1744 hpts_sleep_max = 1; 1745 hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max); 1746 t = 0; 1747 } 1748 if (logging_on) { 1749 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC)); 1750 } 1751 } 1752 } 1753 1754 void 1755 __tcp_set_hpts(struct inpcb *inp, int32_t line) 1756 { 1757 struct tcp_hpts_entry *hpts; 1758 1759 INP_WLOCK_ASSERT(inp); 1760 hpts = tcp_hpts_lock(inp); 1761 if ((inp->inp_in_hpts == 0) && 1762 (inp->inp_hpts_cpu_set == 0)) { 1763 inp->inp_hpts_cpu = hpts_cpuid(inp); 1764 inp->inp_hpts_cpu_set = 1; 1765 } 1766 mtx_unlock(&hpts->p_mtx); 1767 hpts = tcp_input_lock(inp); 1768 if ((inp->inp_input_cpu_set == 0) && 1769 (inp->inp_in_input == 0)) { 1770 inp->inp_input_cpu = hpts_cpuid(inp); 1771 inp->inp_input_cpu_set = 1; 1772 } 1773 mtx_unlock(&hpts->p_mtx); 1774 } 1775 1776 uint16_t 1777 tcp_hpts_delayedby(struct inpcb *inp){ 1778 return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by); 1779 } 1780 1781 static void 1782 tcp_hpts_thread(void *ctx) 1783 { 1784 struct tcp_hpts_entry *hpts; 1785 struct timeval tv; 1786 sbintime_t sb; 1787 1788 hpts = (struct tcp_hpts_entry *)ctx; 1789 mtx_lock(&hpts->p_mtx); 1790 if (hpts->p_direct_wake) { 1791 /* Signaled by input */ 1792 if (logging_on) 1793 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1); 1794 callout_stop(&hpts->co); 1795 } else { 1796 /* Timed out */ 1797 if (callout_pending(&hpts->co) || 1798 !callout_active(&hpts->co)) { 1799 if (logging_on) 1800 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2); 1801 mtx_unlock(&hpts->p_mtx); 1802 return; 1803 } 1804 callout_deactivate(&hpts->co); 1805 if (logging_on) 1806 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3); 1807 } 1808 hpts->p_hpts_active = 1; 1809 (void)tcp_gethptstick(&tv); 1810 tcp_hptsi(hpts, &tv); 1811 HPTS_MTX_ASSERT(hpts); 1812 tv.tv_sec = 0; 1813 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC; 1814 if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) { 1815 tv.tv_usec = tcp_min_hptsi_time; 1816 hpts->p_on_min_sleep = 1; 1817 } else { 1818 /* Clear the min sleep flag */ 1819 hpts->p_on_min_sleep = 0; 1820 } 1821 hpts->p_hpts_active = 0; 1822 sb = tvtosbt(tv); 1823 if (tcp_hpts_callout_skip_swi == 0) { 1824 callout_reset_sbt_on(&hpts->co, sb, 0, 1825 hpts_timeout_swi, hpts, hpts->p_cpu, 1826 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1827 } else { 1828 callout_reset_sbt_on(&hpts->co, sb, 0, 1829 hpts_timeout_dir, hpts, 1830 hpts->p_cpu, 1831 C_PREL(tcp_hpts_precision)); 1832 } 1833 hpts->p_direct_wake = 0; 1834 mtx_unlock(&hpts->p_mtx); 1835 } 1836 1837 #undef timersub 1838 1839 static void 1840 tcp_init_hptsi(void *st) 1841 { 1842 int32_t i, j, error, bound = 0, created = 0; 1843 size_t sz, asz; 1844 struct timeval tv; 1845 sbintime_t sb; 1846 struct tcp_hpts_entry *hpts; 1847 char unit[16]; 1848 uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU; 1849 1850 tcp_pace.rp_proc = NULL; 1851 tcp_pace.rp_num_hptss = ncpus; 1852 hpts_loops = counter_u64_alloc(M_WAITOK); 1853 back_tosleep = counter_u64_alloc(M_WAITOK); 1854 1855 sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *)); 1856 tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); 1857 asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS; 1858 for (i = 0; i < tcp_pace.rp_num_hptss; i++) { 1859 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry), 1860 M_TCPHPTS, M_WAITOK | M_ZERO); 1861 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, 1862 M_TCPHPTS, M_WAITOK); 1863 hpts = tcp_pace.rp_ent[i]; 1864 /* 1865 * Init all the hpts structures that are not specifically 1866 * zero'd by the allocations. Also lets attach them to the 1867 * appropriate sysctl block as well. 1868 */ 1869 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", 1870 "hpts", MTX_DEF | MTX_DUPOK); 1871 TAILQ_INIT(&hpts->p_input); 1872 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) { 1873 TAILQ_INIT(&hpts->p_hptss[j]); 1874 } 1875 sysctl_ctx_init(&hpts->hpts_ctx); 1876 sprintf(unit, "%d", i); 1877 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, 1878 SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts), 1879 OID_AUTO, 1880 unit, 1881 CTLFLAG_RW, 0, 1882 ""); 1883 SYSCTL_ADD_INT(&hpts->hpts_ctx, 1884 SYSCTL_CHILDREN(hpts->hpts_root), 1885 OID_AUTO, "in_qcnt", CTLFLAG_RD, 1886 &hpts->p_on_inqueue_cnt, 0, 1887 "Count TCB's awaiting input processing"); 1888 SYSCTL_ADD_INT(&hpts->hpts_ctx, 1889 SYSCTL_CHILDREN(hpts->hpts_root), 1890 OID_AUTO, "out_qcnt", CTLFLAG_RD, 1891 &hpts->p_on_queue_cnt, 0, 1892 "Count TCB's awaiting output processing"); 1893 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1894 SYSCTL_CHILDREN(hpts->hpts_root), 1895 OID_AUTO, "active", CTLFLAG_RD, 1896 &hpts->p_hpts_active, 0, 1897 "Is the hpts active"); 1898 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1899 SYSCTL_CHILDREN(hpts->hpts_root), 1900 OID_AUTO, "curslot", CTLFLAG_RD, 1901 &hpts->p_cur_slot, 0, 1902 "What the current slot is if active"); 1903 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1904 SYSCTL_CHILDREN(hpts->hpts_root), 1905 OID_AUTO, "curtick", CTLFLAG_RD, 1906 &hpts->p_curtick, 0, 1907 "What the current tick on if active"); 1908 SYSCTL_ADD_UINT(&hpts->hpts_ctx, 1909 SYSCTL_CHILDREN(hpts->hpts_root), 1910 OID_AUTO, "logsize", CTLFLAG_RD, 1911 &hpts->p_logsize, 0, 1912 "Hpts logging buffer size"); 1913 hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2; 1914 hpts->p_num = i; 1915 hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv); 1916 hpts->p_prevtick -= 1; 1917 hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS; 1918 hpts->p_cpu = 0xffff; 1919 hpts->p_nxt_slot = 1; 1920 hpts->p_logsize = tcp_hpts_logging_size; 1921 if (hpts->p_logsize) { 1922 sz = (sizeof(struct hpts_log) * hpts->p_logsize); 1923 hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO); 1924 } 1925 callout_init(&hpts->co, 1); 1926 } 1927 /* 1928 * Now lets start ithreads to handle the hptss. 1929 */ 1930 CPU_FOREACH(i) { 1931 hpts = tcp_pace.rp_ent[i]; 1932 hpts->p_cpu = i; 1933 error = swi_add(&hpts->ie, "hpts", 1934 tcp_hpts_thread, (void *)hpts, 1935 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); 1936 if (error) { 1937 panic("Can't add hpts:%p i:%d err:%d", 1938 hpts, i, error); 1939 } 1940 created++; 1941 if (tcp_bind_threads) { 1942 if (intr_event_bind(hpts->ie, i) == 0) 1943 bound++; 1944 } 1945 tv.tv_sec = 0; 1946 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC; 1947 sb = tvtosbt(tv); 1948 if (tcp_hpts_callout_skip_swi == 0) { 1949 callout_reset_sbt_on(&hpts->co, sb, 0, 1950 hpts_timeout_swi, hpts, hpts->p_cpu, 1951 (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision))); 1952 } else { 1953 callout_reset_sbt_on(&hpts->co, sb, 0, 1954 hpts_timeout_dir, hpts, 1955 hpts->p_cpu, 1956 C_PREL(tcp_hpts_precision)); 1957 } 1958 } 1959 printf("TCP Hpts created %d swi interrupt thread and bound %d\n", 1960 created, bound); 1961 return; 1962 } 1963 1964 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL); 1965