xref: /freebsd/sys/netinet/tcp_hpts.c (revision 47dd1d1b619cc035b82b49a91a25544309ff95ae)
1 /*-
2  * Copyright (c) 2016-2018 Netflix Inc.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  */
26 #include <sys/cdefs.h>
27 __FBSDID("$FreeBSD$");
28 
29 #include "opt_inet.h"
30 #include "opt_inet6.h"
31 #include "opt_ipsec.h"
32 #include "opt_tcpdebug.h"
33 /**
34  * Some notes about usage.
35  *
36  * The tcp_hpts system is designed to provide a high precision timer
37  * system for tcp. Its main purpose is to provide a mechanism for
38  * pacing packets out onto the wire. It can be used in two ways
39  * by a given TCP stack (and those two methods can be used simultaneously).
40  *
41  * First, and probably the main thing its used by Rack and BBR for, it can
42  * be used to call tcp_output() of a transport stack at some time in the future.
43  * The normal way this is done is that tcp_output() of the stack schedules
44  * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The
45  * slot is the time from now that the stack wants to be called but it
46  * must be converted to tcp_hpts's notion of slot. This is done with
47  * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical
48  * call from the tcp_output() routine might look like:
49  *
50  * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));
51  *
52  * The above would schedule tcp_ouput() to be called in 550 useconds.
53  * Note that if using this mechanism the stack will want to add near
54  * its top a check to prevent unwanted calls (from user land or the
55  * arrival of incoming ack's). So it would add something like:
56  *
57  * if (inp->inp_in_hpts)
58  *    return;
59  *
60  * to prevent output processing until the time alotted has gone by.
61  * Of course this is a bare bones example and the stack will probably
62  * have more consideration then just the above.
63  *
64  * Now the tcp_hpts system will call tcp_output in one of two forms,
65  * it will first check to see if the stack as defined a
66  * tfb_tcp_output_wtime() function, if so that is the routine it
67  * will call, if that function is not defined then it will call the
68  * tfb_tcp_output() function. The only difference between these
69  * two calls is that the former passes the time in to the function
70  * so the function does not have to access the time (which tcp_hpts
71  * already has). What these functions do is of course totally up
72  * to the individual tcp stack.
73  *
74  * Now the second function (actually two functions I guess :D)
75  * the tcp_hpts system provides is the  ability to either abort
76  * a connection (later) or process  input on a connection.
77  * Why would you want to do this? To keep processor locality.
78  *
79  * So in order to use the input redirection function the
80  * stack changes its tcp_do_segment() routine to instead
81  * of process the data call the function:
82  *
83  * tcp_queue_pkt_to_input()
84  *
85  * You will note that the arguments to this function look
86  * a lot like tcp_do_segments's arguments. This function
87  * will assure that the tcp_hpts system will
88  * call the functions tfb_tcp_hpts_do_segment() from the
89  * correct CPU. Note that multiple calls can get pushed
90  * into the tcp_hpts system this will be indicated by
91  * the next to last argument to tfb_tcp_hpts_do_segment()
92  * (nxt_pkt). If nxt_pkt is a 1 then another packet is
93  * coming. If nxt_pkt is a 0 then this is the last call
94  * that the tcp_hpts system has available for the tcp stack.
95  *
96  * The other point of the input system is to be able to safely
97  * drop a tcp connection without worrying about the recursive
98  * locking that may be occuring on the INP_WLOCK. So if
99  * a stack wants to drop a connection it calls:
100  *
101  *     tcp_set_inp_to_drop(tp, ETIMEDOUT)
102  *
103  * To schedule the tcp_hpts system to call
104  *
105  *    tcp_drop(tp, drop_reason)
106  *
107  * at a future point. This is quite handy to prevent locking
108  * issues when dropping connections.
109  *
110  */
111 
112 #include <sys/param.h>
113 #include <sys/bus.h>
114 #include <sys/interrupt.h>
115 #include <sys/module.h>
116 #include <sys/kernel.h>
117 #include <sys/hhook.h>
118 #include <sys/malloc.h>
119 #include <sys/mbuf.h>
120 #include <sys/proc.h>		/* for proc0 declaration */
121 #include <sys/socket.h>
122 #include <sys/socketvar.h>
123 #include <sys/sysctl.h>
124 #include <sys/systm.h>
125 #include <sys/refcount.h>
126 #include <sys/sched.h>
127 #include <sys/queue.h>
128 #include <sys/smp.h>
129 #include <sys/counter.h>
130 #include <sys/time.h>
131 #include <sys/kthread.h>
132 #include <sys/kern_prefetch.h>
133 
134 #include <vm/uma.h>
135 
136 #include <net/route.h>
137 #include <net/vnet.h>
138 
139 #define TCPSTATES		/* for logging */
140 
141 #include <netinet/in.h>
142 #include <netinet/in_kdtrace.h>
143 #include <netinet/in_pcb.h>
144 #include <netinet/ip.h>
145 #include <netinet/ip_icmp.h>	/* required for icmp_var.h */
146 #include <netinet/icmp_var.h>	/* for ICMP_BANDLIM */
147 #include <netinet/ip_var.h>
148 #include <netinet/ip6.h>
149 #include <netinet6/in6_pcb.h>
150 #include <netinet6/ip6_var.h>
151 #define	TCPOUTFLAGS
152 #include <netinet/tcp.h>
153 #include <netinet/tcp_fsm.h>
154 #include <netinet/tcp_seq.h>
155 #include <netinet/tcp_timer.h>
156 #include <netinet/tcp_var.h>
157 #include <netinet/tcpip.h>
158 #include <netinet/cc/cc.h>
159 #include <netinet/tcp_hpts.h>
160 
161 #ifdef tcpdebug
162 #include <netinet/tcp_debug.h>
163 #endif				/* tcpdebug */
164 #ifdef tcp_offload
165 #include <netinet/tcp_offload.h>
166 #endif
167 
168 #ifdef ipsec
169 #include <netipsec/ipsec.h>
170 #include <netipsec/ipsec6.h>
171 #endif				/* ipsec */
172 #include "opt_rss.h"
173 
174 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts");
175 #ifdef RSS
176 static int tcp_bind_threads = 1;
177 #else
178 static int tcp_bind_threads = 0;
179 #endif
180 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);
181 
182 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;
183 
184 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);
185 
186 static struct tcp_hptsi tcp_pace;
187 
188 static int
189 tcp_hptsi_lock_inpinfo(struct inpcb *inp,
190     struct tcpcb **tp);
191 static void tcp_wakehpts(struct tcp_hpts_entry *p);
192 static void tcp_wakeinput(struct tcp_hpts_entry *p);
193 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv);
194 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick);
195 static void tcp_hpts_thread(void *ctx);
196 static void tcp_init_hptsi(void *st);
197 
198 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;
199 static int32_t tcp_hpts_callout_skip_swi = 0;
200 
201 SYSCTL_DECL(_net_inet_tcp);
202 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls");
203 
204 #define	timersub(tvp, uvp, vvp)						\
205 	do {								\
206 		(vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;		\
207 		(vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;	\
208 		if ((vvp)->tv_usec < 0) {				\
209 			(vvp)->tv_sec--;				\
210 			(vvp)->tv_usec += 1000000;			\
211 		}							\
212 	} while (0)
213 
214 static int32_t logging_on = 0;
215 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2);
216 static int32_t tcp_hpts_precision = 120;
217 
218 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW,
219     &tcp_hpts_precision, 120,
220     "Value for PRE() precision of callout");
221 
222 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW,
223     &logging_on, 0,
224     "Turn on logging if compiled in");
225 
226 counter_u64_t hpts_loops;
227 
228 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD,
229     &hpts_loops, "Number of times hpts had to loop to catch up");
230 
231 counter_u64_t back_tosleep;
232 
233 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD,
234     &back_tosleep, "Number of times hpts found no tcbs");
235 
236 static int32_t in_newts_every_tcb = 0;
237 
238 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW,
239     &in_newts_every_tcb, 0,
240     "Do we have a new cts every tcb we process for input");
241 static int32_t in_ts_percision = 0;
242 
243 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW,
244     &in_ts_percision, 0,
245     "Do we use percise timestamp for clients on input");
246 static int32_t out_newts_every_tcb = 0;
247 
248 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW,
249     &out_newts_every_tcb, 0,
250     "Do we have a new cts every tcb we process for output");
251 static int32_t out_ts_percision = 0;
252 
253 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW,
254     &out_ts_percision, 0,
255     "Do we use a percise timestamp for every output cts");
256 
257 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW,
258     &hpts_sleep_max, 0,
259     "The maximum time the hpts will sleep <1 - 254>");
260 
261 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW,
262     &tcp_min_hptsi_time, 0,
263     "The minimum time the hpts must sleep before processing more slots");
264 
265 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW,
266     &tcp_hpts_callout_skip_swi, 0,
267     "Do we have the callout call directly to the hpts?");
268 
269 static void
270 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot,
271     uint32_t ticknow, int32_t line)
272 {
273 	struct hpts_log *pl;
274 
275 	HPTS_MTX_ASSERT(hpts);
276 	if (hpts->p_log == NULL)
277 		return;
278 	pl = &hpts->p_log[hpts->p_log_at];
279 	hpts->p_log_at++;
280 	if (hpts->p_log_at >= hpts->p_logsize) {
281 		hpts->p_log_at = 0;
282 		hpts->p_log_wrapped = 1;
283 	}
284 	pl->inp = inp;
285 	if (inp) {
286 		pl->t_paceslot = inp->inp_hptsslot;
287 		pl->t_hptsreq = inp->inp_hpts_request;
288 		pl->p_onhpts = inp->inp_in_hpts;
289 		pl->p_oninput = inp->inp_in_input;
290 	} else {
291 		pl->t_paceslot = 0;
292 		pl->t_hptsreq = 0;
293 		pl->p_onhpts = 0;
294 		pl->p_oninput = 0;
295 	}
296 	pl->is_notempty = 1;
297 	pl->event = event;
298 	pl->line = line;
299 	pl->cts = tcp_get_usecs(NULL);
300 	pl->p_curtick = hpts->p_curtick;
301 	pl->p_prevtick = hpts->p_prevtick;
302 	pl->p_on_queue_cnt = hpts->p_on_queue_cnt;
303 	pl->ticknow = ticknow;
304 	pl->slot_req = slot;
305 	pl->p_nxt_slot = hpts->p_nxt_slot;
306 	pl->p_cur_slot = hpts->p_cur_slot;
307 	pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time;
308 	pl->p_flags = (hpts->p_cpu & 0x7f);
309 	pl->p_flags <<= 7;
310 	pl->p_flags |= (hpts->p_num & 0x7f);
311 	pl->p_flags <<= 2;
312 	if (hpts->p_hpts_active) {
313 		pl->p_flags |= HPTS_HPTS_ACTIVE;
314 	}
315 }
316 
317 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__)
318 
319 static void
320 hpts_timeout_swi(void *arg)
321 {
322 	struct tcp_hpts_entry *hpts;
323 
324 	hpts = (struct tcp_hpts_entry *)arg;
325 	swi_sched(hpts->ie_cookie, 0);
326 }
327 
328 static void
329 hpts_timeout_dir(void *arg)
330 {
331 	tcp_hpts_thread(arg);
332 }
333 
334 static inline void
335 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear)
336 {
337 #ifdef INVARIANTS
338 	if (mtx_owned(&hpts->p_mtx) == 0) {
339 		/* We don't own the mutex? */
340 		panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
341 	}
342 	if (hpts->p_cpu != inp->inp_hpts_cpu) {
343 		/* It is not the right cpu/mutex? */
344 		panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
345 	}
346 	if (inp->inp_in_hpts == 0) {
347 		/* We are not on the hpts? */
348 		panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp);
349 	}
350 	if (TAILQ_EMPTY(head) &&
351 	    (hpts->p_on_queue_cnt != 0)) {
352 		/* We should not be empty with a queue count */
353 		panic("%s hpts:%p hpts bucket empty but cnt:%d",
354 		    __FUNCTION__, hpts, hpts->p_on_queue_cnt);
355 	}
356 #endif
357 	TAILQ_REMOVE(head, inp, inp_hpts);
358 	hpts->p_on_queue_cnt--;
359 	if (hpts->p_on_queue_cnt < 0) {
360 		/* Count should not go negative .. */
361 #ifdef INVARIANTS
362 		panic("Hpts goes negative inp:%p hpts:%p",
363 		    inp, hpts);
364 #endif
365 		hpts->p_on_queue_cnt = 0;
366 	}
367 	if (clear) {
368 		inp->inp_hpts_request = 0;
369 		inp->inp_in_hpts = 0;
370 	}
371 }
372 
373 static inline void
374 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref)
375 {
376 #ifdef INVARIANTS
377 	if (mtx_owned(&hpts->p_mtx) == 0) {
378 		/* We don't own the mutex? */
379 		panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
380 	}
381 	if (hpts->p_cpu != inp->inp_hpts_cpu) {
382 		/* It is not the right cpu/mutex? */
383 		panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
384 	}
385 	if ((noref == 0) && (inp->inp_in_hpts == 1)) {
386 		/* We are already on the hpts? */
387 		panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp);
388 	}
389 #endif
390 	TAILQ_INSERT_TAIL(head, inp, inp_hpts);
391 	inp->inp_in_hpts = 1;
392 	hpts->p_on_queue_cnt++;
393 	if (noref == 0) {
394 		in_pcbref(inp);
395 	}
396 }
397 
398 static inline void
399 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear)
400 {
401 #ifdef INVARIANTS
402 	if (mtx_owned(&hpts->p_mtx) == 0) {
403 		/* We don't own the mutex? */
404 		panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
405 	}
406 	if (hpts->p_cpu != inp->inp_input_cpu) {
407 		/* It is not the right cpu/mutex? */
408 		panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
409 	}
410 	if (inp->inp_in_input == 0) {
411 		/* We are not on the input hpts? */
412 		panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp);
413 	}
414 #endif
415 	TAILQ_REMOVE(&hpts->p_input, inp, inp_input);
416 	hpts->p_on_inqueue_cnt--;
417 	if (hpts->p_on_inqueue_cnt < 0) {
418 #ifdef INVARIANTS
419 		panic("Hpts in goes negative inp:%p hpts:%p",
420 		    inp, hpts);
421 #endif
422 		hpts->p_on_inqueue_cnt = 0;
423 	}
424 #ifdef INVARIANTS
425 	if (TAILQ_EMPTY(&hpts->p_input) &&
426 	    (hpts->p_on_inqueue_cnt != 0)) {
427 		/* We should not be empty with a queue count */
428 		panic("%s hpts:%p in_hpts input empty but cnt:%d",
429 		    __FUNCTION__, hpts, hpts->p_on_inqueue_cnt);
430 	}
431 #endif
432 	if (clear)
433 		inp->inp_in_input = 0;
434 }
435 
436 static inline void
437 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line)
438 {
439 #ifdef INVARIANTS
440 	if (mtx_owned(&hpts->p_mtx) == 0) {
441 		/* We don't own the mutex? */
442 		panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
443 	}
444 	if (hpts->p_cpu != inp->inp_input_cpu) {
445 		/* It is not the right cpu/mutex? */
446 		panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
447 	}
448 	if (inp->inp_in_input == 1) {
449 		/* We are already on the input hpts? */
450 		panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp);
451 	}
452 #endif
453 	TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input);
454 	inp->inp_in_input = 1;
455 	hpts->p_on_inqueue_cnt++;
456 	in_pcbref(inp);
457 }
458 
459 static int
460 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS)
461 {
462 	struct tcp_hpts_entry *hpts;
463 	size_t sz;
464 	int32_t logging_was, i;
465 	int32_t error = 0;
466 
467 	/*
468 	 * HACK: Turn off logging so no locks are required this really needs
469 	 * a memory barrier :)
470 	 */
471 	logging_was = logging_on;
472 	logging_on = 0;
473 	if (!req->oldptr) {
474 		/* How much? */
475 		sz = 0;
476 		for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
477 			hpts = tcp_pace.rp_ent[i];
478 			if (hpts->p_log == NULL)
479 				continue;
480 			sz += (sizeof(struct hpts_log) * hpts->p_logsize);
481 		}
482 		error = SYSCTL_OUT(req, 0, sz);
483 	} else {
484 		for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
485 			hpts = tcp_pace.rp_ent[i];
486 			if (hpts->p_log == NULL)
487 				continue;
488 			if (hpts->p_log_wrapped)
489 				sz = (sizeof(struct hpts_log) * hpts->p_logsize);
490 			else
491 				sz = (sizeof(struct hpts_log) * hpts->p_log_at);
492 			error = SYSCTL_OUT(req, hpts->p_log, sz);
493 		}
494 	}
495 	logging_on = logging_was;
496 	return error;
497 }
498 
499 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
500     0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");
501 
502 
503 /*
504  * Try to get the INP_INFO lock.
505  *
506  * This function always succeeds in getting the lock. It will clear
507  * *tpp and return (1) if something critical changed while the inpcb
508  * was unlocked. Otherwise, it will leave *tpp unchanged and return (0).
509  *
510  * This function relies on the fact that the hpts always holds a
511  * reference on the inpcb while the segment is on the hptsi wheel and
512  * in the input queue.
513  *
514  */
515 static int
516 tcp_hptsi_lock_inpinfo(struct inpcb *inp, struct tcpcb **tpp)
517 {
518 	struct tcp_function_block *tfb;
519 	struct tcpcb *tp;
520 	void *ptr;
521 
522 	/* Try the easy way. */
523 	if (INP_INFO_TRY_RLOCK(&V_tcbinfo))
524 		return (0);
525 
526 	/*
527 	 * OK, let's try the hard way. We'll save the function pointer block
528 	 * to make sure that doesn't change while we aren't holding the
529 	 * lock.
530 	 */
531 	tp = *tpp;
532 	tfb = tp->t_fb;
533 	ptr = tp->t_fb_ptr;
534 	INP_WUNLOCK(inp);
535 	INP_INFO_RLOCK(&V_tcbinfo);
536 	INP_WLOCK(inp);
537 	/* If the session went away, return an error. */
538 	if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
539 	    (inp->inp_flags2 & INP_FREED)) {
540 		*tpp = NULL;
541 		return (1);
542 	}
543 	/*
544 	 * If the function block or stack-specific data block changed,
545 	 * report an error.
546 	 */
547 	tp = intotcpcb(inp);
548 	if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) {
549 		*tpp = NULL;
550 		return (1);
551 	}
552 	return (0);
553 }
554 
555 
556 static void
557 tcp_wakehpts(struct tcp_hpts_entry *hpts)
558 {
559 	HPTS_MTX_ASSERT(hpts);
560 	swi_sched(hpts->ie_cookie, 0);
561 	if (hpts->p_hpts_active == 2) {
562 		/* Rare sleeping on a ENOBUF */
563 		wakeup_one(hpts);
564 	}
565 }
566 
567 static void
568 tcp_wakeinput(struct tcp_hpts_entry *hpts)
569 {
570 	HPTS_MTX_ASSERT(hpts);
571 	swi_sched(hpts->ie_cookie, 0);
572 	if (hpts->p_hpts_active == 2) {
573 		/* Rare sleeping on a ENOBUF */
574 		wakeup_one(hpts);
575 	}
576 }
577 
578 struct tcp_hpts_entry *
579 tcp_cur_hpts(struct inpcb *inp)
580 {
581 	int32_t hpts_num;
582 	struct tcp_hpts_entry *hpts;
583 
584 	hpts_num = inp->inp_hpts_cpu;
585 	hpts = tcp_pace.rp_ent[hpts_num];
586 	return (hpts);
587 }
588 
589 struct tcp_hpts_entry *
590 tcp_hpts_lock(struct inpcb *inp)
591 {
592 	struct tcp_hpts_entry *hpts;
593 	int32_t hpts_num;
594 
595 again:
596 	hpts_num = inp->inp_hpts_cpu;
597 	hpts = tcp_pace.rp_ent[hpts_num];
598 #ifdef INVARIANTS
599 	if (mtx_owned(&hpts->p_mtx)) {
600 		panic("Hpts:%p owns mtx prior-to lock line:%d",
601 		    hpts, __LINE__);
602 	}
603 #endif
604 	mtx_lock(&hpts->p_mtx);
605 	if (hpts_num != inp->inp_hpts_cpu) {
606 		mtx_unlock(&hpts->p_mtx);
607 		goto again;
608 	}
609 	return (hpts);
610 }
611 
612 struct tcp_hpts_entry *
613 tcp_input_lock(struct inpcb *inp)
614 {
615 	struct tcp_hpts_entry *hpts;
616 	int32_t hpts_num;
617 
618 again:
619 	hpts_num = inp->inp_input_cpu;
620 	hpts = tcp_pace.rp_ent[hpts_num];
621 #ifdef INVARIANTS
622 	if (mtx_owned(&hpts->p_mtx)) {
623 		panic("Hpts:%p owns mtx prior-to lock line:%d",
624 		    hpts, __LINE__);
625 	}
626 #endif
627 	mtx_lock(&hpts->p_mtx);
628 	if (hpts_num != inp->inp_input_cpu) {
629 		mtx_unlock(&hpts->p_mtx);
630 		goto again;
631 	}
632 	return (hpts);
633 }
634 
635 static void
636 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line)
637 {
638 	int32_t add_freed;
639 
640 	if (inp->inp_flags2 & INP_FREED) {
641 		/*
642 		 * Need to play a special trick so that in_pcbrele_wlocked
643 		 * does not return 1 when it really should have returned 0.
644 		 */
645 		add_freed = 1;
646 		inp->inp_flags2 &= ~INP_FREED;
647 	} else {
648 		add_freed = 0;
649 	}
650 #ifndef INP_REF_DEBUG
651 	if (in_pcbrele_wlocked(inp)) {
652 		/*
653 		 * This should not happen. We have the inpcb referred to by
654 		 * the main socket (why we are called) and the hpts. It
655 		 * should always return 0.
656 		 */
657 		panic("inpcb:%p release ret 1",
658 		    inp);
659 	}
660 #else
661 	if (__in_pcbrele_wlocked(inp, line)) {
662 		/*
663 		 * This should not happen. We have the inpcb referred to by
664 		 * the main socket (why we are called) and the hpts. It
665 		 * should always return 0.
666 		 */
667 		panic("inpcb:%p release ret 1",
668 		    inp);
669 	}
670 #endif
671 	if (add_freed) {
672 		inp->inp_flags2 |= INP_FREED;
673 	}
674 }
675 
676 static void
677 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
678 {
679 	if (inp->inp_in_hpts) {
680 		hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1);
681 		tcp_remove_hpts_ref(inp, hpts, line);
682 	}
683 }
684 
685 static void
686 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
687 {
688 	HPTS_MTX_ASSERT(hpts);
689 	if (inp->inp_in_input) {
690 		hpts_sane_input_remove(hpts, inp, 1);
691 		tcp_remove_hpts_ref(inp, hpts, line);
692 	}
693 }
694 
695 /*
696  * Called normally with the INP_LOCKED but it
697  * does not matter, the hpts lock is the key
698  * but the lock order allows us to hold the
699  * INP lock and then get the hpts lock.
700  *
701  * Valid values in the flags are
702  * HPTS_REMOVE_OUTPUT - remove from the output of the hpts.
703  * HPTS_REMOVE_INPUT - remove from the input of the hpts.
704  * Note that you can or both values together and get two
705  * actions.
706  */
707 void
708 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line)
709 {
710 	struct tcp_hpts_entry *hpts;
711 
712 	INP_WLOCK_ASSERT(inp);
713 	if (flags & HPTS_REMOVE_OUTPUT) {
714 		hpts = tcp_hpts_lock(inp);
715 		tcp_hpts_remove_locked_output(hpts, inp, flags, line);
716 		mtx_unlock(&hpts->p_mtx);
717 	}
718 	if (flags & HPTS_REMOVE_INPUT) {
719 		hpts = tcp_input_lock(inp);
720 		tcp_hpts_remove_locked_input(hpts, inp, flags, line);
721 		mtx_unlock(&hpts->p_mtx);
722 	}
723 }
724 
725 static inline int
726 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus)
727 {
728 	return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS);
729 }
730 
731 static int
732 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref)
733 {
734 	int32_t need_wake = 0;
735 	uint32_t ticknow = 0;
736 
737 	HPTS_MTX_ASSERT(hpts);
738 	if (inp->inp_in_hpts == 0) {
739 		/* Ok we need to set it on the hpts in the current slot */
740 		if (hpts->p_hpts_active == 0) {
741 			/* A sleeping hpts we want in next slot to run */
742 			if (logging_on) {
743 				tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0,
744 				    hpts_tick(hpts, 1));
745 			}
746 			inp->inp_hptsslot = hpts_tick(hpts, 1);
747 			inp->inp_hpts_request = 0;
748 			if (logging_on) {
749 				tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow);
750 			}
751 			need_wake = 1;
752 		} else if ((void *)inp == hpts->p_inp) {
753 			/*
754 			 * We can't allow you to go into the same slot we
755 			 * are in. We must put you out.
756 			 */
757 			inp->inp_hptsslot = hpts->p_nxt_slot;
758 		} else
759 			inp->inp_hptsslot = hpts->p_cur_slot;
760 		hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
761 		inp->inp_hpts_request = 0;
762 		if (logging_on) {
763 			tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0);
764 		}
765 		if (need_wake) {
766 			/*
767 			 * Activate the hpts if it is sleeping and its
768 			 * timeout is not 1.
769 			 */
770 			if (logging_on) {
771 				tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow);
772 			}
773 			hpts->p_direct_wake = 1;
774 			tcp_wakehpts(hpts);
775 		}
776 	}
777 	return (need_wake);
778 }
779 
780 int
781 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line)
782 {
783 	int32_t ret;
784 	struct tcp_hpts_entry *hpts;
785 
786 	INP_WLOCK_ASSERT(inp);
787 	hpts = tcp_hpts_lock(inp);
788 	ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0);
789 	mtx_unlock(&hpts->p_mtx);
790 	return (ret);
791 }
792 
793 static void
794 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line,
795     struct hpts_diag *diag, int32_t noref)
796 {
797 	int32_t need_new_to = 0;
798 	int32_t need_wakeup = 0;
799 	uint32_t largest_slot;
800 	uint32_t ticknow = 0;
801 	uint32_t slot_calc;
802 
803 	HPTS_MTX_ASSERT(hpts);
804 	if (diag) {
805 		memset(diag, 0, sizeof(struct hpts_diag));
806 		diag->p_hpts_active = hpts->p_hpts_active;
807 		diag->p_nxt_slot = hpts->p_nxt_slot;
808 		diag->p_cur_slot = hpts->p_cur_slot;
809 		diag->slot_req = slot;
810 	}
811 	if ((inp->inp_in_hpts == 0) || noref) {
812 		inp->inp_hpts_request = slot;
813 		if (slot == 0) {
814 			/* Immediate */
815 			tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref);
816 			return;
817 		}
818 		if (hpts->p_hpts_active) {
819 			/*
820 			 * Its slot - 1 since nxt_slot is the next tick that
821 			 * will go off since the hpts is awake
822 			 */
823 			if (logging_on) {
824 				tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0);
825 			}
826 			/*
827 			 * We want to make sure that we don't place a inp in
828 			 * the range of p_cur_slot <-> p_nxt_slot. If we
829 			 * take from p_nxt_slot to the end, plus p_cur_slot
830 			 * and then take away 2, we will know how many is
831 			 * the max slots we can use.
832 			 */
833 			if (hpts->p_nxt_slot > hpts->p_cur_slot) {
834 				/*
835 				 * Non-wrap case nxt_slot <-> cur_slot we
836 				 * don't want to land in. So the diff gives
837 				 * us what is taken away from the number of
838 				 * slots.
839 				 */
840 				largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot);
841 			} else if (hpts->p_nxt_slot == hpts->p_cur_slot) {
842 				largest_slot = NUM_OF_HPTSI_SLOTS - 2;
843 			} else {
844 				/*
845 				 * Wrap case so the diff gives us the number
846 				 * of slots that we can land in.
847 				 */
848 				largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot;
849 			}
850 			/*
851 			 * We take away two so we never have a problem (20
852 			 * usec's) out of 1024000 usecs
853 			 */
854 			largest_slot -= 2;
855 			if (inp->inp_hpts_request > largest_slot) {
856 				/*
857 				 * Restrict max jump of slots and remember
858 				 * leftover
859 				 */
860 				slot = largest_slot;
861 				inp->inp_hpts_request -= largest_slot;
862 			} else {
863 				/* This one will run when we hit it */
864 				inp->inp_hpts_request = 0;
865 			}
866 			if (hpts->p_nxt_slot == hpts->p_cur_slot)
867 				slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS;
868 			else
869 				slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS;
870 			if (slot_calc == hpts->p_cur_slot) {
871 #ifdef INVARIANTS
872 				/* TSNH */
873 				panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n",
874 				    hpts, slot_calc, slot, largest_slot);
875 #endif
876 				if (slot_calc)
877 					slot_calc--;
878 				else
879 					slot_calc = NUM_OF_HPTSI_SLOTS - 1;
880 			}
881 			inp->inp_hptsslot = slot_calc;
882 			if (diag) {
883 				diag->inp_hptsslot = inp->inp_hptsslot;
884 			}
885 		} else {
886 			/*
887 			 * The hpts is sleeping, we need to figure out where
888 			 * it will wake up at and if we need to reschedule
889 			 * its time-out.
890 			 */
891 			uint32_t have_slept, yet_to_sleep;
892 			uint32_t slot_now;
893 			struct timeval tv;
894 
895 			ticknow = tcp_gethptstick(&tv);
896 			slot_now = ticknow % NUM_OF_HPTSI_SLOTS;
897 			/*
898 			 * The user wants to be inserted at (slot_now +
899 			 * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up.
900 			 */
901 			largest_slot = NUM_OF_HPTSI_SLOTS - 2;
902 			if (inp->inp_hpts_request > largest_slot) {
903 				/* Adjust the residual in inp_hpts_request */
904 				slot = largest_slot;
905 				inp->inp_hpts_request -= largest_slot;
906 			} else {
907 				/* No residual it all fits */
908 				inp->inp_hpts_request = 0;
909 			}
910 			inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS;
911 			if (diag) {
912 				diag->slot_now = slot_now;
913 				diag->inp_hptsslot = inp->inp_hptsslot;
914 				diag->p_on_min_sleep = hpts->p_on_min_sleep;
915 			}
916 			if (logging_on) {
917 				tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow);
918 			}
919 			/* Now do we need to restart the hpts's timer? */
920 			if (TSTMP_GT(ticknow, hpts->p_curtick))
921 				have_slept = ticknow - hpts->p_curtick;
922 			else
923 				have_slept = 0;
924 			if (have_slept < hpts->p_hpts_sleep_time) {
925 				/* This should be what happens */
926 				yet_to_sleep = hpts->p_hpts_sleep_time - have_slept;
927 			} else {
928 				/* We are over-due */
929 				yet_to_sleep = 0;
930 				need_wakeup = 1;
931 			}
932 			if (diag) {
933 				diag->have_slept = have_slept;
934 				diag->yet_to_sleep = yet_to_sleep;
935 				diag->hpts_sleep_time = hpts->p_hpts_sleep_time;
936 			}
937 			if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) {
938 				/*
939 				 * We need to reschedule the hptss time-out.
940 				 */
941 				hpts->p_hpts_sleep_time = slot;
942 				need_new_to = slot * HPTS_TICKS_PER_USEC;
943 			}
944 		}
945 		hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
946 		if (logging_on) {
947 			tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow);
948 		}
949 		/*
950 		 * Now how far is the hpts sleeping to? if active is 1, its
951 		 * up and ticking we do nothing, otherwise we may need to
952 		 * reschedule its callout if need_new_to is set from above.
953 		 */
954 		if (need_wakeup) {
955 			if (logging_on) {
956 				tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0);
957 			}
958 			hpts->p_direct_wake = 1;
959 			tcp_wakehpts(hpts);
960 			if (diag) {
961 				diag->need_new_to = 0;
962 				diag->co_ret = 0xffff0000;
963 			}
964 		} else if (need_new_to) {
965 			int32_t co_ret;
966 			struct timeval tv;
967 			sbintime_t sb;
968 
969 			tv.tv_sec = 0;
970 			tv.tv_usec = 0;
971 			while (need_new_to > HPTS_USEC_IN_SEC) {
972 				tv.tv_sec++;
973 				need_new_to -= HPTS_USEC_IN_SEC;
974 			}
975 			tv.tv_usec = need_new_to;
976 			sb = tvtosbt(tv);
977 			if (tcp_hpts_callout_skip_swi == 0) {
978 				co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
979 				    hpts_timeout_swi, hpts, hpts->p_cpu,
980 				    (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
981 			} else {
982 				co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
983 				    hpts_timeout_dir, hpts,
984 				    hpts->p_cpu,
985 				    C_PREL(tcp_hpts_precision));
986 			}
987 			if (diag) {
988 				diag->need_new_to = need_new_to;
989 				diag->co_ret = co_ret;
990 			}
991 		}
992 	} else {
993 #ifdef INVARIANTS
994 		panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp);
995 #endif
996 	}
997 }
998 
999 uint32_t
1000 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){
1001 	struct tcp_hpts_entry *hpts;
1002 	uint32_t slot_on, cts;
1003 	struct timeval tv;
1004 
1005 	/*
1006 	 * We now return the next-slot the hpts will be on, beyond its
1007 	 * current run (if up) or where it was when it stopped if it is
1008 	 * sleeping.
1009 	 */
1010 	INP_WLOCK_ASSERT(inp);
1011 	hpts = tcp_hpts_lock(inp);
1012 	if (in_ts_percision)
1013 		microuptime(&tv);
1014 	else
1015 		getmicrouptime(&tv);
1016 	cts = tcp_tv_to_usectick(&tv);
1017 	tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0);
1018 	slot_on = hpts->p_nxt_slot;
1019 	mtx_unlock(&hpts->p_mtx);
1020 	return (slot_on);
1021 }
1022 
1023 uint32_t
1024 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){
1025 	return (tcp_hpts_insert_diag(inp, slot, line, NULL));
1026 }
1027 
1028 int
1029 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line)
1030 {
1031 	int32_t retval = 0;
1032 
1033 	HPTS_MTX_ASSERT(hpts);
1034 	if (inp->inp_in_input == 0) {
1035 		/* Ok we need to set it on the hpts in the current slot */
1036 		hpts_sane_input_insert(hpts, inp, line);
1037 		retval = 1;
1038 		if (hpts->p_hpts_active == 0) {
1039 			/*
1040 			 * Activate the hpts if it is sleeping.
1041 			 */
1042 			if (logging_on) {
1043 				tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0);
1044 			}
1045 			retval = 2;
1046 			hpts->p_direct_wake = 1;
1047 			tcp_wakeinput(hpts);
1048 		}
1049 	} else if (hpts->p_hpts_active == 0) {
1050 		retval = 4;
1051 		hpts->p_direct_wake = 1;
1052 		tcp_wakeinput(hpts);
1053 	}
1054 	return (retval);
1055 }
1056 
1057 void
1058 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1059     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked)
1060 {
1061 	/* Setup packet for input first */
1062 	INP_WLOCK_ASSERT(tp->t_inpcb);
1063 	m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t));
1064 	m->m_pkthdr.pace_tlen = (uint16_t) tlen;
1065 	m->m_pkthdr.pace_drphdrlen = drop_hdrlen;
1066 	m->m_pkthdr.pace_tos = iptos;
1067 	m->m_pkthdr.pace_lock = (uint8_t) ti_locked;
1068 	if (tp->t_in_pkt == NULL) {
1069 		tp->t_in_pkt = m;
1070 		tp->t_tail_pkt = m;
1071 	} else {
1072 		tp->t_tail_pkt->m_nextpkt = m;
1073 		tp->t_tail_pkt = m;
1074 	}
1075 }
1076 
1077 
1078 int32_t
1079 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1080     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line){
1081 	struct tcp_hpts_entry *hpts;
1082 	int32_t ret;
1083 
1084 	tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos, ti_locked);
1085 	hpts = tcp_input_lock(tp->t_inpcb);
1086 	ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line);
1087 	mtx_unlock(&hpts->p_mtx);
1088 	return (ret);
1089 }
1090 
1091 void
1092 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line)
1093 {
1094 	struct tcp_hpts_entry *hpts;
1095 	struct tcpcb *tp;
1096 
1097 	tp = intotcpcb(inp);
1098 	hpts = tcp_input_lock(tp->t_inpcb);
1099 	if (inp->inp_in_input == 0) {
1100 		/* Ok we need to set it on the hpts in the current slot */
1101 		hpts_sane_input_insert(hpts, inp, line);
1102 		if (hpts->p_hpts_active == 0) {
1103 			/*
1104 			 * Activate the hpts if it is sleeping.
1105 			 */
1106 			hpts->p_direct_wake = 1;
1107 			tcp_wakeinput(hpts);
1108 		}
1109 	} else if (hpts->p_hpts_active == 0) {
1110 		hpts->p_direct_wake = 1;
1111 		tcp_wakeinput(hpts);
1112 	}
1113 	inp->inp_hpts_drop_reas = reason;
1114 	mtx_unlock(&hpts->p_mtx);
1115 }
1116 
1117 static uint16_t
1118 hpts_random_cpu(struct inpcb *inp){
1119 	/*
1120 	 * No flow type set distribute the load randomly.
1121 	 */
1122 	uint16_t cpuid;
1123 	uint32_t ran;
1124 
1125 	/*
1126 	 * If one has been set use it i.e. we want both in and out on the
1127 	 * same hpts.
1128 	 */
1129 	if (inp->inp_input_cpu_set) {
1130 		return (inp->inp_input_cpu);
1131 	} else if (inp->inp_hpts_cpu_set) {
1132 		return (inp->inp_hpts_cpu);
1133 	}
1134 	/* Nothing set use a random number */
1135 	ran = arc4random();
1136 	cpuid = (ran & 0xffff) % mp_ncpus;
1137 	return (cpuid);
1138 }
1139 
1140 static uint16_t
1141 hpts_cpuid(struct inpcb *inp){
1142 	uint16_t cpuid;
1143 
1144 
1145 	/*
1146 	 * If one has been set use it i.e. we want both in and out on the
1147 	 * same hpts.
1148 	 */
1149 	if (inp->inp_input_cpu_set) {
1150 		return (inp->inp_input_cpu);
1151 	} else if (inp->inp_hpts_cpu_set) {
1152 		return (inp->inp_hpts_cpu);
1153 	}
1154 	/* If one is set the other must be the same */
1155 #ifdef	RSS
1156 	cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
1157 	if (cpuid == NETISR_CPUID_NONE)
1158 		return (hpts_random_cpu(inp));
1159 	else
1160 		return (cpuid);
1161 #else
1162 	/*
1163 	 * We don't have a flowid -> cpuid mapping, so cheat and just map
1164 	 * unknown cpuids to curcpu.  Not the best, but apparently better
1165 	 * than defaulting to swi 0.
1166 	 */
1167 	if (inp->inp_flowtype != M_HASHTYPE_NONE) {
1168 		cpuid = inp->inp_flowid % mp_ncpus;
1169 		return (cpuid);
1170 	}
1171 	cpuid = hpts_random_cpu(inp);
1172 	return (cpuid);
1173 #endif
1174 }
1175 
1176 /*
1177  * Do NOT try to optimize the processing of inp's
1178  * by first pulling off all the inp's into a temporary
1179  * list (e.g. TAILQ_CONCAT). If you do that the subtle
1180  * interactions of switching CPU's will kill because of
1181  * problems in the linked list manipulation. Basically
1182  * you would switch cpu's with the hpts mutex locked
1183  * but then while you were processing one of the inp's
1184  * some other one that you switch will get a new
1185  * packet on the different CPU. It will insert it
1186  * on the new hptss input list. Creating a temporary
1187  * link in the inp will not fix it either, since
1188  * the other hpts will be doing the same thing and
1189  * you will both end up using the temporary link.
1190  *
1191  * You will die in an ASSERT for tailq corruption if you
1192  * run INVARIANTS or you will die horribly without
1193  * INVARIANTS in some unknown way with a corrupt linked
1194  * list.
1195  */
1196 static void
1197 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv)
1198 {
1199 	struct mbuf *m, *n;
1200 	struct tcpcb *tp;
1201 	struct inpcb *inp;
1202 	uint16_t drop_reason;
1203 	int16_t set_cpu;
1204 	uint32_t did_prefetch = 0;
1205 	int32_t ti_locked = TI_UNLOCKED;
1206 
1207 	HPTS_MTX_ASSERT(hpts);
1208 	while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) {
1209 		HPTS_MTX_ASSERT(hpts);
1210 		hpts_sane_input_remove(hpts, inp, 0);
1211 		if (inp->inp_input_cpu_set == 0) {
1212 			set_cpu = 1;
1213 		} else {
1214 			set_cpu = 0;
1215 		}
1216 		hpts->p_inp = inp;
1217 		drop_reason = inp->inp_hpts_drop_reas;
1218 		inp->inp_in_input = 0;
1219 		mtx_unlock(&hpts->p_mtx);
1220 		if (drop_reason) {
1221 			INP_INFO_RLOCK(&V_tcbinfo);
1222 			ti_locked = TI_RLOCKED;
1223 		} else {
1224 			ti_locked = TI_UNLOCKED;
1225 		}
1226 		INP_WLOCK(inp);
1227 		if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1228 		    (inp->inp_flags2 & INP_FREED)) {
1229 out:
1230 			hpts->p_inp = NULL;
1231 			if (ti_locked == TI_RLOCKED) {
1232 				INP_INFO_RUNLOCK(&V_tcbinfo);
1233 			}
1234 			if (in_pcbrele_wlocked(inp) == 0) {
1235 				INP_WUNLOCK(inp);
1236 			}
1237 			ti_locked = TI_UNLOCKED;
1238 			mtx_lock(&hpts->p_mtx);
1239 			continue;
1240 		}
1241 		tp = intotcpcb(inp);
1242 		if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1243 			goto out;
1244 		}
1245 		if (drop_reason) {
1246 			/* This tcb is being destroyed for drop_reason */
1247 			m = tp->t_in_pkt;
1248 			if (m)
1249 				n = m->m_nextpkt;
1250 			else
1251 				n = NULL;
1252 			tp->t_in_pkt = NULL;
1253 			while (m) {
1254 				m_freem(m);
1255 				m = n;
1256 				if (m)
1257 					n = m->m_nextpkt;
1258 			}
1259 			tp = tcp_drop(tp, drop_reason);
1260 			INP_INFO_RUNLOCK(&V_tcbinfo);
1261 			if (tp == NULL) {
1262 				INP_WLOCK(inp);
1263 			}
1264 			if (in_pcbrele_wlocked(inp) == 0)
1265 				INP_WUNLOCK(inp);
1266 			mtx_lock(&hpts->p_mtx);
1267 			continue;
1268 		}
1269 		if (set_cpu) {
1270 			/*
1271 			 * Setup so the next time we will move to the right
1272 			 * CPU. This should be a rare event. It will
1273 			 * sometimes happens when we are the client side
1274 			 * (usually not the server). Somehow tcp_output()
1275 			 * gets called before the tcp_do_segment() sets the
1276 			 * intial state. This means the r_cpu and r_hpts_cpu
1277 			 * is 0. We get on the hpts, and then tcp_input()
1278 			 * gets called setting up the r_cpu to the correct
1279 			 * value. The hpts goes off and sees the mis-match.
1280 			 * We simply correct it here and the CPU will switch
1281 			 * to the new hpts nextime the tcb gets added to the
1282 			 * the hpts (not this time) :-)
1283 			 */
1284 			tcp_set_hpts(inp);
1285 		}
1286 		CURVNET_SET(tp->t_vnet);
1287 		m = tp->t_in_pkt;
1288 		n = NULL;
1289 		if (m != NULL &&
1290 		    (m->m_pkthdr.pace_lock == TI_RLOCKED ||
1291 		    tp->t_state != TCPS_ESTABLISHED)) {
1292 			ti_locked = TI_RLOCKED;
1293 			if (tcp_hptsi_lock_inpinfo(inp, &tp)) {
1294 				CURVNET_RESTORE();
1295 				goto out;
1296 			}
1297 			m = tp->t_in_pkt;
1298 		}
1299 		if (in_newts_every_tcb) {
1300 			if (in_ts_percision)
1301 				microuptime(tv);
1302 			else
1303 				getmicrouptime(tv);
1304 		}
1305 		if (tp->t_fb_ptr != NULL) {
1306 			kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1307 			did_prefetch = 1;
1308 		}
1309 		/* Any input work to do, if so do it first */
1310 		if ((m != NULL) && (m == tp->t_in_pkt)) {
1311 			struct tcphdr *th;
1312 			int32_t tlen, drop_hdrlen, nxt_pkt;
1313 			uint8_t iptos;
1314 
1315 			n = m->m_nextpkt;
1316 			tp->t_in_pkt = tp->t_tail_pkt = NULL;
1317 			while (m) {
1318 				th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff);
1319 				tlen = m->m_pkthdr.pace_tlen;
1320 				drop_hdrlen = m->m_pkthdr.pace_drphdrlen;
1321 				iptos = m->m_pkthdr.pace_tos;
1322 				m->m_nextpkt = NULL;
1323 				if (n)
1324 					nxt_pkt = 1;
1325 				else
1326 					nxt_pkt = 0;
1327 				inp->inp_input_calls = 1;
1328 				if (tp->t_fb->tfb_tcp_hpts_do_segment) {
1329 					/* Use the hpts specific do_segment */
1330 					(*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket,
1331 					    tp, drop_hdrlen,
1332 					    tlen, iptos, ti_locked, nxt_pkt, tv);
1333 				} else {
1334 					/* Use the default do_segment */
1335 					(*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket,
1336 					    tp, drop_hdrlen,
1337 					    tlen, iptos, ti_locked);
1338 				}
1339 				/*
1340 				 * Do segment returns unlocked we need the
1341 				 * lock again but we also need some kasserts
1342 				 * here.
1343 				 */
1344 				INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1345 				INP_UNLOCK_ASSERT(inp);
1346 				m = n;
1347 				if (m)
1348 					n = m->m_nextpkt;
1349 				if (m != NULL &&
1350 				    m->m_pkthdr.pace_lock == TI_RLOCKED) {
1351 					INP_INFO_RLOCK(&V_tcbinfo);
1352 					ti_locked = TI_RLOCKED;
1353 				} else
1354 					ti_locked = TI_UNLOCKED;
1355 				INP_WLOCK(inp);
1356 				/*
1357 				 * Since we have an opening here we must
1358 				 * re-check if the tcb went away while we
1359 				 * were getting the lock(s).
1360 				 */
1361 				if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1362 				    (inp->inp_flags2 & INP_FREED)) {
1363 			out_free:
1364 					while (m) {
1365 						m_freem(m);
1366 						m = n;
1367 						if (m)
1368 							n = m->m_nextpkt;
1369 					}
1370 					CURVNET_RESTORE();
1371 					goto out;
1372 				}
1373 				/*
1374 				 * Now that we hold the INP lock, check if
1375 				 * we need to upgrade our lock.
1376 				 */
1377 				if (ti_locked == TI_UNLOCKED &&
1378 				    (tp->t_state != TCPS_ESTABLISHED)) {
1379 					ti_locked = TI_RLOCKED;
1380 					if (tcp_hptsi_lock_inpinfo(inp, &tp))
1381 						goto out_free;
1382 				}
1383 			}	/** end while(m) */
1384 		}		/** end if ((m != NULL)  && (m == tp->t_in_pkt)) */
1385 		if (in_pcbrele_wlocked(inp) == 0)
1386 			INP_WUNLOCK(inp);
1387 		if (ti_locked == TI_RLOCKED)
1388 			INP_INFO_RUNLOCK(&V_tcbinfo);
1389 		INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1390 		INP_UNLOCK_ASSERT(inp);
1391 		ti_locked = TI_UNLOCKED;
1392 		mtx_lock(&hpts->p_mtx);
1393 		hpts->p_inp = NULL;
1394 		CURVNET_RESTORE();
1395 	}
1396 }
1397 
1398 static int
1399 tcp_hpts_est_run(struct tcp_hpts_entry *hpts)
1400 {
1401 	int32_t ticks_to_run;
1402 
1403 	if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) {
1404 		ticks_to_run = hpts->p_curtick - hpts->p_prevtick;
1405 		if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) {
1406 			ticks_to_run = NUM_OF_HPTSI_SLOTS - 2;
1407 		}
1408 	} else {
1409 		if (hpts->p_prevtick == hpts->p_curtick) {
1410 			/* This happens when we get woken up right away */
1411 			return (-1);
1412 		}
1413 		ticks_to_run = 1;
1414 	}
1415 	/* Set in where we will be when we catch up */
1416 	hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS;
1417 	if (hpts->p_nxt_slot == hpts->p_cur_slot) {
1418 		panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d",
1419 		    hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run);
1420 	}
1421 	return (ticks_to_run);
1422 }
1423 
1424 static void
1425 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick)
1426 {
1427 	struct tcpcb *tp;
1428 	struct inpcb *inp = NULL, *ninp;
1429 	struct timeval tv;
1430 	int32_t ticks_to_run, i, error, tick_now, interum_tick;
1431 	int32_t paced_cnt = 0;
1432 	int32_t did_prefetch = 0;
1433 	int32_t prefetch_ninp = 0;
1434 	int32_t prefetch_tp = 0;
1435 	uint32_t cts;
1436 	int16_t set_cpu;
1437 
1438 	HPTS_MTX_ASSERT(hpts);
1439 	hpts->p_curtick = tcp_tv_to_hptstick(ctick);
1440 	cts = tcp_tv_to_usectick(ctick);
1441 	memcpy(&tv, ctick, sizeof(struct timeval));
1442 	hpts->p_cur_slot = hpts_tick(hpts, 1);
1443 
1444 	/* Figure out if we had missed ticks */
1445 again:
1446 	HPTS_MTX_ASSERT(hpts);
1447 	ticks_to_run = tcp_hpts_est_run(hpts);
1448 	if (!TAILQ_EMPTY(&hpts->p_input)) {
1449 		tcp_input_data(hpts, &tv);
1450 	}
1451 #ifdef INVARIANTS
1452 	if (TAILQ_EMPTY(&hpts->p_input) &&
1453 	    (hpts->p_on_inqueue_cnt != 0)) {
1454 		panic("tp:%p in_hpts input empty but cnt:%d",
1455 		    hpts, hpts->p_on_inqueue_cnt);
1456 	}
1457 #endif
1458 	HPTS_MTX_ASSERT(hpts);
1459 	/* Reset the ticks to run and time if we need too */
1460 	interum_tick = tcp_gethptstick(&tv);
1461 	if (interum_tick != hpts->p_curtick) {
1462 		/* Save off the new time we execute to */
1463 		*ctick = tv;
1464 		hpts->p_curtick = interum_tick;
1465 		cts = tcp_tv_to_usectick(&tv);
1466 		hpts->p_cur_slot = hpts_tick(hpts, 1);
1467 		ticks_to_run = tcp_hpts_est_run(hpts);
1468 	}
1469 	if (ticks_to_run == -1) {
1470 		goto no_run;
1471 	}
1472 	if (logging_on) {
1473 		tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0);
1474 	}
1475 	if (hpts->p_on_queue_cnt == 0) {
1476 		goto no_one;
1477 	}
1478 	HPTS_MTX_ASSERT(hpts);
1479 	for (i = 0; i < ticks_to_run; i++) {
1480 		/*
1481 		 * Calculate our delay, if there are no extra ticks there
1482 		 * was not any
1483 		 */
1484 		hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC;
1485 		HPTS_MTX_ASSERT(hpts);
1486 		while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1487 			/* For debugging */
1488 			if (logging_on) {
1489 				tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i);
1490 			}
1491 			hpts->p_inp = inp;
1492 			paced_cnt++;
1493 			if (hpts->p_cur_slot != inp->inp_hptsslot) {
1494 				panic("Hpts:%p inp:%p slot mis-aligned %u vs %u",
1495 				    hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot);
1496 			}
1497 			/* Now pull it */
1498 			if (inp->inp_hpts_cpu_set == 0) {
1499 				set_cpu = 1;
1500 			} else {
1501 				set_cpu = 0;
1502 			}
1503 			hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0);
1504 			if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1505 				/* We prefetch the next inp if possible */
1506 				kern_prefetch(ninp, &prefetch_ninp);
1507 				prefetch_ninp = 1;
1508 			}
1509 			if (inp->inp_hpts_request) {
1510 				/*
1511 				 * This guy is deferred out further in time
1512 				 * then our wheel had on it. Push him back
1513 				 * on the wheel.
1514 				 */
1515 				int32_t remaining_slots;
1516 
1517 				remaining_slots = ticks_to_run - (i + 1);
1518 				if (inp->inp_hpts_request > remaining_slots) {
1519 					/*
1520 					 * Keep INVARIANTS happy by clearing
1521 					 * the flag
1522 					 */
1523 					tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1);
1524 					hpts->p_inp = NULL;
1525 					continue;
1526 				}
1527 				inp->inp_hpts_request = 0;
1528 			}
1529 			/*
1530 			 * We clear the hpts flag here after dealing with
1531 			 * remaining slots. This way anyone looking with the
1532 			 * TCB lock will see its on the hpts until just
1533 			 * before we unlock.
1534 			 */
1535 			inp->inp_in_hpts = 0;
1536 			mtx_unlock(&hpts->p_mtx);
1537 			INP_WLOCK(inp);
1538 			if (in_pcbrele_wlocked(inp)) {
1539 				mtx_lock(&hpts->p_mtx);
1540 				if (logging_on)
1541 					tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1);
1542 				hpts->p_inp = NULL;
1543 				continue;
1544 			}
1545 			if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
1546 out_now:
1547 #ifdef INVARIANTS
1548 				if (mtx_owned(&hpts->p_mtx)) {
1549 					panic("Hpts:%p owns mtx prior-to lock line:%d",
1550 					    hpts, __LINE__);
1551 				}
1552 #endif
1553 				INP_WUNLOCK(inp);
1554 				mtx_lock(&hpts->p_mtx);
1555 				if (logging_on)
1556 					tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3);
1557 				hpts->p_inp = NULL;
1558 				continue;
1559 			}
1560 			tp = intotcpcb(inp);
1561 			if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1562 				goto out_now;
1563 			}
1564 			if (set_cpu) {
1565 				/*
1566 				 * Setup so the next time we will move to
1567 				 * the right CPU. This should be a rare
1568 				 * event. It will sometimes happens when we
1569 				 * are the client side (usually not the
1570 				 * server). Somehow tcp_output() gets called
1571 				 * before the tcp_do_segment() sets the
1572 				 * intial state. This means the r_cpu and
1573 				 * r_hpts_cpu is 0. We get on the hpts, and
1574 				 * then tcp_input() gets called setting up
1575 				 * the r_cpu to the correct value. The hpts
1576 				 * goes off and sees the mis-match. We
1577 				 * simply correct it here and the CPU will
1578 				 * switch to the new hpts nextime the tcb
1579 				 * gets added to the the hpts (not this one)
1580 				 * :-)
1581 				 */
1582 				tcp_set_hpts(inp);
1583 			}
1584 			if (out_newts_every_tcb) {
1585 				struct timeval sv;
1586 
1587 				if (out_ts_percision)
1588 					microuptime(&sv);
1589 				else
1590 					getmicrouptime(&sv);
1591 				cts = tcp_tv_to_usectick(&sv);
1592 			}
1593 			CURVNET_SET(tp->t_vnet);
1594 			/*
1595 			 * There is a hole here, we get the refcnt on the
1596 			 * inp so it will still be preserved but to make
1597 			 * sure we can get the INP we need to hold the p_mtx
1598 			 * above while we pull out the tp/inp,  as long as
1599 			 * fini gets the lock first we are assured of having
1600 			 * a sane INP we can lock and test.
1601 			 */
1602 #ifdef INVARIANTS
1603 			if (mtx_owned(&hpts->p_mtx)) {
1604 				panic("Hpts:%p owns mtx before tcp-output:%d",
1605 				    hpts, __LINE__);
1606 			}
1607 #endif
1608 			if (tp->t_fb_ptr != NULL) {
1609 				kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1610 				did_prefetch = 1;
1611 			}
1612 			inp->inp_hpts_calls = 1;
1613 			if (tp->t_fb->tfb_tcp_output_wtime != NULL) {
1614 				error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv);
1615 			} else {
1616 				error = tp->t_fb->tfb_tcp_output(tp);
1617 			}
1618 			if (ninp && ninp->inp_ppcb) {
1619 				/*
1620 				 * If we have a nxt inp, see if we can
1621 				 * prefetch its ppcb. Note this may seem
1622 				 * "risky" since we have no locks (other
1623 				 * than the previous inp) and there no
1624 				 * assurance that ninp was not pulled while
1625 				 * we were processing inp and freed. If this
1626 				 * occured it could mean that either:
1627 				 *
1628 				 * a) Its NULL (which is fine we won't go
1629 				 * here) <or> b) Its valid (which is cool we
1630 				 * will prefetch it) <or> c) The inp got
1631 				 * freed back to the slab which was
1632 				 * reallocated. Then the piece of memory was
1633 				 * re-used and something else (not an
1634 				 * address) is in inp_ppcb. If that occurs
1635 				 * we don't crash, but take a TLB shootdown
1636 				 * performance hit (same as if it was NULL
1637 				 * and we tried to pre-fetch it).
1638 				 *
1639 				 * Considering that the likelyhood of <c> is
1640 				 * quite rare we will take a risk on doing
1641 				 * this. If performance drops after testing
1642 				 * we can always take this out. NB: the
1643 				 * kern_prefetch on amd64 actually has
1644 				 * protection against a bad address now via
1645 				 * the DMAP_() tests. This will prevent the
1646 				 * TLB hit, and instead if <c> occurs just
1647 				 * cause us to load cache with a useless
1648 				 * address (to us).
1649 				 */
1650 				kern_prefetch(ninp->inp_ppcb, &prefetch_tp);
1651 				prefetch_tp = 1;
1652 			}
1653 			INP_WUNLOCK(inp);
1654 			INP_UNLOCK_ASSERT(inp);
1655 			CURVNET_RESTORE();
1656 #ifdef INVARIANTS
1657 			if (mtx_owned(&hpts->p_mtx)) {
1658 				panic("Hpts:%p owns mtx prior-to lock line:%d",
1659 				    hpts, __LINE__);
1660 			}
1661 #endif
1662 			mtx_lock(&hpts->p_mtx);
1663 			if (logging_on)
1664 				tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4);
1665 			hpts->p_inp = NULL;
1666 		}
1667 		HPTS_MTX_ASSERT(hpts);
1668 		hpts->p_inp = NULL;
1669 		hpts->p_cur_slot++;
1670 		if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) {
1671 			hpts->p_cur_slot = 0;
1672 		}
1673 	}
1674 no_one:
1675 	HPTS_MTX_ASSERT(hpts);
1676 	hpts->p_prevtick = hpts->p_curtick;
1677 	hpts->p_delayed_by = 0;
1678 	/*
1679 	 * Check to see if we took an excess amount of time and need to run
1680 	 * more ticks (if we did not hit eno-bufs).
1681 	 */
1682 	/* Re-run any input that may be there */
1683 	(void)tcp_gethptstick(&tv);
1684 	if (!TAILQ_EMPTY(&hpts->p_input)) {
1685 		tcp_input_data(hpts, &tv);
1686 	}
1687 #ifdef INVARIANTS
1688 	if (TAILQ_EMPTY(&hpts->p_input) &&
1689 	    (hpts->p_on_inqueue_cnt != 0)) {
1690 		panic("tp:%p in_hpts input empty but cnt:%d",
1691 		    hpts, hpts->p_on_inqueue_cnt);
1692 	}
1693 #endif
1694 	tick_now = tcp_gethptstick(&tv);
1695 	if (SEQ_GT(tick_now, hpts->p_prevtick)) {
1696 		struct timeval res;
1697 
1698 		/* Did we really spend a full tick or more in here? */
1699 		timersub(&tv, ctick, &res);
1700 		if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) {
1701 			counter_u64_add(hpts_loops, 1);
1702 			if (logging_on) {
1703 				tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now);
1704 			}
1705 			*ctick = res;
1706 			hpts->p_curtick = tick_now;
1707 			goto again;
1708 		}
1709 	}
1710 no_run:
1711 	{
1712 		uint32_t t = 0, i, fnd = 0;
1713 
1714 		if (hpts->p_on_queue_cnt) {
1715 
1716 
1717 			/*
1718 			 * Find next slot that is occupied and use that to
1719 			 * be the sleep time.
1720 			 */
1721 			for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) {
1722 				if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) {
1723 					fnd = 1;
1724 					break;
1725 				}
1726 				t = (t + 1) % NUM_OF_HPTSI_SLOTS;
1727 			}
1728 			if (fnd) {
1729 				hpts->p_hpts_sleep_time = i;
1730 			} else {
1731 				counter_u64_add(back_tosleep, 1);
1732 #ifdef INVARIANTS
1733 				panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt);
1734 #endif
1735 				hpts->p_on_queue_cnt = 0;
1736 				goto non_found;
1737 			}
1738 			t++;
1739 		} else {
1740 			/* No one on the wheel sleep for all but 2 slots  */
1741 non_found:
1742 			if (hpts_sleep_max == 0)
1743 				hpts_sleep_max = 1;
1744 			hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max);
1745 			t = 0;
1746 		}
1747 		if (logging_on) {
1748 			tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC));
1749 		}
1750 	}
1751 }
1752 
1753 void
1754 __tcp_set_hpts(struct inpcb *inp, int32_t line)
1755 {
1756 	struct tcp_hpts_entry *hpts;
1757 
1758 	INP_WLOCK_ASSERT(inp);
1759 	hpts = tcp_hpts_lock(inp);
1760 	if ((inp->inp_in_hpts == 0) &&
1761 	    (inp->inp_hpts_cpu_set == 0)) {
1762 		inp->inp_hpts_cpu = hpts_cpuid(inp);
1763 		inp->inp_hpts_cpu_set = 1;
1764 	}
1765 	mtx_unlock(&hpts->p_mtx);
1766 	hpts = tcp_input_lock(inp);
1767 	if ((inp->inp_input_cpu_set == 0) &&
1768 	    (inp->inp_in_input == 0)) {
1769 		inp->inp_input_cpu = hpts_cpuid(inp);
1770 		inp->inp_input_cpu_set = 1;
1771 	}
1772 	mtx_unlock(&hpts->p_mtx);
1773 }
1774 
1775 uint16_t
1776 tcp_hpts_delayedby(struct inpcb *inp){
1777 	return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by);
1778 }
1779 
1780 static void
1781 tcp_hpts_thread(void *ctx)
1782 {
1783 	struct tcp_hpts_entry *hpts;
1784 	struct timeval tv;
1785 	sbintime_t sb;
1786 
1787 	hpts = (struct tcp_hpts_entry *)ctx;
1788 	mtx_lock(&hpts->p_mtx);
1789 	if (hpts->p_direct_wake) {
1790 		/* Signaled by input */
1791 		if (logging_on)
1792 			tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1);
1793 		callout_stop(&hpts->co);
1794 	} else {
1795 		/* Timed out */
1796 		if (callout_pending(&hpts->co) ||
1797 		    !callout_active(&hpts->co)) {
1798 			if (logging_on)
1799 				tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2);
1800 			mtx_unlock(&hpts->p_mtx);
1801 			return;
1802 		}
1803 		callout_deactivate(&hpts->co);
1804 		if (logging_on)
1805 			tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3);
1806 	}
1807 	hpts->p_hpts_active = 1;
1808 	(void)tcp_gethptstick(&tv);
1809 	tcp_hptsi(hpts, &tv);
1810 	HPTS_MTX_ASSERT(hpts);
1811 	tv.tv_sec = 0;
1812 	tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1813 	if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) {
1814 		tv.tv_usec = tcp_min_hptsi_time;
1815 		hpts->p_on_min_sleep = 1;
1816 	} else {
1817 		/* Clear the min sleep flag */
1818 		hpts->p_on_min_sleep = 0;
1819 	}
1820 	hpts->p_hpts_active = 0;
1821 	sb = tvtosbt(tv);
1822 	if (tcp_hpts_callout_skip_swi == 0) {
1823 		callout_reset_sbt_on(&hpts->co, sb, 0,
1824 		    hpts_timeout_swi, hpts, hpts->p_cpu,
1825 		    (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1826 	} else {
1827 		callout_reset_sbt_on(&hpts->co, sb, 0,
1828 		    hpts_timeout_dir, hpts,
1829 		    hpts->p_cpu,
1830 		    C_PREL(tcp_hpts_precision));
1831 	}
1832 	hpts->p_direct_wake = 0;
1833 	mtx_unlock(&hpts->p_mtx);
1834 }
1835 
1836 #undef	timersub
1837 
1838 static void
1839 tcp_init_hptsi(void *st)
1840 {
1841 	int32_t i, j, error, bound = 0, created = 0;
1842 	size_t sz, asz;
1843 	struct timeval tv;
1844 	sbintime_t sb;
1845 	struct tcp_hpts_entry *hpts;
1846 	char unit[16];
1847 	uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1848 
1849 	tcp_pace.rp_proc = NULL;
1850 	tcp_pace.rp_num_hptss = ncpus;
1851 	hpts_loops = counter_u64_alloc(M_WAITOK);
1852 	back_tosleep = counter_u64_alloc(M_WAITOK);
1853 
1854 	sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *));
1855 	tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1856 	asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS;
1857 	for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
1858 		tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry),
1859 		    M_TCPHPTS, M_WAITOK | M_ZERO);
1860 		tcp_pace.rp_ent[i]->p_hptss = malloc(asz,
1861 		    M_TCPHPTS, M_WAITOK);
1862 		hpts = tcp_pace.rp_ent[i];
1863 		/*
1864 		 * Init all the hpts structures that are not specifically
1865 		 * zero'd by the allocations. Also lets attach them to the
1866 		 * appropriate sysctl block as well.
1867 		 */
1868 		mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
1869 		    "hpts", MTX_DEF | MTX_DUPOK);
1870 		TAILQ_INIT(&hpts->p_input);
1871 		for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) {
1872 			TAILQ_INIT(&hpts->p_hptss[j]);
1873 		}
1874 		sysctl_ctx_init(&hpts->hpts_ctx);
1875 		sprintf(unit, "%d", i);
1876 		hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx,
1877 		    SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts),
1878 		    OID_AUTO,
1879 		    unit,
1880 		    CTLFLAG_RW, 0,
1881 		    "");
1882 		SYSCTL_ADD_INT(&hpts->hpts_ctx,
1883 		    SYSCTL_CHILDREN(hpts->hpts_root),
1884 		    OID_AUTO, "in_qcnt", CTLFLAG_RD,
1885 		    &hpts->p_on_inqueue_cnt, 0,
1886 		    "Count TCB's awaiting input processing");
1887 		SYSCTL_ADD_INT(&hpts->hpts_ctx,
1888 		    SYSCTL_CHILDREN(hpts->hpts_root),
1889 		    OID_AUTO, "out_qcnt", CTLFLAG_RD,
1890 		    &hpts->p_on_queue_cnt, 0,
1891 		    "Count TCB's awaiting output processing");
1892 		SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1893 		    SYSCTL_CHILDREN(hpts->hpts_root),
1894 		    OID_AUTO, "active", CTLFLAG_RD,
1895 		    &hpts->p_hpts_active, 0,
1896 		    "Is the hpts active");
1897 		SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1898 		    SYSCTL_CHILDREN(hpts->hpts_root),
1899 		    OID_AUTO, "curslot", CTLFLAG_RD,
1900 		    &hpts->p_cur_slot, 0,
1901 		    "What the current slot is if active");
1902 		SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1903 		    SYSCTL_CHILDREN(hpts->hpts_root),
1904 		    OID_AUTO, "curtick", CTLFLAG_RD,
1905 		    &hpts->p_curtick, 0,
1906 		    "What the current tick on if active");
1907 		SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1908 		    SYSCTL_CHILDREN(hpts->hpts_root),
1909 		    OID_AUTO, "logsize", CTLFLAG_RD,
1910 		    &hpts->p_logsize, 0,
1911 		    "Hpts logging buffer size");
1912 		hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2;
1913 		hpts->p_num = i;
1914 		hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv);
1915 		hpts->p_prevtick -= 1;
1916 		hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS;
1917 		hpts->p_cpu = 0xffff;
1918 		hpts->p_nxt_slot = 1;
1919 		hpts->p_logsize = tcp_hpts_logging_size;
1920 		if (hpts->p_logsize) {
1921 			sz = (sizeof(struct hpts_log) * hpts->p_logsize);
1922 			hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1923 		}
1924 		callout_init(&hpts->co, 1);
1925 	}
1926 	/*
1927 	 * Now lets start ithreads to handle the hptss.
1928 	 */
1929 	CPU_FOREACH(i) {
1930 		hpts = tcp_pace.rp_ent[i];
1931 		hpts->p_cpu = i;
1932 		error = swi_add(&hpts->ie, "hpts",
1933 		    tcp_hpts_thread, (void *)hpts,
1934 		    SWI_NET, INTR_MPSAFE, &hpts->ie_cookie);
1935 		if (error) {
1936 			panic("Can't add hpts:%p i:%d err:%d",
1937 			    hpts, i, error);
1938 		}
1939 		created++;
1940 		if (tcp_bind_threads) {
1941 			if (intr_event_bind(hpts->ie, i) == 0)
1942 				bound++;
1943 		}
1944 		tv.tv_sec = 0;
1945 		tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1946 		sb = tvtosbt(tv);
1947 		if (tcp_hpts_callout_skip_swi == 0) {
1948 			callout_reset_sbt_on(&hpts->co, sb, 0,
1949 			    hpts_timeout_swi, hpts, hpts->p_cpu,
1950 			    (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1951 		} else {
1952 			callout_reset_sbt_on(&hpts->co, sb, 0,
1953 			    hpts_timeout_dir, hpts,
1954 			    hpts->p_cpu,
1955 			    C_PREL(tcp_hpts_precision));
1956 		}
1957 	}
1958 	printf("TCP Hpts created %d swi interrupt thread and bound %d\n",
1959 	    created, bound);
1960 	return;
1961 }
1962 
1963 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL);
1964