1 /*- 2 * Copyright (c) 2016-2018 Netflix, Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #ifndef __tcp_hpts_h__ 27 #define __tcp_hpts_h__ 28 29 /* Number of useconds represented by an hpts slot */ 30 #define HPTS_USECS_PER_SLOT 10 31 #define HPTS_MS_TO_SLOTS(x) ((x * 100) + 1) 32 #define HPTS_USEC_TO_SLOTS(x) ((x+9) /10) 33 #define HPTS_USEC_IN_SEC 1000000 34 #define HPTS_MSEC_IN_SEC 1000 35 #define HPTS_USEC_IN_MSEC 1000 36 37 static inline uint32_t 38 tcp_tv_to_hpts_slot(const struct timeval *sv) 39 { 40 return ((sv->tv_sec * 100000) + (sv->tv_usec / HPTS_USECS_PER_SLOT)); 41 } 42 43 static inline uint32_t 44 tcp_tv_to_usec(const struct timeval *sv) 45 { 46 return ((uint32_t) ((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); 47 } 48 49 static inline uint32_t 50 tcp_tv_to_msec(const struct timeval *sv) 51 { 52 return ((uint32_t) ((sv->tv_sec * HPTS_MSEC_IN_SEC) + (sv->tv_usec/HPTS_USEC_IN_MSEC))); 53 } 54 55 static inline uint64_t 56 tcp_tv_to_lusec(const struct timeval *sv) 57 { 58 return ((uint64_t)((sv->tv_sec * HPTS_USEC_IN_SEC) + sv->tv_usec)); 59 } 60 61 struct hpts_diag { 62 uint32_t p_hpts_active; /* bbr->flex7 x */ 63 uint32_t p_nxt_slot; /* bbr->flex1 x */ 64 uint32_t p_cur_slot; /* bbr->flex2 x */ 65 uint32_t p_prev_slot; /* bbr->delivered */ 66 uint32_t p_runningslot; /* bbr->inflight */ 67 uint32_t slot_req; /* bbr->flex3 x */ 68 uint32_t inp_hptsslot; /* bbr->flex4 x */ 69 uint32_t slot_remaining; /* bbr->flex5 x */ 70 uint32_t have_slept; /* bbr->epoch x */ 71 uint32_t hpts_sleep_time; /* bbr->applimited x */ 72 uint32_t yet_to_sleep; /* bbr->lt_epoch x */ 73 uint32_t need_new_to; /* bbr->flex6 x */ 74 uint32_t wheel_slot; /* bbr->bw_inuse x */ 75 uint32_t maxslots; /* bbr->delRate x */ 76 uint32_t wheel_cts; /* bbr->rttProp x */ 77 int32_t co_ret; /* bbr->pkts_out x */ 78 uint32_t p_curtick; /* upper bbr->cur_del_rate */ 79 uint32_t p_lasttick; /* lower bbr->cur_del_rate */ 80 uint8_t p_on_min_sleep; /* bbr->flex8 x */ 81 }; 82 83 /* Magic flags to tell whats cooking on the pacing wheel */ 84 #define PACE_TMR_DELACK 0x01 /* Delayed ack timer running */ 85 #define PACE_TMR_RACK 0x02 /* RACK timer running */ 86 #define PACE_TMR_TLP 0x04 /* TLP timer running */ 87 #define PACE_TMR_RXT 0x08 /* Retransmit timer running */ 88 #define PACE_TMR_PERSIT 0x10 /* Persists timer running */ 89 #define PACE_TMR_KEEP 0x20 /* Keep alive timer running */ 90 #define PACE_PKT_OUTPUT 0x40 /* Output Packets being paced */ 91 #define PACE_TMR_MASK (PACE_TMR_KEEP|PACE_TMR_PERSIT|PACE_TMR_RXT|PACE_TMR_TLP|PACE_TMR_RACK|PACE_TMR_DELACK) 92 93 #ifdef _KERNEL 94 95 /* 96 * The following are the definitions for the kernel HPTS interface for managing 97 * the HPTS ring and the TCBs on it. 98 */ 99 100 void tcp_hpts_init(struct tcpcb *); 101 void tcp_hpts_remove(struct tcpcb *); 102 103 static inline bool 104 tcp_in_hpts(struct tcpcb *tp) 105 { 106 return ((tp->t_in_hpts == IHPTS_ONQUEUE) || 107 ((tp->t_in_hpts == IHPTS_MOVING) && 108 (tp->t_hpts_slot != -1))); 109 } 110 111 /* 112 * To insert a TCB on the hpts you *must* be holding the 113 * INP_WLOCK(). The hpts insert code will then acqurire 114 * the hpts's lock and insert the TCB on the requested 115 * slot possibly waking up the hpts if you are requesting 116 * a time earlier than what the hpts is sleeping to (if 117 * the hpts is sleeping). You may check the inp->inp_in_hpts 118 * flag without the hpts lock. The hpts is the only one 119 * that will clear this flag holding only the hpts lock. This 120 * means that in your tcp_output() routine when you test for 121 * it to be 1 (so you wont call output) it may be transitioning 122 * to 0 (by the hpts). That will be fine since that will just 123 * mean an extra call to tcp_output that most likely will find 124 * the call you executed (when the mis-match occurred) will have 125 * put the TCB back on the hpts and it will return. If your 126 * call did not add it back to the hpts then you will either 127 * over-send or the cwnd will block you from sending more. 128 * 129 * Note you should also be holding the INP_WLOCK() when you 130 * call the remove from the hpts as well. Thoug usually 131 * you are either doing this from a timer, where you need 132 * that INP_WLOCK() or from destroying your TCB where again 133 * you should already have the INP_WLOCK(). 134 */ 135 uint32_t tcp_hpts_insert_diag(struct tcpcb *tp, uint32_t slot, int32_t line, 136 struct hpts_diag *diag); 137 #define tcp_hpts_insert(inp, slot) \ 138 tcp_hpts_insert_diag((inp), (slot), __LINE__, NULL) 139 140 void tcp_set_hpts(struct tcpcb *tp); 141 142 extern int32_t tcp_min_hptsi_time; 143 144 static inline int32_t 145 get_hpts_min_sleep_time(void) 146 { 147 return (tcp_min_hptsi_time + HPTS_USECS_PER_SLOT); 148 } 149 150 static inline uint32_t 151 tcp_gethptstick(struct timeval *sv) 152 { 153 struct timeval tv; 154 155 if (sv == NULL) 156 sv = &tv; 157 microuptime(sv); 158 return (tcp_tv_to_hpts_slot(sv)); 159 } 160 161 static inline uint64_t 162 tcp_get_u64_usecs(struct timeval *tv) 163 { 164 struct timeval tvd; 165 166 if (tv == NULL) 167 tv = &tvd; 168 microuptime(tv); 169 return (tcp_tv_to_lusec(tv)); 170 } 171 172 static inline uint32_t 173 tcp_get_usecs(struct timeval *tv) 174 { 175 struct timeval tvd; 176 177 if (tv == NULL) 178 tv = &tvd; 179 microuptime(tv); 180 return (tcp_tv_to_usec(tv)); 181 } 182 183 /* 184 * LRO HPTS initialization and uninitialization, only for internal use by the 185 * HPTS code. 186 */ 187 void tcp_lro_hpts_init(void); 188 void tcp_lro_hpts_uninit(void); 189 190 #endif /* _KERNEL */ 191 #endif /* __tcp_hpts_h__ */ 192