Lines Matching +full:pre +full:- +full:configurable
1 /*-
3 * SPDX-License-Identifier: BSD-3-Clause
5 * Copyright (c) 2018-2020
73 * Why do the rates cluster in the 1-100Mbps range more
81 * Chelsio - Supporting 16 configurable rates.
82 * Mlx - c4 supporting 13 fixed rates.
83 * Mlx - c5 & c6 supporting 127 configurable rates.
113 * by the hardware will cause a 390 micro-second gap between
115 * would need 416 micro-seconds gap between each send.
126 * delta between the two rates (416 - 390) divided into the rate
129 * run out of data in a single 17MSS send in 6,630 micro-seconds.
132 * in 7,072 micro-seconds. This means that we will refill
151 122500, /* 1Mbps - rate 1 */
152 180500, /* 1.44Mpbs - rate 2 common rate */
153 375000, /* 3Mbps - rate 3 */
154 625000, /* 5Mbps - rate 4 */
155 1250000, /* 10Mbps - rate 5 */
156 1875000, /* 15Mbps - rate 6 */
157 2500000, /* 20Mbps - rate 7 */
158 3125000, /* 25Mbps - rate 8 */
159 3750000, /* 30Mbps - rate 9 */
160 4375000, /* 35Mbps - rate 10 */
161 5000000, /* 40Meg - rate 11 */
162 6250000, /* 50Mbps - rate 12 */
163 12500000, /* 100Mbps - rate 13 */
164 25000000, /* 200Mbps - rate 14 */
165 50000000, /* 400Mbps - rate 15 */
166 100000000, /* 800Mbps - rate 16 */
167 5625000, /* 45Mbps - rate 17 */
168 6875000, /* 55Mbps - rate 19 */
169 7500000, /* 60Mbps - rate 20 */
170 8125000, /* 65Mbps - rate 21 */
171 8750000, /* 70Mbps - rate 22 */
172 9375000, /* 75Mbps - rate 23 */
173 10000000, /* 80Mbps - rate 24 */
174 10625000, /* 85Mbps - rate 25 */
175 11250000, /* 90Mbps - rate 26 */
176 11875000, /* 95Mbps - rate 27 */
177 12500000, /* 100Mbps - rate 28 */
178 13750000, /* 110Mbps - rate 29 */
179 15000000, /* 120Mbps - rate 30 */
180 16250000, /* 130Mbps - rate 31 */
181 17500000, /* 140Mbps - rate 32 */
182 18750000, /* 150Mbps - rate 33 */
183 20000000, /* 160Mbps - rate 34 */
184 21250000, /* 170Mbps - rate 35 */
185 22500000, /* 180Mbps - rate 36 */
186 23750000, /* 190Mbps - rate 37 */
187 26250000, /* 210Mbps - rate 38 */
188 27500000, /* 220Mbps - rate 39 */
189 28750000, /* 230Mbps - rate 40 */
190 30000000, /* 240Mbps - rate 41 */
191 31250000, /* 250Mbps - rate 42 */
192 34375000, /* 275Mbps - rate 43 */
193 37500000, /* 300Mbps - rate 44 */
194 40625000, /* 325Mbps - rate 45 */
195 43750000, /* 350Mbps - rate 46 */
196 46875000, /* 375Mbps - rate 47 */
197 53125000, /* 425Mbps - rate 48 */
198 56250000, /* 450Mbps - rate 49 */
199 59375000, /* 475Mbps - rate 50 */
200 62500000, /* 500Mbps - rate 51 */
201 68750000, /* 550Mbps - rate 52 */
202 75000000, /* 600Mbps - rate 53 */
203 81250000, /* 650Mbps - rate 54 */
204 87500000, /* 700Mbps - rate 55 */
205 93750000, /* 750Mbps - rate 56 */
206 106250000, /* 850Mbps - rate 57 */
207 112500000, /* 900Mbps - rate 58 */
208 125000000, /* 1Gbps - rate 59 */
209 156250000, /* 1.25Gps - rate 60 */
210 187500000, /* 1.5Gps - rate 61 */
211 218750000, /* 1.75Gps - rate 62 */
212 250000000, /* 2Gbps - rate 63 */
213 281250000, /* 2.25Gps - rate 64 */
214 312500000, /* 2.5Gbps - rate 65 */
215 343750000, /* 2.75Gbps - rate 66 */
216 375000000, /* 3Gbps - rate 67 */
217 500000000, /* 4Gbps - rate 68 */
218 625000000, /* 5Gbps - rate 69 */
219 750000000, /* 6Gbps - rate 70 */
220 875000000, /* 7Gbps - rate 71 */
221 1000000000, /* 8Gbps - rate 72 */
222 1125000000, /* 9Gbps - rate 73 */
223 1250000000, /* 10Gbps - rate 74 */
224 1875000000, /* 15Gbps - rate 75 */
225 2500000000 /* 20Gbps - rate 76 */
237 * group (index wise means -1).
240 * 1Meg - 1Gig in 1 Meg steps
300 if (rs->rs_flags & RS_INTF_NO_SUP) { in rl_add_syctl_entries()
301 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
304 &rs->rs_disable, 0, in rl_add_syctl_entries()
307 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
310 &rs->rs_disable, 0, in rl_add_syctl_entries()
313 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
316 &rs->rs_min_seg, 0, in rl_add_syctl_entries()
318 SYSCTL_ADD_U64(&rs->sysctl_ctx, in rl_add_syctl_entries()
321 &rs->rs_flow_limit, 0, in rl_add_syctl_entries()
323 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
326 &rs->rs_highest_valid, 0, in rl_add_syctl_entries()
328 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
331 &rs->rs_lowest_valid, 0, in rl_add_syctl_entries()
333 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
336 &rs->rs_flags, 0, in rl_add_syctl_entries()
338 SYSCTL_ADD_S32(&rs->sysctl_ctx, in rl_add_syctl_entries()
341 &rs->rs_rate_cnt, 0, in rl_add_syctl_entries()
343 SYSCTL_ADD_U64(&rs->sysctl_ctx, in rl_add_syctl_entries()
346 &rs->rs_flows_using, 0, in rl_add_syctl_entries()
349 if (rs->rs_rlt && rs->rs_rate_cnt > 0) { in rl_add_syctl_entries()
355 rl_rates = SYSCTL_ADD_NODE(&rs->sysctl_ctx, in rl_add_syctl_entries()
361 for( i = 0; i < rs->rs_rate_cnt; i++) { in rl_add_syctl_entries()
363 rl_rate_num = SYSCTL_ADD_NODE(&rs->sysctl_ctx, in rl_add_syctl_entries()
369 SYSCTL_ADD_U32(&rs->sysctl_ctx, in rl_add_syctl_entries()
372 &rs->rs_rlt[i].flags, 0, in rl_add_syctl_entries()
374 SYSCTL_ADD_U32(&rs->sysctl_ctx, in rl_add_syctl_entries()
377 &rs->rs_rlt[i].time_between, 0, in rl_add_syctl_entries()
379 SYSCTL_ADD_LONG(&rs->sysctl_ctx, in rl_add_syctl_entries()
382 &rs->rs_rlt[i].rate, in rl_add_syctl_entries()
384 SYSCTL_ADD_LONG(&rs->sysctl_ctx, in rl_add_syctl_entries()
387 &rs->rs_rlt[i].using, in rl_add_syctl_entries()
389 SYSCTL_ADD_LONG(&rs->sysctl_ctx, in rl_add_syctl_entries()
392 &rs->rs_rlt[i].rs_num_enobufs, in rl_add_syctl_entries()
409 rs->rs_flags &= ~RS_FUNERAL_SCHD; in rs_destroy()
426 do_free_rs = (rs->rs_flows_using == 0); in rs_destroy()
427 rs_number_dead--; in rs_destroy()
431 sysctl_ctx_free(&rs->sysctl_ctx); in rs_destroy()
432 free(rs->rs_rlt, M_TCPPACE); in rs_destroy()
444 if (rs->rs_flags & RS_FUNERAL_SCHD) in rs_defer_destroy()
450 rs->rs_flags |= RS_FUNERAL_SCHD; in rs_defer_destroy()
451 NET_EPOCH_CALL(rs_destroy, &rs->rs_epoch_ctx); in rs_defer_destroy()
503 for(i = 0, at_low = 0, at_high = RS_NEXT_ORDER_GROUP; i < rs->rs_rate_cnt; i++) { in populate_canned_table()
504 rs->rs_rlt[i].flags = 0; in populate_canned_table()
505 rs->rs_rlt[i].time_between = 0; in populate_canned_table()
509 rs->rs_rlt[i].rate = rate_table_act[at_low]; in populate_canned_table()
514 rs->rs_rlt[i].rate = rate_table_act[at_high]; in populate_canned_table()
539 if (ifp->if_ratelimit_query == NULL) { in rt_setup_new_rs()
544 printf("Warning:No query functions for %s:%d-- failed\n", in rt_setup_new_rs()
545 ifp->if_dname, ifp->if_dunit); in rt_setup_new_rs()
557 ifp->if_ratelimit_query(ifp, &rl); in rt_setup_new_rs()
561 * the rate-limiting. in rt_setup_new_rs()
564 rs->rs_ifp = ifp; in rt_setup_new_rs()
565 rs->rs_if_dunit = ifp->if_dunit; in rt_setup_new_rs()
566 rs->rs_flags = RS_INTF_NO_SUP; in rt_setup_new_rs()
567 rs->rs_disable = 1; in rt_setup_new_rs()
569 sysctl_ctx_init(&rs->sysctl_ctx); in rt_setup_new_rs()
570 rl_sysctl_root = SYSCTL_ADD_NODE(&rs->sysctl_ctx, in rt_setup_new_rs()
573 rs->rs_ifp->if_xname, in rt_setup_new_rs()
585 rs->rs_ifp = ifp; in rt_setup_new_rs()
586 rs->rs_if_dunit = ifp->if_dunit; in rt_setup_new_rs()
587 rs->rs_flags = RS_IS_DEFF; in rt_setup_new_rs()
589 sysctl_ctx_init(&rs->sysctl_ctx); in rt_setup_new_rs()
590 rl_sysctl_root = SYSCTL_ADD_NODE(&rs->sysctl_ctx, in rt_setup_new_rs()
593 rs->rs_ifp->if_xname, in rt_setup_new_rs()
605 rs->rs_ifp = ifp; in rt_setup_new_rs()
606 rs->rs_if_dunit = ifp->if_dunit; in rt_setup_new_rs()
607 rs->rs_rate_cnt = rl.number_of_rates; in rt_setup_new_rs()
608 rs->rs_min_seg = rl.min_segment_burst; in rt_setup_new_rs()
609 rs->rs_highest_valid = 0; in rt_setup_new_rs()
610 rs->rs_flow_limit = rl.max_flows; in rt_setup_new_rs()
611 rs->rs_flags = RS_IS_INTF | RS_NO_PRE; in rt_setup_new_rs()
612 rs->rs_disable = 0; in rt_setup_new_rs()
616 rs->rs_ifp = ifp; in rt_setup_new_rs()
617 rs->rs_if_dunit = ifp->if_dunit; in rt_setup_new_rs()
618 rs->rs_rate_cnt = rl.number_of_rates; in rt_setup_new_rs()
619 rs->rs_min_seg = rl.min_segment_burst; in rt_setup_new_rs()
620 rs->rs_disable = 0; in rt_setup_new_rs()
621 rs->rs_flow_limit = rl.max_flows; in rt_setup_new_rs()
623 if ((rs->rs_rate_cnt > MAX_HDWR_RATES) && in rt_setup_new_rs()
624 (rs->rs_rate_cnt < ALL_HARDWARE_RATES)) { in rt_setup_new_rs()
629 rs->rs_rate_cnt = MAX_HDWR_RATES; in rt_setup_new_rs()
631 if (rs->rs_rate_cnt <= RS_ORDERED_COUNT) in rt_setup_new_rs()
632 rs->rs_flags = RS_IS_INTF; in rt_setup_new_rs()
634 rs->rs_flags = RS_IS_INTF | RS_INT_TBL; in rt_setup_new_rs()
635 if (rs->rs_rate_cnt >= ALL_HARDWARE_RATES) in rt_setup_new_rs()
636 rs->rs_rate_cnt = ALL_HARDWARE_RATES; in rt_setup_new_rs()
641 sz = sizeof(struct tcp_hwrate_limit_table) * rs->rs_rate_cnt; in rt_setup_new_rs()
642 rs->rs_rlt = malloc(sz, M_TCPPACE, M_NOWAIT); in rt_setup_new_rs()
643 if (rs->rs_rlt == NULL) { in rt_setup_new_rs()
650 if (rs->rs_rate_cnt >= ALL_HARDWARE_RATES) { in rt_setup_new_rs()
657 rs->rs_rlt[0].rate = 12500; /* 100k */ in rt_setup_new_rs()
658 rs->rs_rlt[1].rate = 25000; /* 200k */ in rt_setup_new_rs()
659 rs->rs_rlt[2].rate = 62500; /* 500k */ in rt_setup_new_rs()
661 * populate 1Meg - 1000meg. in rt_setup_new_rs()
663 for(i = 3, rat = 125000; i< (ALL_HARDWARE_RATES-1); i++) { in rt_setup_new_rs()
664 rs->rs_rlt[i].rate = rat; in rt_setup_new_rs()
667 rs->rs_rlt[(ALL_HARDWARE_RATES-1)].rate = 1250000000; in rt_setup_new_rs()
668 } else if (rs->rs_flags & RS_INT_TBL) { in rt_setup_new_rs()
676 for (i=0; i<rs->rs_rate_cnt; i++) { in rt_setup_new_rs()
677 rs->rs_rlt[i].rate = rate_table_act[i]; in rt_setup_new_rs()
678 rs->rs_rlt[i].time_between = 0; in rt_setup_new_rs()
679 rs->rs_rlt[i].flags = 0; in rt_setup_new_rs()
682 for (i = (rs->rs_rate_cnt - 1); i >= 0; i--) { in rt_setup_new_rs()
688 rs->rs_rlt[i].ptbl = rs; in rt_setup_new_rs()
689 rs->rs_rlt[i].tag = NULL; in rt_setup_new_rs()
690 rs->rs_rlt[i].using = 0; in rt_setup_new_rs()
691 rs->rs_rlt[i].rs_num_enobufs = 0; in rt_setup_new_rs()
696 res = lentim / rs->rs_rlt[i].rate; in rt_setup_new_rs()
698 rs->rs_rlt[i].time_between = res; in rt_setup_new_rs()
700 rs->rs_rlt[i].time_between = 1; in rt_setup_new_rs()
701 if (rs->rs_flags & RS_NO_PRE) { in rt_setup_new_rs()
702 rs->rs_rlt[i].flags = HDWRPACE_INITED; in rt_setup_new_rs()
703 rs->rs_lowest_valid = i; in rt_setup_new_rs()
708 (ifp->if_ratelimit_query)) { in rt_setup_new_rs()
709 err = ifp->if_ratelimit_setup(ifp, in rt_setup_new_rs()
710 rs->rs_rlt[i].rate, i); in rt_setup_new_rs()
722 rs->rs_rlt[i].rate, in rt_setup_new_rs()
723 &rs->rs_rlt[i].tag); in rt_setup_new_rs()
726 if (i == (rs->rs_rate_cnt - 1)) { in rt_setup_new_rs()
728 * Huh - first rate and we can't get in rt_setup_new_rs()
731 free(rs->rs_rlt, M_TCPPACE); in rt_setup_new_rs()
741 rs->rs_rlt[i].flags = HDWRPACE_INITED | HDWRPACE_TAGPRESENT; in rt_setup_new_rs()
742 rs->rs_lowest_valid = i; in rt_setup_new_rs()
747 if (rs->rs_rlt[(rs->rs_rate_cnt - 1)].flags & HDWRPACE_INITED) in rt_setup_new_rs()
748 rs->rs_highest_valid = rs->rs_rate_cnt - 1; in rt_setup_new_rs()
750 free(rs->rs_rlt, M_TCPPACE); in rt_setup_new_rs()
754 sysctl_ctx_init(&rs->sysctl_ctx); in rt_setup_new_rs()
755 rl_sysctl_root = SYSCTL_ADD_NODE(&rs->sysctl_ctx, in rt_setup_new_rs()
758 rs->rs_ifp->if_xname, in rt_setup_new_rs()
785 (rs->rs_lowest_valid <= 2)){ in tcp_int_find_suitable_rate()
791 for(i = rs->rs_lowest_valid; i < 3; i++) { in tcp_int_find_suitable_rate()
792 if (bytes_per_sec <= rs->rs_rlt[i].rate) { in tcp_int_find_suitable_rate()
793 rte = &rs->rs_rlt[i]; in tcp_int_find_suitable_rate()
795 } else if (rs->rs_rlt[i].flags & HDWRPACE_INITED) { in tcp_int_find_suitable_rate()
796 arte = &rs->rs_rlt[i]; in tcp_int_find_suitable_rate()
798 previous_rate = rs->rs_rlt[i].rate; in tcp_int_find_suitable_rate()
802 (rs->rs_rlt[(ALL_HARDWARE_RATES-1)].flags & HDWRPACE_INITED)){ in tcp_int_find_suitable_rate()
808 rte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
810 arte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
811 previous_rate = rs->rs_rlt[(ALL_HARDWARE_RATES-2)].rate; in tcp_int_find_suitable_rate()
815 * If we reach here its in our table (between 1Meg - 1000Meg), in tcp_int_find_suitable_rate()
825 if (ind_calc > (ALL_HARDWARE_RATES-1)) { in tcp_int_find_suitable_rate()
827 ind_calc = ALL_HARDWARE_RATES-1; in tcp_int_find_suitable_rate()
829 if ((ind_calc >= rs->rs_lowest_valid) && in tcp_int_find_suitable_rate()
830 (ind_calc <= rs->rs_highest_valid)) { in tcp_int_find_suitable_rate()
831 rte = &rs->rs_rlt[ind_calc]; in tcp_int_find_suitable_rate()
833 previous_rate = rs->rs_rlt[(ind_calc-1)].rate; in tcp_int_find_suitable_rate()
837 (rs->rs_lowest_valid <= 2)){ in tcp_int_find_suitable_rate()
838 for(i = rs->rs_lowest_valid; i < 3; i++) { in tcp_int_find_suitable_rate()
839 if (bytes_per_sec == rs->rs_rlt[i].rate) { in tcp_int_find_suitable_rate()
840 rte = &rs->rs_rlt[i]; in tcp_int_find_suitable_rate()
845 (rs->rs_rlt[(ALL_HARDWARE_RATES-1)].flags & HDWRPACE_INITED)) { in tcp_int_find_suitable_rate()
847 if (bytes_per_sec == rs->rs_rlt[(ALL_HARDWARE_RATES-1)].rate) { in tcp_int_find_suitable_rate()
849 rte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
857 if (ind_calc > (ALL_HARDWARE_RATES-1)) { in tcp_int_find_suitable_rate()
859 ind_calc = ALL_HARDWARE_RATES-1; in tcp_int_find_suitable_rate()
861 if (rs->rs_rlt[ind_calc].flags & HDWRPACE_INITED) in tcp_int_find_suitable_rate()
862 rte = &rs->rs_rlt[ind_calc]; in tcp_int_find_suitable_rate()
868 (rs->rs_lowest_valid <= 2)){ in tcp_int_find_suitable_rate()
869 arte = &rs->rs_rlt[3]; /* set alternate to 1Meg */ in tcp_int_find_suitable_rate()
870 for (i=2; i>=rs->rs_lowest_valid; i--) { in tcp_int_find_suitable_rate()
871 if (bytes_per_sec < rs->rs_rlt[i].rate) { in tcp_int_find_suitable_rate()
872 rte = &rs->rs_rlt[i]; in tcp_int_find_suitable_rate()
874 previous_rate = rs->rs_rlt[(i-1)].rate; in tcp_int_find_suitable_rate()
878 (bytes_per_sec == rs->rs_rlt[i].rate)) { in tcp_int_find_suitable_rate()
879 rte = &rs->rs_rlt[i]; in tcp_int_find_suitable_rate()
881 previous_rate = rs->rs_rlt[(i-1)].rate; in tcp_int_find_suitable_rate()
885 arte = &rs->rs_rlt[i]; /* new alternate */ in tcp_int_find_suitable_rate()
889 if ((bytes_per_sec < rs->rs_rlt[(ALL_HARDWARE_RATES-1)].rate) && in tcp_int_find_suitable_rate()
890 (rs->rs_rlt[(ALL_HARDWARE_RATES-1)].flags & HDWRPACE_INITED)){ in tcp_int_find_suitable_rate()
892 rte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
894 (bytes_per_sec == rs->rs_rlt[(ALL_HARDWARE_RATES-1)].rate) && in tcp_int_find_suitable_rate()
895 (rs->rs_rlt[(ALL_HARDWARE_RATES-1)].flags & HDWRPACE_INITED)) { in tcp_int_find_suitable_rate()
897 rte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
898 } else if (rs->rs_rlt[(ALL_HARDWARE_RATES-1)].flags & HDWRPACE_INITED) { in tcp_int_find_suitable_rate()
900 arte = &rs->rs_rlt[(ALL_HARDWARE_RATES-1)]; in tcp_int_find_suitable_rate()
902 previous_rate = rs->rs_rlt[(ALL_HARDWARE_RATES-2)].rate; in tcp_int_find_suitable_rate()
904 /* Its in our range 1Meg - 1Gig */ in tcp_int_find_suitable_rate()
908 if (ind_calc > (ALL_HARDWARE_RATES-1)) { in tcp_int_find_suitable_rate()
910 ind_calc = (ALL_HARDWARE_RATES-1); in tcp_int_find_suitable_rate()
912 rte = &rs->rs_rlt[ind_calc]; in tcp_int_find_suitable_rate()
914 previous_rate = rs->rs_rlt[(ind_calc-1)].rate; in tcp_int_find_suitable_rate()
918 ind_calc = (mbits_per_sec + (RS_ONE_MEGABIT_PERSEC-1))/RS_ONE_MEGABIT_PERSEC; in tcp_int_find_suitable_rate()
920 if (ind_calc > (ALL_HARDWARE_RATES-1)) { in tcp_int_find_suitable_rate()
922 ind_calc = ALL_HARDWARE_RATES-1; in tcp_int_find_suitable_rate()
924 if (rs->rs_rlt[ind_calc].flags & HDWRPACE_INITED) { in tcp_int_find_suitable_rate()
925 rte = &rs->rs_rlt[ind_calc]; in tcp_int_find_suitable_rate()
927 previous_rate = rs->rs_rlt[(ind_calc-1)].rate; in tcp_int_find_suitable_rate()
953 * RS_PACING_EXACT_MATCH - look for an exact match to rate. in tcp_find_suitable_rate()
954 * RS_PACING_GT - must be greater than. in tcp_find_suitable_rate()
955 * RS_PACING_GEQ - must be greater than or equal. in tcp_find_suitable_rate()
956 * RS_PACING_LT - must be less than. in tcp_find_suitable_rate()
957 * RS_PACING_SUB_OK - If we don't meet criteria a in tcp_find_suitable_rate()
964 if ((rs->rs_flags & RS_INT_TBL) && in tcp_find_suitable_rate()
965 (rs->rs_rate_cnt >= ALL_HARDWARE_RATES)) { in tcp_find_suitable_rate()
969 * from 1Meg - 1000Meg in 1Meg increments. in tcp_find_suitable_rate()
981 for (i = rs->rs_lowest_valid, matched = 0; i <= rs->rs_highest_valid; i++) { in tcp_find_suitable_rate()
983 (bytes_per_sec == rs->rs_rlt[i].rate)) { in tcp_find_suitable_rate()
984 rte = &rs->rs_rlt[i]; in tcp_find_suitable_rate()
990 (bytes_per_sec <= rs->rs_rlt[i].rate)) { in tcp_find_suitable_rate()
991 rte = &rs->rs_rlt[i]; in tcp_find_suitable_rate()
997 previous_rate = rs->rs_rlt[i].rate; in tcp_find_suitable_rate()
998 if (bytes_per_sec > rs->rs_rlt[i].rate) in tcp_find_suitable_rate()
1005 rte = &rs->rs_rlt[rs->rs_lowest_valid]; in tcp_find_suitable_rate()
1013 for (i = rs->rs_highest_valid, matched = 0; i >= rs->rs_lowest_valid; i--) { in tcp_find_suitable_rate()
1014 if (rs->rs_rlt[i].rate > bytes_per_sec) { in tcp_find_suitable_rate()
1016 rte = &rs->rs_rlt[i]; in tcp_find_suitable_rate()
1019 (bytes_per_sec == rs->rs_rlt[i].rate)) { in tcp_find_suitable_rate()
1022 rte = &rs->rs_rlt[i]; in tcp_find_suitable_rate()
1031 if (rs->rs_rlt[i].rate < bytes_per_sec) { in tcp_find_suitable_rate()
1037 *lower_rate = rs->rs_rlt[i].rate; in tcp_find_suitable_rate()
1044 rte = &rs->rs_rlt[rs->rs_highest_valid]; in tcp_find_suitable_rate()
1057 .rate_limit.hdr.flowid = inp->inp_flowid, in rt_find_real_interface()
1058 .rate_limit.hdr.numa_domain = inp->inp_numa_domain, in rt_find_real_interface()
1064 params.rate_limit.hdr.flowtype = ((inp->inp_vflag & INP_IPV6) ? in rt_find_real_interface()
1077 while (ntag->sw->next_snd_tag != NULL) { in rt_find_real_interface()
1078 ntag = ntag->sw->next_snd_tag(ntag); in rt_find_real_interface()
1080 tifp = ntag->ifp; in rt_find_real_interface()
1091 atomic_add_long(&decon_rte->using, 1); in rl_increment_using()
1100 atomic_subtract_long(&decon_rte->using, 1); in rl_decrement_using()
1109 atomic_add_long(&decon_rte->rs_num_enobufs, 1); in tcp_rl_log_enobuf()
1123 * as we get a compiler fix 10/2/20 -- RRS
1131 if ((rs->rs_ifp == ifp) && in find_rs_for_ifp()
1132 (rs->rs_if_dunit == ifp->if_dunit)) { in find_rs_for_ifp()
1163 (rs->rs_flags & RS_INTF_NO_SUP) || in rt_setup_rate()
1164 (rs->rs_flags & RS_IS_DEAD)) { in rt_setup_rate()
1167 * the IF-UP was processed below, <or> in rt_setup_rate()
1182 if ((rs == NULL) || (rs->rs_disable != 0)) { in rt_setup_rate()
1188 if (rs->rs_flags & RS_IS_DEFF) { in rt_setup_rate()
1194 if (rs->rs_disable && error) in rt_setup_rate()
1205 if (rs->rs_flow_limit && in rt_setup_rate()
1206 ((rs->rs_flows_using + 1) > rs->rs_flow_limit)) { in rt_setup_rate()
1215 inp->inp_flowtype, in rt_setup_rate()
1216 inp->inp_flowid, in rt_setup_rate()
1217 rte->rate, in rt_setup_rate()
1218 &inp->inp_snd_tag); in rt_setup_rate()
1225 KASSERT((inp->inp_snd_tag != NULL) , in rt_setup_rate()
1227 inp, rte, (unsigned long long)rte->rate, rs)); in rt_setup_rate()
1238 atomic_add_64(&rs->rs_flows_using, 1); in rt_setup_rate()
1251 if (((ifp->if_capenable & IFCAP_TXRTLMT) == 0) || in tcp_rl_ifnet_link()
1254 * We only care on an interface going up that is rate-limit in tcp_rl_ifnet_link()
1285 rs_number_alive--; in tcp_rl_ifnet_departure()
1286 rs->rs_flags |= RS_IS_DEAD; in tcp_rl_ifnet_departure()
1287 for (i = 0; i < rs->rs_rate_cnt; i++) { in tcp_rl_ifnet_departure()
1288 if (rs->rs_rlt[i].flags & HDWRPACE_TAGPRESENT) { in tcp_rl_ifnet_departure()
1289 in_pcbdetach_tag(rs->rs_rlt[i].tag); in tcp_rl_ifnet_departure()
1290 rs->rs_rlt[i].tag = NULL; in tcp_rl_ifnet_departure()
1292 rs->rs_rlt[i].flags = HDWRPACE_IFPDEPARTED; in tcp_rl_ifnet_departure()
1294 if (rs->rs_flows_using == 0) in tcp_rl_ifnet_departure()
1318 rs_number_alive--; in tcp_rl_shutdown()
1319 rs->rs_flags |= RS_IS_DEAD; in tcp_rl_shutdown()
1320 for (i = 0; i < rs->rs_rate_cnt; i++) { in tcp_rl_shutdown()
1321 if (rs->rs_rlt[i].flags & HDWRPACE_TAGPRESENT) { in tcp_rl_shutdown()
1322 in_pcbdetach_tag(rs->rs_rlt[i].tag); in tcp_rl_shutdown()
1323 rs->rs_rlt[i].tag = NULL; in tcp_rl_shutdown()
1325 rs->rs_rlt[i].flags = HDWRPACE_IFPDEPARTED; in tcp_rl_shutdown()
1327 if (rs->rs_flows_using == 0) in tcp_rl_shutdown()
1346 if (inp->inp_snd_tag == NULL) { in tcp_set_pacing_rate()
1350 if ((ifp->if_capenable & IFCAP_TXRTLMT) == 0) { in tcp_set_pacing_rate()
1358 if (tp->t_nic_ktls_xmit != 0) { in tcp_set_pacing_rate()
1359 tls = tptosocket(tp)->so_snd.sb_tls_info; in tcp_set_pacing_rate()
1361 if ((ifp->if_capenable & IFCAP_TXTLS_RTLMT) == 0 || in tcp_set_pacing_rate()
1362 tls->mode != TCP_TLS_MODE_IFNET) { in tcp_set_pacing_rate()
1373 if (rte != NULL && tls != NULL && tls->snd_tag != NULL) { in tcp_set_pacing_rate()
1379 MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS); in tcp_set_pacing_rate()
1392 tp->t_pacing_rate = rte->rate; in tcp_set_pacing_rate()
1421 if (tp->t_nic_ktls_xmit) { in tcp_chg_pacing_rate()
1422 tls = tptosocket(tp)->so_snd.sb_tls_info; in tcp_chg_pacing_rate()
1423 if (tls->mode != TCP_TLS_MODE_IFNET) in tcp_chg_pacing_rate()
1425 else if (tls->snd_tag != NULL && in tcp_chg_pacing_rate()
1426 tls->snd_tag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT) { in tcp_chg_pacing_rate()
1427 if (!tls->reset_pending) { in tcp_chg_pacing_rate()
1450 if (inp->inp_snd_tag == NULL) { in tcp_chg_pacing_rate()
1457 rs = crte->ptbl; in tcp_chg_pacing_rate()
1458 if ((rs->rs_flags & RS_IS_DEAD) || in tcp_chg_pacing_rate()
1459 (crte->flags & HDWRPACE_IFPDEPARTED)) { in tcp_chg_pacing_rate()
1486 err = ktls_modify_txrtlmt(tls, nrte->rate); in tcp_chg_pacing_rate()
1489 err = in_pcbmodify_txrtlmt(inp, nrte->rate); in tcp_chg_pacing_rate()
1492 uint64_t pre; in tcp_chg_pacing_rate() local
1496 pre = atomic_fetchadd_64(&lrs->rs_flows_using, -1); in tcp_chg_pacing_rate()
1497 /* Do we still have a snd-tag attached? */ in tcp_chg_pacing_rate()
1498 if (inp->inp_snd_tag) in tcp_chg_pacing_rate()
1501 if (pre == 1) { in tcp_chg_pacing_rate()
1509 if (lrs->rs_flags & RS_IS_DEAD) in tcp_chg_pacing_rate()
1524 tp->t_pacing_rate = nrte->rate; in tcp_chg_pacing_rate()
1534 uint64_t pre; in tcp_rel_pacing_rate() local
1538 tp->t_pacing_rate = -1; in tcp_rel_pacing_rate()
1539 crs = crte->ptbl; in tcp_rel_pacing_rate()
1546 pre = atomic_fetchadd_64(&rs->rs_flows_using, -1); in tcp_rel_pacing_rate()
1547 if (pre == 1) { in tcp_rel_pacing_rate()
1555 if (rs->rs_flags & RS_IS_DEAD) in tcp_rel_pacing_rate()
1597 &tptosocket(tp)->so_rcv, in tcp_log_pacing_size()
1598 &tptosocket(tp)->so_snd, in tcp_log_pacing_size()
1638 new_tso = (bytes + segsiz - 1) / segsiz; in tcp_get_pacing_burst_size_w_divisor()
1673 * just return the non-hardware TSO size. in tcp_get_pacing_burst_size_w_divisor()
1690 if (te->rate > FIVE_HUNDRED_MBPS) in tcp_get_pacing_burst_size_w_divisor()
1692 if (te->rate == bw) { in tcp_get_pacing_burst_size_w_divisor()
1696 te->rate, te->time_between, (uint32_t)0, in tcp_get_pacing_burst_size_w_divisor()
1702 if (res > te->time_between) { in tcp_get_pacing_burst_size_w_divisor()
1706 delta = res - te->time_between; in tcp_get_pacing_burst_size_w_divisor()
1707 segs = (res_div + delta - 1)/delta; in tcp_get_pacing_burst_size_w_divisor()
1716 te->rate, te->time_between, (uint32_t)res, in tcp_get_pacing_burst_size_w_divisor()
1730 * hardware. Send back the non-hardware in tcp_get_pacing_burst_size_w_divisor()
1734 te->rate, te->time_between, (uint32_t)res, in tcp_get_pacing_burst_size_w_divisor()
1737 *err = -1; in tcp_get_pacing_burst_size_w_divisor()
1755 } else if (rs->rs_flags & RS_IS_DEFF) { in tcp_hw_highest_rate_ifp()
1768 rate_ret = rs->rs_rlt[rs->rs_highest_valid].rate; in tcp_hw_highest_rate_ifp()