1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "bpf_tracing_net.h" 4 #include <bpf/bpf_helpers.h> 5 #include <bpf/bpf_tracing.h> 6 7 char _license[] SEC("license") = "GPL"; 8 9 #define USEC_PER_SEC 1000000UL 10 11 static unsigned int tcp_left_out(const struct tcp_sock *tp) 12 { 13 return tp->sacked_out + tp->lost_out; 14 } 15 16 static unsigned int tcp_packets_in_flight(const struct tcp_sock *tp) 17 { 18 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out; 19 } 20 21 SEC("struct_ops") 22 void BPF_PROG(write_sk_pacing_init, struct sock *sk) 23 { 24 #ifdef ENABLE_ATOMICS_TESTS 25 __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE, 26 SK_PACING_NEEDED); 27 #else 28 sk->sk_pacing_status = SK_PACING_NEEDED; 29 #endif 30 } 31 32 SEC("struct_ops") 33 void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk, 34 const struct rate_sample *rs) 35 { 36 struct tcp_sock *tp = tcp_sk(sk); 37 unsigned long rate = 38 ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) / 39 (tp->srtt_us ?: 1U << 3); 40 sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate); 41 tp->app_limited = (tp->delivered + tcp_packets_in_flight(tp)) ?: 1; 42 } 43 44 SEC("struct_ops") 45 __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk) 46 { 47 return tcp_sk(sk)->snd_ssthresh; 48 } 49 50 SEC("struct_ops") 51 __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk) 52 { 53 return tcp_sk(sk)->snd_cwnd; 54 } 55 56 SEC(".struct_ops") 57 struct tcp_congestion_ops write_sk_pacing = { 58 .init = (void *)write_sk_pacing_init, 59 .cong_control = (void *)write_sk_pacing_cong_control, 60 .ssthresh = (void *)write_sk_pacing_ssthresh, 61 .undo_cwnd = (void *)write_sk_pacing_undo_cwnd, 62 .name = "bpf_w_sk_pacing", 63 }; 64