1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _PROTO_MEMORY_H 3 #define _PROTO_MEMORY_H 4 5 #include <net/sock.h> 6 #include <net/hotdata.h> 7 8 /* 1 MB per cpu, in page units */ 9 #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) 10 11 static inline bool sk_has_memory_pressure(const struct sock *sk) 12 { 13 return sk->sk_prot->memory_pressure != NULL; 14 } 15 16 static inline bool 17 proto_memory_pressure(const struct proto *prot) 18 { 19 if (!prot->memory_pressure) 20 return false; 21 return !!READ_ONCE(*prot->memory_pressure); 22 } 23 24 static inline bool sk_under_global_memory_pressure(const struct sock *sk) 25 { 26 return proto_memory_pressure(sk->sk_prot); 27 } 28 29 static inline bool sk_under_memory_pressure(const struct sock *sk) 30 { 31 if (!sk->sk_prot->memory_pressure) 32 return false; 33 34 if (mem_cgroup_sk_enabled(sk) && 35 mem_cgroup_sk_under_memory_pressure(sk)) 36 return true; 37 38 if (sk->sk_bypass_prot_mem) 39 return false; 40 41 return !!READ_ONCE(*sk->sk_prot->memory_pressure); 42 } 43 44 static inline long 45 proto_memory_allocated(const struct proto *prot) 46 { 47 return max(0L, atomic_long_read(prot->memory_allocated)); 48 } 49 50 static inline long 51 sk_memory_allocated(const struct sock *sk) 52 { 53 return proto_memory_allocated(sk->sk_prot); 54 } 55 56 static inline void proto_memory_pcpu_drain(struct proto *proto) 57 { 58 int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0); 59 60 if (val) 61 atomic_long_add(val, proto->memory_allocated); 62 } 63 64 static inline void 65 sk_memory_allocated_add(const struct sock *sk, int val) 66 { 67 struct proto *proto = sk->sk_prot; 68 69 val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val); 70 71 if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) 72 proto_memory_pcpu_drain(proto); 73 } 74 75 static inline void 76 sk_memory_allocated_sub(const struct sock *sk, int val) 77 { 78 struct proto *proto = sk->sk_prot; 79 80 val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val); 81 82 if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) 83 proto_memory_pcpu_drain(proto); 84 } 85 86 #endif /* _PROTO_MEMORY_H */ 87