1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 #ifndef __TASK_LOCAL_DATA_BPF_H 3 #define __TASK_LOCAL_DATA_BPF_H 4 5 /* 6 * Task local data is a library that facilitates sharing per-task data 7 * between user space and bpf programs. 8 * 9 * 10 * USAGE 11 * 12 * A TLD, an entry of data in task local data, first needs to be created by the 13 * user space. This is done by calling user space API, TLD_DEFINE_KEY() or 14 * tld_create_key(), with the name of the TLD and the size. 15 * 16 * TLD_DEFINE_KEY(prio, "priority", sizeof(int)); 17 * 18 * or 19 * 20 * void func_call(...) { 21 * tld_key_t prio, in_cs; 22 * 23 * prio = tld_create_key("priority", sizeof(int)); 24 * in_cs = tld_create_key("in_critical_section", sizeof(bool)); 25 * ... 26 * 27 * A key associated with the TLD, which has an opaque type tld_key_t, will be 28 * initialized or returned. It can be used to get a pointer to the TLD in the 29 * user space by calling tld_get_data(). 30 * 31 * In a bpf program, tld_object_init() first needs to be called to initialized a 32 * tld_object on the stack. Then, TLDs can be accessed by calling tld_get_data(). 33 * The API will try to fetch the key by the name and use it to locate the data. 34 * A pointer to the TLD will be returned. It also caches the key in a task local 35 * storage map, tld_key_map, whose value type, struct tld_keys, must be defined 36 * by the developer. 37 * 38 * struct tld_keys { 39 * tld_key_t prio; 40 * tld_key_t in_cs; 41 * }; 42 * 43 * SEC("struct_ops") 44 * void prog(struct task_struct task, ...) 45 * { 46 * struct tld_object tld_obj; 47 * int err, *p; 48 * 49 * err = tld_object_init(task, &tld_obj); 50 * if (err) 51 * return; 52 * 53 * p = tld_get_data(&tld_obj, prio, "priority", sizeof(int)); 54 * if (p) 55 * // do something depending on *p 56 */ 57 #include <errno.h> 58 #include <bpf/bpf_helpers.h> 59 60 #define TLD_ROUND_MASK(x, y) ((__typeof__(x))((y) - 1)) 61 #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) 62 63 #define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1) 64 #define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64)) 65 66 #ifndef TLD_NAME_LEN 67 #define TLD_NAME_LEN 62 68 #endif 69 70 #ifndef TLD_KEY_MAP_CREATE_RETRY 71 #define TLD_KEY_MAP_CREATE_RETRY 10 72 #endif 73 74 typedef struct { 75 __s16 off; 76 } tld_key_t; 77 78 struct tld_metadata { 79 char name[TLD_NAME_LEN]; 80 __u16 size; 81 }; 82 83 struct tld_meta_u { 84 __u16 cnt; 85 __u16 size; 86 struct tld_metadata metadata[TLD_MAX_DATA_CNT]; 87 }; 88 89 struct tld_data_u { 90 __u64 unused; 91 char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8))); 92 }; 93 94 struct tld_map_value { 95 struct tld_data_u __uptr *data; 96 struct tld_meta_u __uptr *meta; 97 __u16 start; /* offset of tld_data_u->data in a page */ 98 }; 99 100 typedef struct tld_uptr_dummy { 101 struct tld_data_u data[0]; 102 struct tld_meta_u meta[0]; 103 } *tld_uptr_dummy_t; 104 105 struct tld_object { 106 struct tld_map_value *data_map; 107 struct tld_keys *key_map; 108 /* 109 * Force the compiler to generate the actual definition of tld_meta_u 110 * and tld_data_u in BTF. Without it, tld_meta_u and u_tld_data will 111 * be BTF_KIND_FWD. 112 */ 113 tld_uptr_dummy_t dummy[0]; 114 }; 115 116 /* 117 * Map value of tld_key_map for caching keys. Must be defined by the developer. 118 * Members should be tld_key_t and passed to the 3rd argument of tld_fetch_key(). 119 */ 120 struct tld_keys; 121 122 struct { 123 __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 124 __uint(map_flags, BPF_F_NO_PREALLOC); 125 __type(key, int); 126 __type(value, struct tld_map_value); 127 } tld_data_map SEC(".maps"); 128 129 struct { 130 __uint(type, BPF_MAP_TYPE_TASK_STORAGE); 131 __uint(map_flags, BPF_F_NO_PREALLOC); 132 __type(key, int); 133 __type(value, struct tld_keys); 134 } tld_key_map SEC(".maps"); 135 136 /** 137 * tld_object_init() - Initialize a tld_object. 138 * 139 * @task: The task_struct of the target task 140 * @tld_obj: A pointer to a tld_object to be initialized 141 * 142 * Return 0 on success; -ENODATA if the user space did not initialize task local data 143 * for the current task through tld_get_data(); -ENOMEM if the creation of tld_key_map 144 * fails 145 */ 146 __attribute__((unused)) 147 static int tld_object_init(struct task_struct *task, struct tld_object *tld_obj) 148 { 149 int i; 150 151 tld_obj->data_map = bpf_task_storage_get(&tld_data_map, task, 0, 0); 152 if (!tld_obj->data_map) 153 return -ENODATA; 154 155 bpf_for(i, 0, TLD_KEY_MAP_CREATE_RETRY) { 156 tld_obj->key_map = bpf_task_storage_get(&tld_key_map, task, 0, 157 BPF_LOCAL_STORAGE_GET_F_CREATE); 158 if (likely(tld_obj->key_map)) 159 break; 160 } 161 if (!tld_obj->key_map) 162 return -ENOMEM; 163 164 return 0; 165 } 166 167 /* 168 * Return the offset of TLD if @name is found. Otherwise, return the current TLD count 169 * using the nonpositive range so that the next tld_get_data() can skip fetching key if 170 * no new TLD is added or start comparing name from the first newly added TLD. 171 */ 172 __attribute__((unused)) 173 static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_start) 174 { 175 struct tld_metadata *metadata; 176 int i, cnt, start, off = 0; 177 178 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta) 179 return 0; 180 181 start = tld_obj->data_map->start; 182 cnt = tld_obj->data_map->meta->cnt; 183 metadata = tld_obj->data_map->meta->metadata; 184 185 bpf_for(i, 0, cnt) { 186 if (i >= TLD_MAX_DATA_CNT) 187 break; 188 189 if (i >= i_start && !bpf_strncmp(metadata[i].name, TLD_NAME_LEN, name)) 190 return start + off; 191 192 off += TLD_ROUND_UP(metadata[i].size, 8); 193 if (off > TLD_DATA_SIZE) 194 break; 195 } 196 197 return -cnt; 198 } 199 200 /** 201 * tld_get_data() - Retrieve a pointer to the TLD associated with the name. 202 * 203 * @tld_obj: A pointer to a valid tld_object initialized by tld_object_init() 204 * @key: The cached key of the TLD in tld_key_map 205 * @name: The name of the key associated with a TLD 206 * @size: The size of the TLD. Must be a known constant value 207 * 208 * Return a pointer to the TLD associated with @name; NULL if not found or @size is too 209 * big. @key is used to cache the key if the TLD is found to speed up subsequent calls. 210 * It should be defined as an member of tld_keys of tld_key_t type by the developer. 211 */ 212 #define tld_get_data(tld_obj, key, name, size) \ 213 ({ \ 214 void *data = NULL, *_data = (tld_obj)->data_map->data; \ 215 long off = (tld_obj)->key_map->key.off; \ 216 int cnt; \ 217 \ 218 if (likely(_data)) { \ 219 if (likely(off > 0)) { \ 220 barrier_var(off); \ 221 if (likely(off < __PAGE_SIZE - size)) \ 222 data = _data + off; \ 223 } else { \ 224 cnt = -(off); \ 225 if (likely((tld_obj)->data_map->meta) && \ 226 cnt < (tld_obj)->data_map->meta->cnt) { \ 227 off = __tld_fetch_key(tld_obj, name, cnt); \ 228 (tld_obj)->key_map->key.off = off; \ 229 \ 230 if (likely(off < __PAGE_SIZE - size)) { \ 231 barrier_var(off); \ 232 if (off > 0) \ 233 data = _data + off; \ 234 } \ 235 } \ 236 } \ 237 } \ 238 data; \ 239 }) 240 241 #endif 242