1 #ifndef __BPF_EXPERIMENTAL__ 2 #define __BPF_EXPERIMENTAL__ 3 4 #include <vmlinux.h> 5 #include <bpf/bpf_tracing.h> 6 #include <bpf/bpf_helpers.h> 7 #include <bpf/bpf_core_read.h> 8 9 #define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node))) 10 11 /* Description 12 * Allocates an object of the type represented by 'local_type_id' in 13 * program BTF. User may use the bpf_core_type_id_local macro to pass the 14 * type ID of a struct in program BTF. 15 * 16 * The 'local_type_id' parameter must be a known constant. 17 * The 'meta' parameter is rewritten by the verifier, no need for BPF 18 * program to set it. 19 * Returns 20 * A pointer to an object of the type corresponding to the passed in 21 * 'local_type_id', or NULL on failure. 22 */ 23 extern void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 24 25 /* Convenience macro to wrap over bpf_obj_new_impl */ 26 #define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL)) 27 28 /* Description 29 * Free an allocated object. All fields of the object that require 30 * destruction will be destructed before the storage is freed. 31 * 32 * The 'meta' parameter is rewritten by the verifier, no need for BPF 33 * program to set it. 34 * Returns 35 * Void. 36 */ 37 extern void bpf_obj_drop_impl(void *kptr, void *meta) __ksym; 38 39 /* Convenience macro to wrap over bpf_obj_drop_impl */ 40 #define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL) 41 42 /* Description 43 * Increment the refcount on a refcounted local kptr, turning the 44 * non-owning reference input into an owning reference in the process. 45 * 46 * The 'meta' parameter is rewritten by the verifier, no need for BPF 47 * program to set it. 48 * Returns 49 * An owning reference to the object pointed to by 'kptr' 50 */ 51 extern void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym; 52 53 /* Convenience macro to wrap over bpf_refcount_acquire_impl */ 54 #define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL) 55 56 /* Description 57 * Add a new entry to the beginning of the BPF linked list. 58 * 59 * The 'meta' and 'off' parameters are rewritten by the verifier, no need 60 * for BPF programs to set them 61 * Returns 62 * 0 if the node was successfully added 63 * -EINVAL if the node wasn't added because it's already in a list 64 */ 65 extern int bpf_list_push_front_impl(struct bpf_list_head *head, 66 struct bpf_list_node *node, 67 void *meta, __u64 off) __ksym; 68 69 /* Convenience macro to wrap over bpf_list_push_front_impl */ 70 #define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0) 71 72 /* Description 73 * Add a new entry to the end of the BPF linked list. 74 * 75 * The 'meta' and 'off' parameters are rewritten by the verifier, no need 76 * for BPF programs to set them 77 * Returns 78 * 0 if the node was successfully added 79 * -EINVAL if the node wasn't added because it's already in a list 80 */ 81 extern int bpf_list_push_back_impl(struct bpf_list_head *head, 82 struct bpf_list_node *node, 83 void *meta, __u64 off) __ksym; 84 85 /* Convenience macro to wrap over bpf_list_push_back_impl */ 86 #define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0) 87 88 /* Description 89 * Remove the entry at the beginning of the BPF linked list. 90 * Returns 91 * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. 92 */ 93 extern struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym; 94 95 /* Description 96 * Remove the entry at the end of the BPF linked list. 97 * Returns 98 * Pointer to bpf_list_node of deleted entry, or NULL if list is empty. 99 */ 100 extern struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym; 101 102 /* Description 103 * Remove 'node' from rbtree with root 'root' 104 * Returns 105 * Pointer to the removed node, or NULL if 'root' didn't contain 'node' 106 */ 107 extern struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root, 108 struct bpf_rb_node *node) __ksym; 109 110 /* Description 111 * Add 'node' to rbtree with root 'root' using comparator 'less' 112 * 113 * The 'meta' and 'off' parameters are rewritten by the verifier, no need 114 * for BPF programs to set them 115 * Returns 116 * 0 if the node was successfully added 117 * -EINVAL if the node wasn't added because it's already in a tree 118 */ 119 extern int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node, 120 bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b), 121 void *meta, __u64 off) __ksym; 122 123 /* Convenience macro to wrap over bpf_rbtree_add_impl */ 124 #define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0) 125 126 /* Description 127 * Return the first (leftmost) node in input tree 128 * Returns 129 * Pointer to the node, which is _not_ removed from the tree. If the tree 130 * contains no nodes, returns NULL. 131 */ 132 extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym; 133 134 /* Description 135 * Allocates a percpu object of the type represented by 'local_type_id' in 136 * program BTF. User may use the bpf_core_type_id_local macro to pass the 137 * type ID of a struct in program BTF. 138 * 139 * The 'local_type_id' parameter must be a known constant. 140 * The 'meta' parameter is rewritten by the verifier, no need for BPF 141 * program to set it. 142 * Returns 143 * A pointer to a percpu object of the type corresponding to the passed in 144 * 'local_type_id', or NULL on failure. 145 */ 146 extern void *bpf_percpu_obj_new_impl(__u64 local_type_id, void *meta) __ksym; 147 148 /* Convenience macro to wrap over bpf_percpu_obj_new_impl */ 149 #define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new_impl(bpf_core_type_id_local(type), NULL)) 150 151 /* Description 152 * Free an allocated percpu object. All fields of the object that require 153 * destruction will be destructed before the storage is freed. 154 * 155 * The 'meta' parameter is rewritten by the verifier, no need for BPF 156 * program to set it. 157 * Returns 158 * Void. 159 */ 160 extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym; 161 162 /* Convenience macro to wrap over bpf_obj_drop_impl */ 163 #define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL) 164 165 /* Description 166 * Throw a BPF exception from the program, immediately terminating its 167 * execution and unwinding the stack. The supplied 'cookie' parameter 168 * will be the return value of the program when an exception is thrown, 169 * and the default exception callback is used. Otherwise, if an exception 170 * callback is set using the '__exception_cb(callback)' declaration tag 171 * on the main program, the 'cookie' parameter will be the callback's only 172 * input argument. 173 * 174 * Thus, in case of default exception callback, 'cookie' is subjected to 175 * constraints on the program's return value (as with R0 on exit). 176 * Otherwise, the return value of the marked exception callback will be 177 * subjected to the same checks. 178 * 179 * Note that throwing an exception with lingering resources (locks, 180 * references, etc.) will lead to a verification error. 181 * 182 * Note that callbacks *cannot* call this helper. 183 * Returns 184 * Never. 185 * Throws 186 * An exception with the specified 'cookie' value. 187 */ 188 extern void bpf_throw(u64 cookie) __ksym; 189 190 /* This macro must be used to mark the exception callback corresponding to the 191 * main program. For example: 192 * 193 * int exception_cb(u64 cookie) { 194 * return cookie; 195 * } 196 * 197 * SEC("tc") 198 * __exception_cb(exception_cb) 199 * int main_prog(struct __sk_buff *ctx) { 200 * ... 201 * return TC_ACT_OK; 202 * } 203 * 204 * Here, exception callback for the main program will be 'exception_cb'. Note 205 * that this attribute can only be used once, and multiple exception callbacks 206 * specified for the main program will lead to verification error. 207 */ 208 #define __exception_cb(name) __attribute__((btf_decl_tag("exception_callback:" #name))) 209 210 #define __bpf_assert_signed(x) _Generic((x), \ 211 unsigned long: 0, \ 212 unsigned long long: 0, \ 213 signed long: 1, \ 214 signed long long: 1 \ 215 ) 216 217 #define __bpf_assert_check(LHS, op, RHS) \ 218 _Static_assert(sizeof(&(LHS)), "1st argument must be an lvalue expression"); \ 219 _Static_assert(sizeof(LHS) == 8, "Only 8-byte integers are supported\n"); \ 220 _Static_assert(__builtin_constant_p(__bpf_assert_signed(LHS)), "internal static assert"); \ 221 _Static_assert(__builtin_constant_p((RHS)), "2nd argument must be a constant expression") 222 223 #define __bpf_assert(LHS, op, cons, RHS, VAL) \ 224 ({ \ 225 (void)bpf_throw; \ 226 asm volatile ("if %[lhs] " op " %[rhs] goto +2; r1 = %[value]; call bpf_throw" \ 227 : : [lhs] "r"(LHS), [rhs] cons(RHS), [value] "ri"(VAL) : ); \ 228 }) 229 230 #define __bpf_assert_op_sign(LHS, op, cons, RHS, VAL, supp_sign) \ 231 ({ \ 232 __bpf_assert_check(LHS, op, RHS); \ 233 if (__bpf_assert_signed(LHS) && !(supp_sign)) \ 234 __bpf_assert(LHS, "s" #op, cons, RHS, VAL); \ 235 else \ 236 __bpf_assert(LHS, #op, cons, RHS, VAL); \ 237 }) 238 239 #define __bpf_assert_op(LHS, op, RHS, VAL, supp_sign) \ 240 ({ \ 241 if (sizeof(typeof(RHS)) == 8) { \ 242 const typeof(RHS) rhs_var = (RHS); \ 243 __bpf_assert_op_sign(LHS, op, "r", rhs_var, VAL, supp_sign); \ 244 } else { \ 245 __bpf_assert_op_sign(LHS, op, "i", RHS, VAL, supp_sign); \ 246 } \ 247 }) 248 249 /* Description 250 * Assert that a conditional expression is true. 251 * Returns 252 * Void. 253 * Throws 254 * An exception with the value zero when the assertion fails. 255 */ 256 #define bpf_assert(cond) if (!(cond)) bpf_throw(0); 257 258 /* Description 259 * Assert that a conditional expression is true. 260 * Returns 261 * Void. 262 * Throws 263 * An exception with the specified value when the assertion fails. 264 */ 265 #define bpf_assert_with(cond, value) if (!(cond)) bpf_throw(value); 266 267 /* Description 268 * Assert that LHS is equal to RHS. This statement updates the known value 269 * of LHS during verification. Note that RHS must be a constant value, and 270 * must fit within the data type of LHS. 271 * Returns 272 * Void. 273 * Throws 274 * An exception with the value zero when the assertion fails. 275 */ 276 #define bpf_assert_eq(LHS, RHS) \ 277 ({ \ 278 barrier_var(LHS); \ 279 __bpf_assert_op(LHS, ==, RHS, 0, true); \ 280 }) 281 282 /* Description 283 * Assert that LHS is equal to RHS. This statement updates the known value 284 * of LHS during verification. Note that RHS must be a constant value, and 285 * must fit within the data type of LHS. 286 * Returns 287 * Void. 288 * Throws 289 * An exception with the specified value when the assertion fails. 290 */ 291 #define bpf_assert_eq_with(LHS, RHS, value) \ 292 ({ \ 293 barrier_var(LHS); \ 294 __bpf_assert_op(LHS, ==, RHS, value, true); \ 295 }) 296 297 /* Description 298 * Assert that LHS is less than RHS. This statement updates the known 299 * bounds of LHS during verification. Note that RHS must be a constant 300 * value, and must fit within the data type of LHS. 301 * Returns 302 * Void. 303 * Throws 304 * An exception with the value zero when the assertion fails. 305 */ 306 #define bpf_assert_lt(LHS, RHS) \ 307 ({ \ 308 barrier_var(LHS); \ 309 __bpf_assert_op(LHS, <, RHS, 0, false); \ 310 }) 311 312 /* Description 313 * Assert that LHS is less than RHS. This statement updates the known 314 * bounds of LHS during verification. Note that RHS must be a constant 315 * value, and must fit within the data type of LHS. 316 * Returns 317 * Void. 318 * Throws 319 * An exception with the specified value when the assertion fails. 320 */ 321 #define bpf_assert_lt_with(LHS, RHS, value) \ 322 ({ \ 323 barrier_var(LHS); \ 324 __bpf_assert_op(LHS, <, RHS, value, false); \ 325 }) 326 327 /* Description 328 * Assert that LHS is greater than RHS. This statement updates the known 329 * bounds of LHS during verification. Note that RHS must be a constant 330 * value, and must fit within the data type of LHS. 331 * Returns 332 * Void. 333 * Throws 334 * An exception with the value zero when the assertion fails. 335 */ 336 #define bpf_assert_gt(LHS, RHS) \ 337 ({ \ 338 barrier_var(LHS); \ 339 __bpf_assert_op(LHS, >, RHS, 0, false); \ 340 }) 341 342 /* Description 343 * Assert that LHS is greater than RHS. This statement updates the known 344 * bounds of LHS during verification. Note that RHS must be a constant 345 * value, and must fit within the data type of LHS. 346 * Returns 347 * Void. 348 * Throws 349 * An exception with the specified value when the assertion fails. 350 */ 351 #define bpf_assert_gt_with(LHS, RHS, value) \ 352 ({ \ 353 barrier_var(LHS); \ 354 __bpf_assert_op(LHS, >, RHS, value, false); \ 355 }) 356 357 /* Description 358 * Assert that LHS is less than or equal to RHS. This statement updates the 359 * known bounds of LHS during verification. Note that RHS must be a 360 * constant value, and must fit within the data type of LHS. 361 * Returns 362 * Void. 363 * Throws 364 * An exception with the value zero when the assertion fails. 365 */ 366 #define bpf_assert_le(LHS, RHS) \ 367 ({ \ 368 barrier_var(LHS); \ 369 __bpf_assert_op(LHS, <=, RHS, 0, false); \ 370 }) 371 372 /* Description 373 * Assert that LHS is less than or equal to RHS. This statement updates the 374 * known bounds of LHS during verification. Note that RHS must be a 375 * constant value, and must fit within the data type of LHS. 376 * Returns 377 * Void. 378 * Throws 379 * An exception with the specified value when the assertion fails. 380 */ 381 #define bpf_assert_le_with(LHS, RHS, value) \ 382 ({ \ 383 barrier_var(LHS); \ 384 __bpf_assert_op(LHS, <=, RHS, value, false); \ 385 }) 386 387 /* Description 388 * Assert that LHS is greater than or equal to RHS. This statement updates 389 * the known bounds of LHS during verification. Note that RHS must be a 390 * constant value, and must fit within the data type of LHS. 391 * Returns 392 * Void. 393 * Throws 394 * An exception with the value zero when the assertion fails. 395 */ 396 #define bpf_assert_ge(LHS, RHS) \ 397 ({ \ 398 barrier_var(LHS); \ 399 __bpf_assert_op(LHS, >=, RHS, 0, false); \ 400 }) 401 402 /* Description 403 * Assert that LHS is greater than or equal to RHS. This statement updates 404 * the known bounds of LHS during verification. Note that RHS must be a 405 * constant value, and must fit within the data type of LHS. 406 * Returns 407 * Void. 408 * Throws 409 * An exception with the specified value when the assertion fails. 410 */ 411 #define bpf_assert_ge_with(LHS, RHS, value) \ 412 ({ \ 413 barrier_var(LHS); \ 414 __bpf_assert_op(LHS, >=, RHS, value, false); \ 415 }) 416 417 /* Description 418 * Assert that LHS is in the range [BEG, END] (inclusive of both). This 419 * statement updates the known bounds of LHS during verification. Note 420 * that both BEG and END must be constant values, and must fit within the 421 * data type of LHS. 422 * Returns 423 * Void. 424 * Throws 425 * An exception with the value zero when the assertion fails. 426 */ 427 #define bpf_assert_range(LHS, BEG, END) \ 428 ({ \ 429 _Static_assert(BEG <= END, "BEG must be <= END"); \ 430 barrier_var(LHS); \ 431 __bpf_assert_op(LHS, >=, BEG, 0, false); \ 432 __bpf_assert_op(LHS, <=, END, 0, false); \ 433 }) 434 435 /* Description 436 * Assert that LHS is in the range [BEG, END] (inclusive of both). This 437 * statement updates the known bounds of LHS during verification. Note 438 * that both BEG and END must be constant values, and must fit within the 439 * data type of LHS. 440 * Returns 441 * Void. 442 * Throws 443 * An exception with the specified value when the assertion fails. 444 */ 445 #define bpf_assert_range_with(LHS, BEG, END, value) \ 446 ({ \ 447 _Static_assert(BEG <= END, "BEG must be <= END"); \ 448 barrier_var(LHS); \ 449 __bpf_assert_op(LHS, >=, BEG, value, false); \ 450 __bpf_assert_op(LHS, <=, END, value, false); \ 451 }) 452 453 #endif 454