1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <vmlinux.h> 4 #include <bpf/bpf_core_read.h> 5 #include <bpf/bpf_helpers.h> 6 #include <bpf/bpf_tracing.h> 7 #include "bpf_misc.h" 8 9 char _license[] SEC("license") = "GPL"; 10 11 /* typically virtio scsi has max SGs of 6 */ 12 #define VIRTIO_MAX_SGS 6 13 14 /* Verifier will fail with SG_MAX = 128. The failure can be 15 * workarounded with a smaller SG_MAX, e.g. 10. 16 */ 17 #define WORKAROUND 18 #ifdef WORKAROUND 19 #define SG_MAX 10 20 #else 21 /* typically virtio blk has max SEG of 128 */ 22 #define SG_MAX 128 23 #endif 24 25 #define SG_CHAIN 0x01UL 26 #define SG_END 0x02UL 27 28 #define sg_is_chain(sg) ((sg)->page_link & SG_CHAIN) 29 #define sg_is_last(sg) ((sg)->page_link & SG_END) 30 #define sg_chain_ptr(sg) \ 31 ((struct scatterlist *) ((sg)->page_link & ~(SG_CHAIN | SG_END))) 32 33 static inline struct scatterlist *__sg_next(struct scatterlist *sgp) 34 { 35 struct scatterlist sg; 36 37 bpf_probe_read_kernel(&sg, sizeof(sg), sgp); 38 if (sg_is_last(&sg)) 39 return NULL; 40 41 sgp++; 42 43 bpf_probe_read_kernel(&sg, sizeof(sg), sgp); 44 if (sg_is_chain(&sg)) 45 sgp = sg_chain_ptr(&sg); 46 47 return sgp; 48 } 49 50 static inline struct scatterlist *get_sgp(struct scatterlist **sgs, int i) 51 { 52 struct scatterlist *sgp; 53 54 bpf_probe_read_kernel(&sgp, sizeof(sgp), sgs + i); 55 return sgp; 56 } 57 58 int run_once = 0; 59 int result = 0; 60 61 SEC("kprobe/virtqueue_add_sgs") 62 int BPF_KPROBE(trace_virtqueue_add_sgs, void *unused, struct scatterlist **sgs, 63 unsigned int out_sgs, unsigned int in_sgs) 64 { 65 struct scatterlist *sgp = NULL; 66 __u64 length1 = 0, length2 = 0; 67 unsigned int i, n, len; 68 69 if (run_once != 0) 70 return 0; 71 72 for (i = 0; (i < VIRTIO_MAX_SGS) && (i < out_sgs); i++) { 73 __sink(out_sgs); 74 for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX); 75 sgp = __sg_next(sgp)) { 76 len = BPF_CORE_READ(sgp, length); 77 length1 += len; 78 n++; 79 } 80 } 81 82 for (i = 0; (i < VIRTIO_MAX_SGS) && (i < in_sgs); i++) { 83 __sink(in_sgs); 84 for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX); 85 sgp = __sg_next(sgp)) { 86 len = BPF_CORE_READ(sgp, length); 87 length2 += len; 88 n++; 89 } 90 } 91 92 run_once = 1; 93 result = length2 - length1; 94 return 0; 95 } 96