1 /* Copyright (c) 2017 Facebook 2 * 3 * This program is free software; you can redistribute it and/or 4 * modify it under the terms of version 2 of the GNU General Public 5 * License as published by the Free Software Foundation. 6 */ 7 #include <linux/bpf.h> 8 #include <linux/slab.h> 9 #include <linux/vmalloc.h> 10 #include <linux/etherdevice.h> 11 #include <linux/filter.h> 12 #include <linux/sched/signal.h> 13 14 static __always_inline u32 bpf_test_run_one(struct bpf_prog *prog, void *ctx) 15 { 16 u32 ret; 17 18 preempt_disable(); 19 rcu_read_lock(); 20 ret = BPF_PROG_RUN(prog, ctx); 21 rcu_read_unlock(); 22 preempt_enable(); 23 24 return ret; 25 } 26 27 static u32 bpf_test_run(struct bpf_prog *prog, void *ctx, u32 repeat, u32 *time) 28 { 29 u64 time_start, time_spent = 0; 30 u32 ret = 0, i; 31 32 if (!repeat) 33 repeat = 1; 34 time_start = ktime_get_ns(); 35 for (i = 0; i < repeat; i++) { 36 ret = bpf_test_run_one(prog, ctx); 37 if (need_resched()) { 38 if (signal_pending(current)) 39 break; 40 time_spent += ktime_get_ns() - time_start; 41 cond_resched(); 42 time_start = ktime_get_ns(); 43 } 44 } 45 time_spent += ktime_get_ns() - time_start; 46 do_div(time_spent, repeat); 47 *time = time_spent > U32_MAX ? U32_MAX : (u32)time_spent; 48 49 return ret; 50 } 51 52 static int bpf_test_finish(union bpf_attr __user *uattr, const void *data, 53 u32 size, u32 retval, u32 duration) 54 { 55 void __user *data_out = u64_to_user_ptr(uattr->test.data_out); 56 int err = -EFAULT; 57 58 if (data_out && copy_to_user(data_out, data, size)) 59 goto out; 60 if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) 61 goto out; 62 if (copy_to_user(&uattr->test.retval, &retval, sizeof(retval))) 63 goto out; 64 if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) 65 goto out; 66 err = 0; 67 out: 68 return err; 69 } 70 71 static void *bpf_test_init(const union bpf_attr *kattr, u32 size, 72 u32 headroom, u32 tailroom) 73 { 74 void __user *data_in = u64_to_user_ptr(kattr->test.data_in); 75 void *data; 76 77 if (size < ETH_HLEN || size > PAGE_SIZE - headroom - tailroom) 78 return ERR_PTR(-EINVAL); 79 80 data = kzalloc(size + headroom + tailroom, GFP_USER); 81 if (!data) 82 return ERR_PTR(-ENOMEM); 83 84 if (copy_from_user(data + headroom, data_in, size)) { 85 kfree(data); 86 return ERR_PTR(-EFAULT); 87 } 88 return data; 89 } 90 91 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, 92 union bpf_attr __user *uattr) 93 { 94 bool is_l2 = false, is_direct_pkt_access = false; 95 u32 size = kattr->test.data_size_in; 96 u32 repeat = kattr->test.repeat; 97 u32 retval, duration; 98 struct sk_buff *skb; 99 void *data; 100 int ret; 101 102 data = bpf_test_init(kattr, size, NET_SKB_PAD, 103 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); 104 if (IS_ERR(data)) 105 return PTR_ERR(data); 106 107 switch (prog->type) { 108 case BPF_PROG_TYPE_SCHED_CLS: 109 case BPF_PROG_TYPE_SCHED_ACT: 110 is_l2 = true; 111 /* fall through */ 112 case BPF_PROG_TYPE_LWT_IN: 113 case BPF_PROG_TYPE_LWT_OUT: 114 case BPF_PROG_TYPE_LWT_XMIT: 115 is_direct_pkt_access = true; 116 break; 117 default: 118 break; 119 } 120 121 skb = build_skb(data, 0); 122 if (!skb) { 123 kfree(data); 124 return -ENOMEM; 125 } 126 127 skb_reserve(skb, NET_SKB_PAD); 128 __skb_put(skb, size); 129 skb->protocol = eth_type_trans(skb, current->nsproxy->net_ns->loopback_dev); 130 skb_reset_network_header(skb); 131 132 if (is_l2) 133 __skb_push(skb, ETH_HLEN); 134 if (is_direct_pkt_access) 135 bpf_compute_data_end(skb); 136 retval = bpf_test_run(prog, skb, repeat, &duration); 137 if (!is_l2) 138 __skb_push(skb, ETH_HLEN); 139 size = skb->len; 140 /* bpf program can never convert linear skb to non-linear */ 141 if (WARN_ON_ONCE(skb_is_nonlinear(skb))) 142 size = skb_headlen(skb); 143 ret = bpf_test_finish(uattr, skb->data, size, retval, duration); 144 kfree_skb(skb); 145 return ret; 146 } 147 148 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, 149 union bpf_attr __user *uattr) 150 { 151 u32 size = kattr->test.data_size_in; 152 u32 repeat = kattr->test.repeat; 153 struct xdp_buff xdp = {}; 154 u32 retval, duration; 155 void *data; 156 int ret; 157 158 data = bpf_test_init(kattr, size, XDP_PACKET_HEADROOM, 0); 159 if (IS_ERR(data)) 160 return PTR_ERR(data); 161 162 xdp.data_hard_start = data; 163 xdp.data = data + XDP_PACKET_HEADROOM; 164 xdp.data_end = xdp.data + size; 165 166 retval = bpf_test_run(prog, &xdp, repeat, &duration); 167 if (xdp.data != data + XDP_PACKET_HEADROOM) 168 size = xdp.data_end - xdp.data; 169 ret = bpf_test_finish(uattr, xdp.data, size, retval, duration); 170 kfree(data); 171 return ret; 172 } 173