1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Traceprobe fetch helper inlines 4 */ 5 6 static nokprobe_inline void 7 fetch_store_raw(unsigned long val, struct fetch_insn *code, void *buf) 8 { 9 switch (code->size) { 10 case 1: 11 *(u8 *)buf = (u8)val; 12 break; 13 case 2: 14 *(u16 *)buf = (u16)val; 15 break; 16 case 4: 17 *(u32 *)buf = (u32)val; 18 break; 19 case 8: 20 //TBD: 32bit signed 21 *(u64 *)buf = (u64)val; 22 break; 23 default: 24 *(unsigned long *)buf = val; 25 } 26 } 27 28 static nokprobe_inline void 29 fetch_apply_bitfield(struct fetch_insn *code, void *buf) 30 { 31 switch (code->basesize) { 32 case 1: 33 *(u8 *)buf <<= code->lshift; 34 *(u8 *)buf >>= code->rshift; 35 break; 36 case 2: 37 *(u16 *)buf <<= code->lshift; 38 *(u16 *)buf >>= code->rshift; 39 break; 40 case 4: 41 *(u32 *)buf <<= code->lshift; 42 *(u32 *)buf >>= code->rshift; 43 break; 44 case 8: 45 *(u64 *)buf <<= code->lshift; 46 *(u64 *)buf >>= code->rshift; 47 break; 48 } 49 } 50 51 /* 52 * These functions must be defined for each callsite. 53 * Return consumed dynamic data size (>= 0), or error (< 0). 54 * If dest is NULL, don't store result and return required dynamic data size. 55 */ 56 static int 57 process_fetch_insn(struct fetch_insn *code, void *rec, void *edata, 58 void *dest, void *base); 59 static nokprobe_inline int fetch_store_strlen(unsigned long addr); 60 static nokprobe_inline int 61 fetch_store_string(unsigned long addr, void *dest, void *base); 62 static nokprobe_inline int fetch_store_strlen_user(unsigned long addr); 63 static nokprobe_inline int 64 fetch_store_string_user(unsigned long addr, void *dest, void *base); 65 static nokprobe_inline int 66 probe_mem_read(void *dest, void *src, size_t size); 67 static nokprobe_inline int 68 probe_mem_read_user(void *dest, void *src, size_t size); 69 70 static nokprobe_inline int 71 fetch_store_symstrlen(unsigned long addr) 72 { 73 char namebuf[KSYM_SYMBOL_LEN]; 74 int ret; 75 76 ret = sprint_symbol(namebuf, addr); 77 if (ret < 0) 78 return 0; 79 80 return ret + 1; 81 } 82 83 /* 84 * Fetch a null-terminated symbol string + offset. Caller MUST set *(u32 *)buf 85 * with max length and relative data location. 86 */ 87 static nokprobe_inline int 88 fetch_store_symstring(unsigned long addr, void *dest, void *base) 89 { 90 int maxlen = get_loc_len(*(u32 *)dest); 91 void *__dest; 92 93 if (unlikely(!maxlen)) 94 return -ENOMEM; 95 96 __dest = get_loc_data(dest, base); 97 98 return sprint_symbol(__dest, addr); 99 } 100 101 /* common part of process_fetch_insn*/ 102 static nokprobe_inline int 103 process_common_fetch_insn(struct fetch_insn *code, unsigned long *val) 104 { 105 switch (code->op) { 106 case FETCH_OP_IMM: 107 *val = code->immediate; 108 break; 109 case FETCH_OP_COMM: 110 *val = (unsigned long)current->comm; 111 break; 112 case FETCH_OP_IMMSTR: 113 *val = (unsigned long)code->data; 114 break; 115 case FETCH_OP_CURRENT: 116 *val = (unsigned long)current; 117 break; 118 default: 119 return -EILSEQ; 120 } 121 return 0; 122 } 123 124 /* From the 2nd stage, routine is same */ 125 static nokprobe_inline int 126 process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val, 127 void *dest, void *base) 128 { 129 struct fetch_insn *s3 = NULL; 130 int total = 0, ret = 0, i = 0; 131 u32 loc = 0; 132 unsigned long lval, llval = val; 133 134 stage2: 135 /* 2nd stage: dereference memory if needed */ 136 do { 137 lval = val; 138 switch (code->op) { 139 case FETCH_OP_DEREF: 140 ret = probe_mem_read(&val, (void *)val + code->offset, 141 sizeof(val)); 142 break; 143 case FETCH_OP_UDEREF: 144 ret = probe_mem_read_user(&val, 145 (void *)val + code->offset, sizeof(val)); 146 break; 147 case FETCH_OP_CPU_PTR: 148 val = (unsigned long)this_cpu_ptr((void __percpu *)val); 149 ret = 0; 150 break; 151 default: 152 lval = llval; 153 goto out; 154 } 155 if (ret) 156 return ret; 157 llval = lval; 158 code++; 159 } while (1); 160 out: 161 162 s3 = code; 163 stage3: 164 /* 3rd stage: store value to buffer */ 165 if (unlikely(!dest)) { 166 switch (code->op) { 167 case FETCH_OP_ST_STRING: 168 ret = fetch_store_strlen(val + code->offset); 169 code++; 170 goto array; 171 case FETCH_OP_ST_USTRING: 172 ret = fetch_store_strlen_user(val + code->offset); 173 code++; 174 goto array; 175 case FETCH_OP_ST_SYMSTR: 176 ret = fetch_store_symstrlen(val + code->offset); 177 code++; 178 goto array; 179 default: 180 return -EILSEQ; 181 } 182 } 183 184 switch (code->op) { 185 case FETCH_OP_ST_RAW: 186 fetch_store_raw(val, code, dest); 187 break; 188 case FETCH_OP_ST_MEM: 189 probe_mem_read(dest, (void *)val + code->offset, code->size); 190 break; 191 case FETCH_OP_ST_UMEM: 192 probe_mem_read_user(dest, (void *)val + code->offset, code->size); 193 break; 194 case FETCH_OP_ST_STRING: 195 loc = *(u32 *)dest; 196 ret = fetch_store_string(val + code->offset, dest, base); 197 break; 198 case FETCH_OP_ST_USTRING: 199 loc = *(u32 *)dest; 200 ret = fetch_store_string_user(val + code->offset, dest, base); 201 break; 202 case FETCH_OP_ST_SYMSTR: 203 loc = *(u32 *)dest; 204 ret = fetch_store_symstring(val + code->offset, dest, base); 205 break; 206 default: 207 return -EILSEQ; 208 } 209 code++; 210 211 /* 4th stage: modify stored value if needed */ 212 if (code->op == FETCH_OP_MOD_BF) { 213 fetch_apply_bitfield(code, dest); 214 code++; 215 } 216 217 array: 218 /* the last stage: Loop on array */ 219 if (code->op == FETCH_OP_LP_ARRAY) { 220 if (ret < 0) 221 ret = 0; 222 total += ret; 223 if (++i < code->param) { 224 code = s3; 225 if (s3->op != FETCH_OP_ST_STRING && 226 s3->op != FETCH_OP_ST_USTRING) { 227 dest += s3->size; 228 val += s3->size; 229 goto stage3; 230 } 231 code--; 232 val = lval + sizeof(char *); 233 if (dest) { 234 dest += sizeof(u32); 235 *(u32 *)dest = update_data_loc(loc, ret); 236 } 237 goto stage2; 238 } 239 code++; 240 ret = total; 241 } 242 243 return code->op == FETCH_OP_END ? ret : -EILSEQ; 244 } 245 246 /* Sum up total data length for dynamic arrays (strings) */ 247 static nokprobe_inline int 248 __get_data_size(struct trace_probe *tp, void *regs, void *edata) 249 { 250 struct probe_arg *arg; 251 int i, len, ret = 0; 252 253 for (i = 0; i < tp->nr_args; i++) { 254 arg = tp->args + i; 255 if (unlikely(arg->dynamic)) { 256 len = process_fetch_insn(arg->code, regs, edata, NULL, NULL); 257 if (len > 0) 258 ret += len; 259 } 260 } 261 262 return ret; 263 } 264 265 /* Store the value of each argument */ 266 static nokprobe_inline void 267 store_trace_args(void *data, struct trace_probe *tp, void *rec, void *edata, 268 int header_size, int maxlen) 269 { 270 struct probe_arg *arg; 271 void *base = data - header_size; 272 void *dyndata = data + tp->size; 273 u32 *dl; /* Data location */ 274 int ret, i; 275 276 for (i = 0; i < tp->nr_args; i++) { 277 arg = tp->args + i; 278 dl = data + arg->offset; 279 /* Point the dynamic data area if needed */ 280 if (unlikely(arg->dynamic)) 281 *dl = make_data_loc(maxlen, dyndata - base); 282 ret = process_fetch_insn(arg->code, rec, edata, dl, base); 283 if (arg->dynamic && likely(ret > 0)) { 284 dyndata += ret; 285 maxlen -= ret; 286 } 287 } 288 } 289