1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #include <linux/sizes.h>
6 #include <linux/uaccess.h>
7 #include <linux/set_memory.h>
8 #include <linux/stop_machine.h>
9
10 #include <asm/cacheflush.h>
11 #include <asm/inst.h>
12
13 static DEFINE_RAW_SPINLOCK(patch_lock);
14
simu_pc(struct pt_regs * regs,union loongarch_instruction insn)15 void simu_pc(struct pt_regs *regs, union loongarch_instruction insn)
16 {
17 unsigned long pc = regs->csr_era;
18 unsigned int rd = insn.reg1i20_format.rd;
19 unsigned int imm = insn.reg1i20_format.immediate;
20
21 if (pc & 3) {
22 pr_warn("%s: invalid pc 0x%lx\n", __func__, pc);
23 return;
24 }
25
26 switch (insn.reg1i20_format.opcode) {
27 case pcaddi_op:
28 regs->regs[rd] = pc + sign_extend64(imm << 2, 21);
29 break;
30 case pcaddu12i_op:
31 regs->regs[rd] = pc + sign_extend64(imm << 12, 31);
32 break;
33 case pcaddu18i_op:
34 regs->regs[rd] = pc + sign_extend64(imm << 18, 37);
35 break;
36 case pcalau12i_op:
37 regs->regs[rd] = pc + sign_extend64(imm << 12, 31);
38 regs->regs[rd] &= ~((1 << 12) - 1);
39 break;
40 default:
41 pr_info("%s: unknown opcode\n", __func__);
42 return;
43 }
44
45 regs->csr_era += LOONGARCH_INSN_SIZE;
46 }
47
simu_branch(struct pt_regs * regs,union loongarch_instruction insn)48 void simu_branch(struct pt_regs *regs, union loongarch_instruction insn)
49 {
50 unsigned int imm, imm_l, imm_h, rd, rj;
51 unsigned long pc = regs->csr_era;
52
53 if (pc & 3) {
54 pr_warn("%s: invalid pc 0x%lx\n", __func__, pc);
55 return;
56 }
57
58 imm_l = insn.reg0i26_format.immediate_l;
59 imm_h = insn.reg0i26_format.immediate_h;
60 switch (insn.reg0i26_format.opcode) {
61 case b_op:
62 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27);
63 return;
64 case bl_op:
65 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 27);
66 regs->regs[1] = pc + LOONGARCH_INSN_SIZE;
67 return;
68 }
69
70 imm_l = insn.reg1i21_format.immediate_l;
71 imm_h = insn.reg1i21_format.immediate_h;
72 rj = insn.reg1i21_format.rj;
73 switch (insn.reg1i21_format.opcode) {
74 case beqz_op:
75 if (regs->regs[rj] == 0)
76 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22);
77 else
78 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
79 return;
80 case bnez_op:
81 if (regs->regs[rj] != 0)
82 regs->csr_era = pc + sign_extend64((imm_h << 16 | imm_l) << 2, 22);
83 else
84 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
85 return;
86 }
87
88 imm = insn.reg2i16_format.immediate;
89 rj = insn.reg2i16_format.rj;
90 rd = insn.reg2i16_format.rd;
91 switch (insn.reg2i16_format.opcode) {
92 case beq_op:
93 if (regs->regs[rj] == regs->regs[rd])
94 regs->csr_era = pc + sign_extend64(imm << 2, 17);
95 else
96 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
97 break;
98 case bne_op:
99 if (regs->regs[rj] != regs->regs[rd])
100 regs->csr_era = pc + sign_extend64(imm << 2, 17);
101 else
102 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
103 break;
104 case blt_op:
105 if ((long)regs->regs[rj] < (long)regs->regs[rd])
106 regs->csr_era = pc + sign_extend64(imm << 2, 17);
107 else
108 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
109 break;
110 case bge_op:
111 if ((long)regs->regs[rj] >= (long)regs->regs[rd])
112 regs->csr_era = pc + sign_extend64(imm << 2, 17);
113 else
114 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
115 break;
116 case bltu_op:
117 if (regs->regs[rj] < regs->regs[rd])
118 regs->csr_era = pc + sign_extend64(imm << 2, 17);
119 else
120 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
121 break;
122 case bgeu_op:
123 if (regs->regs[rj] >= regs->regs[rd])
124 regs->csr_era = pc + sign_extend64(imm << 2, 17);
125 else
126 regs->csr_era = pc + LOONGARCH_INSN_SIZE;
127 break;
128 case jirl_op:
129 regs->csr_era = regs->regs[rj] + sign_extend64(imm << 2, 17);
130 regs->regs[rd] = pc + LOONGARCH_INSN_SIZE;
131 break;
132 default:
133 pr_info("%s: unknown opcode\n", __func__);
134 return;
135 }
136 }
137
insns_not_supported(union loongarch_instruction insn)138 bool insns_not_supported(union loongarch_instruction insn)
139 {
140 switch (insn.reg3_format.opcode) {
141 case amswapw_op ... ammindbdu_op:
142 pr_notice("atomic memory access instructions are not supported\n");
143 return true;
144 case scq_op:
145 pr_notice("sc.q instruction is not supported\n");
146 return true;
147 }
148
149 switch (insn.reg2i14_format.opcode) {
150 case llw_op:
151 case lld_op:
152 case scw_op:
153 case scd_op:
154 pr_notice("ll and sc instructions are not supported\n");
155 return true;
156 }
157
158 switch (insn.reg2_format.opcode) {
159 case llacqw_op:
160 case llacqd_op:
161 case screlw_op:
162 case screld_op:
163 pr_notice("llacq and screl instructions are not supported\n");
164 return true;
165 }
166
167 switch (insn.reg1i21_format.opcode) {
168 case bceqz_op:
169 pr_notice("bceqz and bcnez instructions are not supported\n");
170 return true;
171 }
172
173 return false;
174 }
175
insns_need_simulation(union loongarch_instruction insn)176 bool insns_need_simulation(union loongarch_instruction insn)
177 {
178 if (is_pc_ins(&insn))
179 return true;
180
181 if (is_branch_ins(&insn))
182 return true;
183
184 return false;
185 }
186
arch_simulate_insn(union loongarch_instruction insn,struct pt_regs * regs)187 void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs)
188 {
189 if (is_pc_ins(&insn))
190 simu_pc(regs, insn);
191 else if (is_branch_ins(&insn))
192 simu_branch(regs, insn);
193 }
194
larch_insn_read(void * addr,u32 * insnp)195 int larch_insn_read(void *addr, u32 *insnp)
196 {
197 int ret;
198 u32 val;
199
200 ret = copy_from_kernel_nofault(&val, addr, LOONGARCH_INSN_SIZE);
201 if (!ret)
202 *insnp = val;
203
204 return ret;
205 }
206
larch_insn_write(void * addr,u32 insn)207 int larch_insn_write(void *addr, u32 insn)
208 {
209 int ret;
210 unsigned long flags = 0;
211
212 raw_spin_lock_irqsave(&patch_lock, flags);
213 ret = copy_to_kernel_nofault(addr, &insn, LOONGARCH_INSN_SIZE);
214 raw_spin_unlock_irqrestore(&patch_lock, flags);
215
216 return ret;
217 }
218
larch_insn_patch_text(void * addr,u32 insn)219 int larch_insn_patch_text(void *addr, u32 insn)
220 {
221 int ret;
222 u32 *tp = addr;
223
224 if ((unsigned long)tp & 3)
225 return -EINVAL;
226
227 ret = larch_insn_write(tp, insn);
228 if (!ret)
229 flush_icache_range((unsigned long)tp,
230 (unsigned long)tp + LOONGARCH_INSN_SIZE);
231
232 return ret;
233 }
234
235 struct insn_copy {
236 void *dst;
237 void *src;
238 size_t len;
239 unsigned int cpu;
240 };
241
text_copy_cb(void * data)242 static int text_copy_cb(void *data)
243 {
244 int ret = 0;
245 struct insn_copy *copy = data;
246
247 if (smp_processor_id() == copy->cpu) {
248 ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len);
249 if (ret) {
250 pr_err("%s: operation failed\n", __func__);
251 return ret;
252 }
253 }
254
255 flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len);
256
257 return 0;
258 }
259
larch_insn_text_copy(void * dst,void * src,size_t len)260 int larch_insn_text_copy(void *dst, void *src, size_t len)
261 {
262 int ret = 0;
263 int err = 0;
264 size_t start, end;
265 struct insn_copy copy = {
266 .dst = dst,
267 .src = src,
268 .len = len,
269 .cpu = raw_smp_processor_id(),
270 };
271
272 /*
273 * Ensure copy.cpu won't be hot removed before stop_machine.
274 * If it is removed nobody will really update the text.
275 */
276 lockdep_assert_cpus_held();
277
278 start = round_down((size_t)dst, PAGE_SIZE);
279 end = round_up((size_t)dst + len, PAGE_SIZE);
280
281 err = set_memory_rw(start, (end - start) / PAGE_SIZE);
282 if (err) {
283 pr_info("%s: set_memory_rw() failed\n", __func__);
284 return err;
285 }
286
287 ret = stop_machine_cpuslocked(text_copy_cb, ©, cpu_online_mask);
288
289 err = set_memory_rox(start, (end - start) / PAGE_SIZE);
290 if (err) {
291 pr_info("%s: set_memory_rox() failed\n", __func__);
292 return err;
293 }
294
295 return ret;
296 }
297
larch_insn_gen_nop(void)298 u32 larch_insn_gen_nop(void)
299 {
300 return INSN_NOP;
301 }
302
larch_insn_gen_b(unsigned long pc,unsigned long dest)303 u32 larch_insn_gen_b(unsigned long pc, unsigned long dest)
304 {
305 long offset = dest - pc;
306 union loongarch_instruction insn;
307
308 if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) {
309 pr_warn("The generated b instruction is out of range.\n");
310 return INSN_BREAK;
311 }
312
313 emit_b(&insn, offset >> 2);
314
315 return insn.word;
316 }
317
larch_insn_gen_bl(unsigned long pc,unsigned long dest)318 u32 larch_insn_gen_bl(unsigned long pc, unsigned long dest)
319 {
320 long offset = dest - pc;
321 union loongarch_instruction insn;
322
323 if ((offset & 3) || offset < -SZ_128M || offset >= SZ_128M) {
324 pr_warn("The generated bl instruction is out of range.\n");
325 return INSN_BREAK;
326 }
327
328 emit_bl(&insn, offset >> 2);
329
330 return insn.word;
331 }
332
larch_insn_gen_break(int imm)333 u32 larch_insn_gen_break(int imm)
334 {
335 union loongarch_instruction insn;
336
337 if (imm < 0 || imm >= SZ_32K) {
338 pr_warn("The generated break instruction is out of range.\n");
339 return INSN_BREAK;
340 }
341
342 emit_break(&insn, imm);
343
344 return insn.word;
345 }
346
larch_insn_gen_or(enum loongarch_gpr rd,enum loongarch_gpr rj,enum loongarch_gpr rk)347 u32 larch_insn_gen_or(enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)
348 {
349 union loongarch_instruction insn;
350
351 emit_or(&insn, rd, rj, rk);
352
353 return insn.word;
354 }
355
larch_insn_gen_move(enum loongarch_gpr rd,enum loongarch_gpr rj)356 u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj)
357 {
358 return larch_insn_gen_or(rd, rj, 0);
359 }
360
larch_insn_gen_lu12iw(enum loongarch_gpr rd,int imm)361 u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm)
362 {
363 union loongarch_instruction insn;
364
365 if (imm < -SZ_512K || imm >= SZ_512K) {
366 pr_warn("The generated lu12i.w instruction is out of range.\n");
367 return INSN_BREAK;
368 }
369
370 emit_lu12iw(&insn, rd, imm);
371
372 return insn.word;
373 }
374
larch_insn_gen_lu32id(enum loongarch_gpr rd,int imm)375 u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm)
376 {
377 union loongarch_instruction insn;
378
379 if (imm < -SZ_512K || imm >= SZ_512K) {
380 pr_warn("The generated lu32i.d instruction is out of range.\n");
381 return INSN_BREAK;
382 }
383
384 emit_lu32id(&insn, rd, imm);
385
386 return insn.word;
387 }
388
larch_insn_gen_lu52id(enum loongarch_gpr rd,enum loongarch_gpr rj,int imm)389 u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
390 {
391 union loongarch_instruction insn;
392
393 if (imm < -SZ_2K || imm >= SZ_2K) {
394 pr_warn("The generated lu52i.d instruction is out of range.\n");
395 return INSN_BREAK;
396 }
397
398 emit_lu52id(&insn, rd, rj, imm);
399
400 return insn.word;
401 }
402
larch_insn_gen_beq(enum loongarch_gpr rd,enum loongarch_gpr rj,int imm)403 u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
404 {
405 union loongarch_instruction insn;
406
407 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
408 pr_warn("The generated beq instruction is out of range.\n");
409 return INSN_BREAK;
410 }
411
412 emit_beq(&insn, rj, rd, imm >> 2);
413
414 return insn.word;
415 }
416
larch_insn_gen_bne(enum loongarch_gpr rd,enum loongarch_gpr rj,int imm)417 u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
418 {
419 union loongarch_instruction insn;
420
421 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
422 pr_warn("The generated bne instruction is out of range.\n");
423 return INSN_BREAK;
424 }
425
426 emit_bne(&insn, rj, rd, imm >> 2);
427
428 return insn.word;
429 }
430
larch_insn_gen_jirl(enum loongarch_gpr rd,enum loongarch_gpr rj,int imm)431 u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
432 {
433 union loongarch_instruction insn;
434
435 if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) {
436 pr_warn("The generated jirl instruction is out of range.\n");
437 return INSN_BREAK;
438 }
439
440 emit_jirl(&insn, rd, rj, imm >> 2);
441
442 return insn.word;
443 }
444