1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Huawei Ltd. 4 * Author: Jiang Liu <liuj97@gmail.com> 5 * 6 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com> 7 */ 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/bug.h> 11 #include <linux/printk.h> 12 #include <linux/sizes.h> 13 #include <linux/types.h> 14 15 #include <asm/debug-monitors.h> 16 #include <asm/errno.h> 17 #include <asm/insn.h> 18 #include <asm/kprobes.h> 19 20 #define AARCH64_INSN_SF_BIT BIT(31) 21 #define AARCH64_INSN_N_BIT BIT(22) 22 #define AARCH64_INSN_LSL_12 BIT(22) 23 24 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, 25 u32 *maskp, int *shiftp) 26 { 27 u32 mask; 28 int shift; 29 30 switch (type) { 31 case AARCH64_INSN_IMM_26: 32 mask = BIT(26) - 1; 33 shift = 0; 34 break; 35 case AARCH64_INSN_IMM_19: 36 mask = BIT(19) - 1; 37 shift = 5; 38 break; 39 case AARCH64_INSN_IMM_16: 40 mask = BIT(16) - 1; 41 shift = 5; 42 break; 43 case AARCH64_INSN_IMM_14: 44 mask = BIT(14) - 1; 45 shift = 5; 46 break; 47 case AARCH64_INSN_IMM_12: 48 mask = BIT(12) - 1; 49 shift = 10; 50 break; 51 case AARCH64_INSN_IMM_9: 52 mask = BIT(9) - 1; 53 shift = 12; 54 break; 55 case AARCH64_INSN_IMM_7: 56 mask = BIT(7) - 1; 57 shift = 15; 58 break; 59 case AARCH64_INSN_IMM_6: 60 case AARCH64_INSN_IMM_S: 61 mask = BIT(6) - 1; 62 shift = 10; 63 break; 64 case AARCH64_INSN_IMM_R: 65 mask = BIT(6) - 1; 66 shift = 16; 67 break; 68 case AARCH64_INSN_IMM_N: 69 mask = 1; 70 shift = 22; 71 break; 72 default: 73 return -EINVAL; 74 } 75 76 *maskp = mask; 77 *shiftp = shift; 78 79 return 0; 80 } 81 82 #define ADR_IMM_HILOSPLIT 2 83 #define ADR_IMM_SIZE SZ_2M 84 #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1) 85 #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1) 86 #define ADR_IMM_LOSHIFT 29 87 #define ADR_IMM_HISHIFT 5 88 89 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn) 90 { 91 u32 immlo, immhi, mask; 92 int shift; 93 94 switch (type) { 95 case AARCH64_INSN_IMM_ADR: 96 shift = 0; 97 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK; 98 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK; 99 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo; 100 mask = ADR_IMM_SIZE - 1; 101 break; 102 default: 103 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { 104 pr_err("%s: unknown immediate encoding %d\n", __func__, 105 type); 106 return 0; 107 } 108 } 109 110 return (insn >> shift) & mask; 111 } 112 113 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, 114 u32 insn, u64 imm) 115 { 116 u32 immlo, immhi, mask; 117 int shift; 118 119 if (insn == AARCH64_BREAK_FAULT) 120 return AARCH64_BREAK_FAULT; 121 122 switch (type) { 123 case AARCH64_INSN_IMM_ADR: 124 shift = 0; 125 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT; 126 imm >>= ADR_IMM_HILOSPLIT; 127 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT; 128 imm = immlo | immhi; 129 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) | 130 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT)); 131 break; 132 default: 133 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { 134 pr_err("%s: unknown immediate encoding %d\n", __func__, 135 type); 136 return AARCH64_BREAK_FAULT; 137 } 138 } 139 140 /* Update the immediate field. */ 141 insn &= ~(mask << shift); 142 insn |= (imm & mask) << shift; 143 144 return insn; 145 } 146 147 u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type, 148 u32 insn) 149 { 150 int shift; 151 152 switch (type) { 153 case AARCH64_INSN_REGTYPE_RT: 154 case AARCH64_INSN_REGTYPE_RD: 155 shift = 0; 156 break; 157 case AARCH64_INSN_REGTYPE_RN: 158 shift = 5; 159 break; 160 case AARCH64_INSN_REGTYPE_RT2: 161 case AARCH64_INSN_REGTYPE_RA: 162 shift = 10; 163 break; 164 case AARCH64_INSN_REGTYPE_RM: 165 shift = 16; 166 break; 167 default: 168 pr_err("%s: unknown register type encoding %d\n", __func__, 169 type); 170 return 0; 171 } 172 173 return (insn >> shift) & GENMASK(4, 0); 174 } 175 176 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type, 177 u32 insn, 178 enum aarch64_insn_register reg) 179 { 180 int shift; 181 182 if (insn == AARCH64_BREAK_FAULT) 183 return AARCH64_BREAK_FAULT; 184 185 if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) { 186 pr_err("%s: unknown register encoding %d\n", __func__, reg); 187 return AARCH64_BREAK_FAULT; 188 } 189 190 switch (type) { 191 case AARCH64_INSN_REGTYPE_RT: 192 case AARCH64_INSN_REGTYPE_RD: 193 shift = 0; 194 break; 195 case AARCH64_INSN_REGTYPE_RN: 196 shift = 5; 197 break; 198 case AARCH64_INSN_REGTYPE_RT2: 199 case AARCH64_INSN_REGTYPE_RA: 200 shift = 10; 201 break; 202 case AARCH64_INSN_REGTYPE_RM: 203 case AARCH64_INSN_REGTYPE_RS: 204 shift = 16; 205 break; 206 default: 207 pr_err("%s: unknown register type encoding %d\n", __func__, 208 type); 209 return AARCH64_BREAK_FAULT; 210 } 211 212 insn &= ~(GENMASK(4, 0) << shift); 213 insn |= reg << shift; 214 215 return insn; 216 } 217 218 static const u32 aarch64_insn_ldst_size[] = { 219 [AARCH64_INSN_SIZE_8] = 0, 220 [AARCH64_INSN_SIZE_16] = 1, 221 [AARCH64_INSN_SIZE_32] = 2, 222 [AARCH64_INSN_SIZE_64] = 3, 223 }; 224 225 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type, 226 u32 insn) 227 { 228 u32 size; 229 230 if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) { 231 pr_err("%s: unknown size encoding %d\n", __func__, type); 232 return AARCH64_BREAK_FAULT; 233 } 234 235 size = aarch64_insn_ldst_size[type]; 236 insn &= ~GENMASK(31, 30); 237 insn |= size << 30; 238 239 return insn; 240 } 241 242 static inline long label_imm_common(unsigned long pc, unsigned long addr, 243 long range) 244 { 245 long offset; 246 247 if ((pc & 0x3) || (addr & 0x3)) { 248 pr_err("%s: A64 instructions must be word aligned\n", __func__); 249 return range; 250 } 251 252 offset = ((long)addr - (long)pc); 253 254 if (offset < -range || offset >= range) { 255 pr_err("%s: offset out of range\n", __func__); 256 return range; 257 } 258 259 return offset; 260 } 261 262 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, 263 enum aarch64_insn_branch_type type) 264 { 265 u32 insn; 266 long offset; 267 268 /* 269 * B/BL support [-128M, 128M) offset 270 * ARM64 virtual address arrangement guarantees all kernel and module 271 * texts are within +/-128M. 272 */ 273 offset = label_imm_common(pc, addr, SZ_128M); 274 if (offset >= SZ_128M) 275 return AARCH64_BREAK_FAULT; 276 277 switch (type) { 278 case AARCH64_INSN_BRANCH_LINK: 279 insn = aarch64_insn_get_bl_value(); 280 break; 281 case AARCH64_INSN_BRANCH_NOLINK: 282 insn = aarch64_insn_get_b_value(); 283 break; 284 default: 285 pr_err("%s: unknown branch encoding %d\n", __func__, type); 286 return AARCH64_BREAK_FAULT; 287 } 288 289 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, 290 offset >> 2); 291 } 292 293 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr, 294 enum aarch64_insn_register reg, 295 enum aarch64_insn_variant variant, 296 enum aarch64_insn_branch_type type) 297 { 298 u32 insn; 299 long offset; 300 301 offset = label_imm_common(pc, addr, SZ_1M); 302 if (offset >= SZ_1M) 303 return AARCH64_BREAK_FAULT; 304 305 switch (type) { 306 case AARCH64_INSN_BRANCH_COMP_ZERO: 307 insn = aarch64_insn_get_cbz_value(); 308 break; 309 case AARCH64_INSN_BRANCH_COMP_NONZERO: 310 insn = aarch64_insn_get_cbnz_value(); 311 break; 312 default: 313 pr_err("%s: unknown branch encoding %d\n", __func__, type); 314 return AARCH64_BREAK_FAULT; 315 } 316 317 switch (variant) { 318 case AARCH64_INSN_VARIANT_32BIT: 319 break; 320 case AARCH64_INSN_VARIANT_64BIT: 321 insn |= AARCH64_INSN_SF_BIT; 322 break; 323 default: 324 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 325 return AARCH64_BREAK_FAULT; 326 } 327 328 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 329 330 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 331 offset >> 2); 332 } 333 334 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, 335 enum aarch64_insn_condition cond) 336 { 337 u32 insn; 338 long offset; 339 340 offset = label_imm_common(pc, addr, SZ_1M); 341 if (offset >= SZ_1M) 342 return AARCH64_BREAK_FAULT; 343 344 insn = aarch64_insn_get_bcond_value(); 345 346 if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) { 347 pr_err("%s: unknown condition encoding %d\n", __func__, cond); 348 return AARCH64_BREAK_FAULT; 349 } 350 insn |= cond; 351 352 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 353 offset >> 2); 354 } 355 356 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg, 357 enum aarch64_insn_branch_type type) 358 { 359 u32 insn; 360 361 switch (type) { 362 case AARCH64_INSN_BRANCH_NOLINK: 363 insn = aarch64_insn_get_br_value(); 364 break; 365 case AARCH64_INSN_BRANCH_LINK: 366 insn = aarch64_insn_get_blr_value(); 367 break; 368 case AARCH64_INSN_BRANCH_RETURN: 369 insn = aarch64_insn_get_ret_value(); 370 break; 371 default: 372 pr_err("%s: unknown branch encoding %d\n", __func__, type); 373 return AARCH64_BREAK_FAULT; 374 } 375 376 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg); 377 } 378 379 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg, 380 enum aarch64_insn_register base, 381 enum aarch64_insn_register offset, 382 enum aarch64_insn_size_type size, 383 enum aarch64_insn_ldst_type type) 384 { 385 u32 insn; 386 387 switch (type) { 388 case AARCH64_INSN_LDST_LOAD_REG_OFFSET: 389 insn = aarch64_insn_get_ldr_reg_value(); 390 break; 391 case AARCH64_INSN_LDST_SIGNED_LOAD_REG_OFFSET: 392 insn = aarch64_insn_get_signed_ldr_reg_value(); 393 break; 394 case AARCH64_INSN_LDST_STORE_REG_OFFSET: 395 insn = aarch64_insn_get_str_reg_value(); 396 break; 397 default: 398 pr_err("%s: unknown load/store encoding %d\n", __func__, type); 399 return AARCH64_BREAK_FAULT; 400 } 401 402 insn = aarch64_insn_encode_ldst_size(size, insn); 403 404 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 405 406 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 407 base); 408 409 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, 410 offset); 411 } 412 413 u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg, 414 enum aarch64_insn_register base, 415 unsigned int imm, 416 enum aarch64_insn_size_type size, 417 enum aarch64_insn_ldst_type type) 418 { 419 u32 insn; 420 u32 shift; 421 422 if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) { 423 pr_err("%s: unknown size encoding %d\n", __func__, type); 424 return AARCH64_BREAK_FAULT; 425 } 426 427 shift = aarch64_insn_ldst_size[size]; 428 if (imm & ~(BIT(12 + shift) - BIT(shift))) { 429 pr_err("%s: invalid imm: %d\n", __func__, imm); 430 return AARCH64_BREAK_FAULT; 431 } 432 433 imm >>= shift; 434 435 switch (type) { 436 case AARCH64_INSN_LDST_LOAD_IMM_OFFSET: 437 insn = aarch64_insn_get_ldr_imm_value(); 438 break; 439 case AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET: 440 insn = aarch64_insn_get_signed_load_imm_value(); 441 break; 442 case AARCH64_INSN_LDST_STORE_IMM_OFFSET: 443 insn = aarch64_insn_get_str_imm_value(); 444 break; 445 default: 446 pr_err("%s: unknown load/store encoding %d\n", __func__, type); 447 return AARCH64_BREAK_FAULT; 448 } 449 450 insn = aarch64_insn_encode_ldst_size(size, insn); 451 452 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 453 454 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 455 base); 456 457 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); 458 } 459 460 u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr, 461 enum aarch64_insn_register reg, 462 bool is64bit) 463 { 464 u32 insn; 465 long offset; 466 467 offset = label_imm_common(pc, addr, SZ_1M); 468 if (offset >= SZ_1M) 469 return AARCH64_BREAK_FAULT; 470 471 insn = aarch64_insn_get_ldr_lit_value(); 472 473 if (is64bit) 474 insn |= BIT(30); 475 476 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg); 477 478 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 479 offset >> 2); 480 } 481 482 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, 483 enum aarch64_insn_register reg2, 484 enum aarch64_insn_register base, 485 int offset, 486 enum aarch64_insn_variant variant, 487 enum aarch64_insn_ldst_type type) 488 { 489 u32 insn; 490 int shift; 491 492 switch (type) { 493 case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX: 494 insn = aarch64_insn_get_ldp_pre_value(); 495 break; 496 case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX: 497 insn = aarch64_insn_get_stp_pre_value(); 498 break; 499 case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX: 500 insn = aarch64_insn_get_ldp_post_value(); 501 break; 502 case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX: 503 insn = aarch64_insn_get_stp_post_value(); 504 break; 505 default: 506 pr_err("%s: unknown load/store encoding %d\n", __func__, type); 507 return AARCH64_BREAK_FAULT; 508 } 509 510 switch (variant) { 511 case AARCH64_INSN_VARIANT_32BIT: 512 if ((offset & 0x3) || (offset < -256) || (offset > 252)) { 513 pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n", 514 __func__, offset); 515 return AARCH64_BREAK_FAULT; 516 } 517 shift = 2; 518 break; 519 case AARCH64_INSN_VARIANT_64BIT: 520 if ((offset & 0x7) || (offset < -512) || (offset > 504)) { 521 pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n", 522 __func__, offset); 523 return AARCH64_BREAK_FAULT; 524 } 525 shift = 3; 526 insn |= AARCH64_INSN_SF_BIT; 527 break; 528 default: 529 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 530 return AARCH64_BREAK_FAULT; 531 } 532 533 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 534 reg1); 535 536 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn, 537 reg2); 538 539 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 540 base); 541 542 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn, 543 offset >> shift); 544 } 545 546 u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg, 547 enum aarch64_insn_register base, 548 enum aarch64_insn_size_type size, 549 enum aarch64_insn_ldst_type type) 550 { 551 u32 insn; 552 553 switch (type) { 554 case AARCH64_INSN_LDST_LOAD_ACQ: 555 insn = aarch64_insn_get_load_acq_value(); 556 break; 557 case AARCH64_INSN_LDST_STORE_REL: 558 insn = aarch64_insn_get_store_rel_value(); 559 break; 560 default: 561 pr_err("%s: unknown load-acquire/store-release encoding %d\n", 562 __func__, type); 563 return AARCH64_BREAK_FAULT; 564 } 565 566 insn = aarch64_insn_encode_ldst_size(size, insn); 567 568 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 569 reg); 570 571 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 572 base); 573 } 574 575 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg, 576 enum aarch64_insn_register base, 577 enum aarch64_insn_register state, 578 enum aarch64_insn_size_type size, 579 enum aarch64_insn_ldst_type type) 580 { 581 u32 insn; 582 583 switch (type) { 584 case AARCH64_INSN_LDST_LOAD_EX: 585 case AARCH64_INSN_LDST_LOAD_ACQ_EX: 586 insn = aarch64_insn_get_load_ex_value(); 587 if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX) 588 insn |= BIT(15); 589 break; 590 case AARCH64_INSN_LDST_STORE_EX: 591 case AARCH64_INSN_LDST_STORE_REL_EX: 592 insn = aarch64_insn_get_store_ex_value(); 593 if (type == AARCH64_INSN_LDST_STORE_REL_EX) 594 insn |= BIT(15); 595 break; 596 default: 597 pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type); 598 return AARCH64_BREAK_FAULT; 599 } 600 601 insn = aarch64_insn_encode_ldst_size(size, insn); 602 603 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 604 reg); 605 606 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 607 base); 608 609 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn, 610 AARCH64_INSN_REG_ZR); 611 612 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, 613 state); 614 } 615 616 static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type, 617 u32 insn) 618 { 619 u32 order; 620 621 switch (type) { 622 case AARCH64_INSN_MEM_ORDER_NONE: 623 order = 0; 624 break; 625 case AARCH64_INSN_MEM_ORDER_ACQ: 626 order = 2; 627 break; 628 case AARCH64_INSN_MEM_ORDER_REL: 629 order = 1; 630 break; 631 case AARCH64_INSN_MEM_ORDER_ACQREL: 632 order = 3; 633 break; 634 default: 635 pr_err("%s: unknown mem order %d\n", __func__, type); 636 return AARCH64_BREAK_FAULT; 637 } 638 639 insn &= ~GENMASK(23, 22); 640 insn |= order << 22; 641 642 return insn; 643 } 644 645 u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result, 646 enum aarch64_insn_register address, 647 enum aarch64_insn_register value, 648 enum aarch64_insn_size_type size, 649 enum aarch64_insn_mem_atomic_op op, 650 enum aarch64_insn_mem_order_type order) 651 { 652 u32 insn; 653 654 switch (op) { 655 case AARCH64_INSN_MEM_ATOMIC_ADD: 656 insn = aarch64_insn_get_ldadd_value(); 657 break; 658 case AARCH64_INSN_MEM_ATOMIC_CLR: 659 insn = aarch64_insn_get_ldclr_value(); 660 break; 661 case AARCH64_INSN_MEM_ATOMIC_EOR: 662 insn = aarch64_insn_get_ldeor_value(); 663 break; 664 case AARCH64_INSN_MEM_ATOMIC_SET: 665 insn = aarch64_insn_get_ldset_value(); 666 break; 667 case AARCH64_INSN_MEM_ATOMIC_SWP: 668 insn = aarch64_insn_get_swp_value(); 669 break; 670 default: 671 pr_err("%s: unimplemented mem atomic op %d\n", __func__, op); 672 return AARCH64_BREAK_FAULT; 673 } 674 675 switch (size) { 676 case AARCH64_INSN_SIZE_32: 677 case AARCH64_INSN_SIZE_64: 678 break; 679 default: 680 pr_err("%s: unimplemented size encoding %d\n", __func__, size); 681 return AARCH64_BREAK_FAULT; 682 } 683 684 insn = aarch64_insn_encode_ldst_size(size, insn); 685 686 insn = aarch64_insn_encode_ldst_order(order, insn); 687 688 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 689 result); 690 691 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 692 address); 693 694 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, 695 value); 696 } 697 698 static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type, 699 u32 insn) 700 { 701 u32 order; 702 703 switch (type) { 704 case AARCH64_INSN_MEM_ORDER_NONE: 705 order = 0; 706 break; 707 case AARCH64_INSN_MEM_ORDER_ACQ: 708 order = BIT(22); 709 break; 710 case AARCH64_INSN_MEM_ORDER_REL: 711 order = BIT(15); 712 break; 713 case AARCH64_INSN_MEM_ORDER_ACQREL: 714 order = BIT(15) | BIT(22); 715 break; 716 default: 717 pr_err("%s: unknown mem order %d\n", __func__, type); 718 return AARCH64_BREAK_FAULT; 719 } 720 721 insn &= ~(BIT(15) | BIT(22)); 722 insn |= order; 723 724 return insn; 725 } 726 727 u32 aarch64_insn_gen_cas(enum aarch64_insn_register result, 728 enum aarch64_insn_register address, 729 enum aarch64_insn_register value, 730 enum aarch64_insn_size_type size, 731 enum aarch64_insn_mem_order_type order) 732 { 733 u32 insn; 734 735 switch (size) { 736 case AARCH64_INSN_SIZE_32: 737 case AARCH64_INSN_SIZE_64: 738 break; 739 default: 740 pr_err("%s: unimplemented size encoding %d\n", __func__, size); 741 return AARCH64_BREAK_FAULT; 742 } 743 744 insn = aarch64_insn_get_cas_value(); 745 746 insn = aarch64_insn_encode_ldst_size(size, insn); 747 748 insn = aarch64_insn_encode_cas_order(order, insn); 749 750 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, 751 result); 752 753 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 754 address); 755 756 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn, 757 value); 758 } 759 760 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst, 761 enum aarch64_insn_register src, 762 int imm, enum aarch64_insn_variant variant, 763 enum aarch64_insn_adsb_type type) 764 { 765 u32 insn; 766 767 switch (type) { 768 case AARCH64_INSN_ADSB_ADD: 769 insn = aarch64_insn_get_add_imm_value(); 770 break; 771 case AARCH64_INSN_ADSB_SUB: 772 insn = aarch64_insn_get_sub_imm_value(); 773 break; 774 case AARCH64_INSN_ADSB_ADD_SETFLAGS: 775 insn = aarch64_insn_get_adds_imm_value(); 776 break; 777 case AARCH64_INSN_ADSB_SUB_SETFLAGS: 778 insn = aarch64_insn_get_subs_imm_value(); 779 break; 780 default: 781 pr_err("%s: unknown add/sub encoding %d\n", __func__, type); 782 return AARCH64_BREAK_FAULT; 783 } 784 785 switch (variant) { 786 case AARCH64_INSN_VARIANT_32BIT: 787 break; 788 case AARCH64_INSN_VARIANT_64BIT: 789 insn |= AARCH64_INSN_SF_BIT; 790 break; 791 default: 792 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 793 return AARCH64_BREAK_FAULT; 794 } 795 796 /* We can't encode more than a 24bit value (12bit + 12bit shift) */ 797 if (imm & ~(BIT(24) - 1)) 798 goto out; 799 800 /* If we have something in the top 12 bits... */ 801 if (imm & ~(SZ_4K - 1)) { 802 /* ... and in the low 12 bits -> error */ 803 if (imm & (SZ_4K - 1)) 804 goto out; 805 806 imm >>= 12; 807 insn |= AARCH64_INSN_LSL_12; 808 } 809 810 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 811 812 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 813 814 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm); 815 816 out: 817 pr_err("%s: invalid immediate encoding %d\n", __func__, imm); 818 return AARCH64_BREAK_FAULT; 819 } 820 821 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst, 822 enum aarch64_insn_register src, 823 int immr, int imms, 824 enum aarch64_insn_variant variant, 825 enum aarch64_insn_bitfield_type type) 826 { 827 u32 insn; 828 u32 mask; 829 830 switch (type) { 831 case AARCH64_INSN_BITFIELD_MOVE: 832 insn = aarch64_insn_get_bfm_value(); 833 break; 834 case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED: 835 insn = aarch64_insn_get_ubfm_value(); 836 break; 837 case AARCH64_INSN_BITFIELD_MOVE_SIGNED: 838 insn = aarch64_insn_get_sbfm_value(); 839 break; 840 default: 841 pr_err("%s: unknown bitfield encoding %d\n", __func__, type); 842 return AARCH64_BREAK_FAULT; 843 } 844 845 switch (variant) { 846 case AARCH64_INSN_VARIANT_32BIT: 847 mask = GENMASK(4, 0); 848 break; 849 case AARCH64_INSN_VARIANT_64BIT: 850 insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT; 851 mask = GENMASK(5, 0); 852 break; 853 default: 854 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 855 return AARCH64_BREAK_FAULT; 856 } 857 858 if (immr & ~mask) { 859 pr_err("%s: invalid immr encoding %d\n", __func__, immr); 860 return AARCH64_BREAK_FAULT; 861 } 862 if (imms & ~mask) { 863 pr_err("%s: invalid imms encoding %d\n", __func__, imms); 864 return AARCH64_BREAK_FAULT; 865 } 866 867 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 868 869 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 870 871 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr); 872 873 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms); 874 } 875 876 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst, 877 int imm, int shift, 878 enum aarch64_insn_variant variant, 879 enum aarch64_insn_movewide_type type) 880 { 881 u32 insn; 882 883 switch (type) { 884 case AARCH64_INSN_MOVEWIDE_ZERO: 885 insn = aarch64_insn_get_movz_value(); 886 break; 887 case AARCH64_INSN_MOVEWIDE_KEEP: 888 insn = aarch64_insn_get_movk_value(); 889 break; 890 case AARCH64_INSN_MOVEWIDE_INVERSE: 891 insn = aarch64_insn_get_movn_value(); 892 break; 893 default: 894 pr_err("%s: unknown movewide encoding %d\n", __func__, type); 895 return AARCH64_BREAK_FAULT; 896 } 897 898 if (imm & ~(SZ_64K - 1)) { 899 pr_err("%s: invalid immediate encoding %d\n", __func__, imm); 900 return AARCH64_BREAK_FAULT; 901 } 902 903 switch (variant) { 904 case AARCH64_INSN_VARIANT_32BIT: 905 if (shift != 0 && shift != 16) { 906 pr_err("%s: invalid shift encoding %d\n", __func__, 907 shift); 908 return AARCH64_BREAK_FAULT; 909 } 910 break; 911 case AARCH64_INSN_VARIANT_64BIT: 912 insn |= AARCH64_INSN_SF_BIT; 913 if (shift != 0 && shift != 16 && shift != 32 && shift != 48) { 914 pr_err("%s: invalid shift encoding %d\n", __func__, 915 shift); 916 return AARCH64_BREAK_FAULT; 917 } 918 break; 919 default: 920 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 921 return AARCH64_BREAK_FAULT; 922 } 923 924 insn |= (shift >> 4) << 21; 925 926 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 927 928 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm); 929 } 930 931 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst, 932 enum aarch64_insn_register src, 933 enum aarch64_insn_register reg, 934 int shift, 935 enum aarch64_insn_variant variant, 936 enum aarch64_insn_adsb_type type) 937 { 938 u32 insn; 939 940 switch (type) { 941 case AARCH64_INSN_ADSB_ADD: 942 insn = aarch64_insn_get_add_value(); 943 break; 944 case AARCH64_INSN_ADSB_SUB: 945 insn = aarch64_insn_get_sub_value(); 946 break; 947 case AARCH64_INSN_ADSB_ADD_SETFLAGS: 948 insn = aarch64_insn_get_adds_value(); 949 break; 950 case AARCH64_INSN_ADSB_SUB_SETFLAGS: 951 insn = aarch64_insn_get_subs_value(); 952 break; 953 default: 954 pr_err("%s: unknown add/sub encoding %d\n", __func__, type); 955 return AARCH64_BREAK_FAULT; 956 } 957 958 switch (variant) { 959 case AARCH64_INSN_VARIANT_32BIT: 960 if (shift & ~(SZ_32 - 1)) { 961 pr_err("%s: invalid shift encoding %d\n", __func__, 962 shift); 963 return AARCH64_BREAK_FAULT; 964 } 965 break; 966 case AARCH64_INSN_VARIANT_64BIT: 967 insn |= AARCH64_INSN_SF_BIT; 968 if (shift & ~(SZ_64 - 1)) { 969 pr_err("%s: invalid shift encoding %d\n", __func__, 970 shift); 971 return AARCH64_BREAK_FAULT; 972 } 973 break; 974 default: 975 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 976 return AARCH64_BREAK_FAULT; 977 } 978 979 980 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 981 982 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 983 984 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 985 986 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); 987 } 988 989 /* 990 * Unlike the shifted-register form, register 31 is not XZR everywhere here: 991 * it encodes SP for @src, and for @dst too unless @type sets the flags. Only 992 * @reg keeps the XZR meaning. 993 */ 994 u32 aarch64_insn_gen_add_sub_extended_reg(enum aarch64_insn_register dst, 995 enum aarch64_insn_register src, 996 enum aarch64_insn_register reg, 997 enum aarch64_insn_extend_type extend, 998 int shift, 999 enum aarch64_insn_variant variant, 1000 enum aarch64_insn_adsb_type type) 1001 { 1002 u32 insn; 1003 1004 switch (type) { 1005 case AARCH64_INSN_ADSB_ADD: 1006 insn = aarch64_insn_get_add_ext_value(); 1007 break; 1008 case AARCH64_INSN_ADSB_SUB: 1009 insn = aarch64_insn_get_sub_ext_value(); 1010 break; 1011 case AARCH64_INSN_ADSB_ADD_SETFLAGS: 1012 insn = aarch64_insn_get_adds_ext_value(); 1013 break; 1014 case AARCH64_INSN_ADSB_SUB_SETFLAGS: 1015 insn = aarch64_insn_get_subs_ext_value(); 1016 break; 1017 default: 1018 pr_err("%s: unknown add/sub encoding %d\n", __func__, type); 1019 return AARCH64_BREAK_FAULT; 1020 } 1021 1022 switch (variant) { 1023 case AARCH64_INSN_VARIANT_32BIT: 1024 break; 1025 case AARCH64_INSN_VARIANT_64BIT: 1026 insn |= AARCH64_INSN_SF_BIT; 1027 break; 1028 default: 1029 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1030 return AARCH64_BREAK_FAULT; 1031 } 1032 1033 if (shift < 0 || shift > 4) { 1034 pr_err("%s: invalid shift encoding %d\n", __func__, shift); 1035 return AARCH64_BREAK_FAULT; 1036 } 1037 1038 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1039 1040 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 1041 1042 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 1043 1044 /* option in bits [15:13] and imm3 in [12:10] together fill IMM_6 */ 1045 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, 1046 (extend << 3) | shift); 1047 } 1048 1049 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst, 1050 enum aarch64_insn_register src, 1051 enum aarch64_insn_variant variant, 1052 enum aarch64_insn_data1_type type) 1053 { 1054 u32 insn; 1055 1056 switch (type) { 1057 case AARCH64_INSN_DATA1_REVERSE_16: 1058 insn = aarch64_insn_get_rev16_value(); 1059 break; 1060 case AARCH64_INSN_DATA1_REVERSE_32: 1061 insn = aarch64_insn_get_rev32_value(); 1062 break; 1063 case AARCH64_INSN_DATA1_REVERSE_64: 1064 if (variant != AARCH64_INSN_VARIANT_64BIT) { 1065 pr_err("%s: invalid variant for reverse64 %d\n", 1066 __func__, variant); 1067 return AARCH64_BREAK_FAULT; 1068 } 1069 insn = aarch64_insn_get_rev64_value(); 1070 break; 1071 default: 1072 pr_err("%s: unknown data1 encoding %d\n", __func__, type); 1073 return AARCH64_BREAK_FAULT; 1074 } 1075 1076 switch (variant) { 1077 case AARCH64_INSN_VARIANT_32BIT: 1078 break; 1079 case AARCH64_INSN_VARIANT_64BIT: 1080 insn |= AARCH64_INSN_SF_BIT; 1081 break; 1082 default: 1083 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1084 return AARCH64_BREAK_FAULT; 1085 } 1086 1087 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1088 1089 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 1090 } 1091 1092 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst, 1093 enum aarch64_insn_register src, 1094 enum aarch64_insn_register reg, 1095 enum aarch64_insn_variant variant, 1096 enum aarch64_insn_data2_type type) 1097 { 1098 u32 insn; 1099 1100 switch (type) { 1101 case AARCH64_INSN_DATA2_UDIV: 1102 insn = aarch64_insn_get_udiv_value(); 1103 break; 1104 case AARCH64_INSN_DATA2_SDIV: 1105 insn = aarch64_insn_get_sdiv_value(); 1106 break; 1107 case AARCH64_INSN_DATA2_LSLV: 1108 insn = aarch64_insn_get_lslv_value(); 1109 break; 1110 case AARCH64_INSN_DATA2_LSRV: 1111 insn = aarch64_insn_get_lsrv_value(); 1112 break; 1113 case AARCH64_INSN_DATA2_ASRV: 1114 insn = aarch64_insn_get_asrv_value(); 1115 break; 1116 case AARCH64_INSN_DATA2_RORV: 1117 insn = aarch64_insn_get_rorv_value(); 1118 break; 1119 default: 1120 pr_err("%s: unknown data2 encoding %d\n", __func__, type); 1121 return AARCH64_BREAK_FAULT; 1122 } 1123 1124 switch (variant) { 1125 case AARCH64_INSN_VARIANT_32BIT: 1126 break; 1127 case AARCH64_INSN_VARIANT_64BIT: 1128 insn |= AARCH64_INSN_SF_BIT; 1129 break; 1130 default: 1131 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1132 return AARCH64_BREAK_FAULT; 1133 } 1134 1135 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1136 1137 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 1138 1139 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 1140 } 1141 1142 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst, 1143 enum aarch64_insn_register src, 1144 enum aarch64_insn_register reg1, 1145 enum aarch64_insn_register reg2, 1146 enum aarch64_insn_variant variant, 1147 enum aarch64_insn_data3_type type) 1148 { 1149 u32 insn; 1150 1151 switch (type) { 1152 case AARCH64_INSN_DATA3_MADD: 1153 insn = aarch64_insn_get_madd_value(); 1154 break; 1155 case AARCH64_INSN_DATA3_MSUB: 1156 insn = aarch64_insn_get_msub_value(); 1157 break; 1158 default: 1159 pr_err("%s: unknown data3 encoding %d\n", __func__, type); 1160 return AARCH64_BREAK_FAULT; 1161 } 1162 1163 switch (variant) { 1164 case AARCH64_INSN_VARIANT_32BIT: 1165 break; 1166 case AARCH64_INSN_VARIANT_64BIT: 1167 insn |= AARCH64_INSN_SF_BIT; 1168 break; 1169 default: 1170 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1171 return AARCH64_BREAK_FAULT; 1172 } 1173 1174 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1175 1176 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src); 1177 1178 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, 1179 reg1); 1180 1181 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, 1182 reg2); 1183 } 1184 1185 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, 1186 enum aarch64_insn_register src, 1187 enum aarch64_insn_register reg, 1188 int shift, 1189 enum aarch64_insn_variant variant, 1190 enum aarch64_insn_logic_type type) 1191 { 1192 u32 insn; 1193 1194 switch (type) { 1195 case AARCH64_INSN_LOGIC_AND: 1196 insn = aarch64_insn_get_and_value(); 1197 break; 1198 case AARCH64_INSN_LOGIC_BIC: 1199 insn = aarch64_insn_get_bic_value(); 1200 break; 1201 case AARCH64_INSN_LOGIC_ORR: 1202 insn = aarch64_insn_get_orr_value(); 1203 break; 1204 case AARCH64_INSN_LOGIC_ORN: 1205 insn = aarch64_insn_get_orn_value(); 1206 break; 1207 case AARCH64_INSN_LOGIC_EOR: 1208 insn = aarch64_insn_get_eor_value(); 1209 break; 1210 case AARCH64_INSN_LOGIC_EON: 1211 insn = aarch64_insn_get_eon_value(); 1212 break; 1213 case AARCH64_INSN_LOGIC_AND_SETFLAGS: 1214 insn = aarch64_insn_get_ands_value(); 1215 break; 1216 case AARCH64_INSN_LOGIC_BIC_SETFLAGS: 1217 insn = aarch64_insn_get_bics_value(); 1218 break; 1219 default: 1220 pr_err("%s: unknown logical encoding %d\n", __func__, type); 1221 return AARCH64_BREAK_FAULT; 1222 } 1223 1224 switch (variant) { 1225 case AARCH64_INSN_VARIANT_32BIT: 1226 if (shift & ~(SZ_32 - 1)) { 1227 pr_err("%s: invalid shift encoding %d\n", __func__, 1228 shift); 1229 return AARCH64_BREAK_FAULT; 1230 } 1231 break; 1232 case AARCH64_INSN_VARIANT_64BIT: 1233 insn |= AARCH64_INSN_SF_BIT; 1234 if (shift & ~(SZ_64 - 1)) { 1235 pr_err("%s: invalid shift encoding %d\n", __func__, 1236 shift); 1237 return AARCH64_BREAK_FAULT; 1238 } 1239 break; 1240 default: 1241 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1242 return AARCH64_BREAK_FAULT; 1243 } 1244 1245 1246 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst); 1247 1248 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src); 1249 1250 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg); 1251 1252 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); 1253 } 1254 1255 /* 1256 * MOV (register) is architecturally an alias of ORR (shifted register) where 1257 * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m> 1258 */ 1259 u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst, 1260 enum aarch64_insn_register src, 1261 enum aarch64_insn_variant variant) 1262 { 1263 return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR, 1264 src, 0, variant, 1265 AARCH64_INSN_LOGIC_ORR); 1266 } 1267 1268 u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr, 1269 enum aarch64_insn_register reg, 1270 enum aarch64_insn_adr_type type) 1271 { 1272 u32 insn; 1273 s32 offset; 1274 1275 switch (type) { 1276 case AARCH64_INSN_ADR_TYPE_ADR: 1277 insn = aarch64_insn_get_adr_value(); 1278 offset = addr - pc; 1279 break; 1280 case AARCH64_INSN_ADR_TYPE_ADRP: 1281 insn = aarch64_insn_get_adrp_value(); 1282 offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12; 1283 break; 1284 default: 1285 pr_err("%s: unknown adr encoding %d\n", __func__, type); 1286 return AARCH64_BREAK_FAULT; 1287 } 1288 1289 if (offset < -SZ_1M || offset >= SZ_1M) 1290 return AARCH64_BREAK_FAULT; 1291 1292 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg); 1293 1294 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset); 1295 } 1296 1297 /* 1298 * Decode the imm field of a branch, and return the byte offset as a 1299 * signed value (so it can be used when computing a new branch 1300 * target). 1301 */ 1302 s32 aarch64_get_branch_offset(u32 insn) 1303 { 1304 s32 imm; 1305 1306 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { 1307 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn); 1308 return (imm << 6) >> 4; 1309 } 1310 1311 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || 1312 aarch64_insn_is_bcond(insn)) { 1313 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn); 1314 return (imm << 13) >> 11; 1315 } 1316 1317 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) { 1318 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn); 1319 return (imm << 18) >> 16; 1320 } 1321 1322 /* Unhandled instruction */ 1323 BUG(); 1324 } 1325 1326 /* 1327 * Encode the displacement of a branch in the imm field and return the 1328 * updated instruction. 1329 */ 1330 u32 aarch64_set_branch_offset(u32 insn, s32 offset) 1331 { 1332 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) 1333 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, 1334 offset >> 2); 1335 1336 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || 1337 aarch64_insn_is_bcond(insn)) 1338 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, 1339 offset >> 2); 1340 1341 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) 1342 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn, 1343 offset >> 2); 1344 1345 /* Unhandled instruction */ 1346 BUG(); 1347 } 1348 1349 s32 aarch64_insn_adrp_get_offset(u32 insn) 1350 { 1351 BUG_ON(!aarch64_insn_is_adrp(insn)); 1352 return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12; 1353 } 1354 1355 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset) 1356 { 1357 BUG_ON(!aarch64_insn_is_adrp(insn)); 1358 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, 1359 offset >> 12); 1360 } 1361 1362 /* 1363 * Extract the Op/CR data from a msr/mrs instruction. 1364 */ 1365 u32 aarch64_insn_extract_system_reg(u32 insn) 1366 { 1367 return (insn & 0x1FFFE0) >> 5; 1368 } 1369 1370 bool aarch32_insn_is_wide(u32 insn) 1371 { 1372 return insn >= 0xe800; 1373 } 1374 1375 /* 1376 * Macros/defines for extracting register numbers from instruction. 1377 */ 1378 u32 aarch32_insn_extract_reg_num(u32 insn, int offset) 1379 { 1380 return (insn & (0xf << offset)) >> offset; 1381 } 1382 1383 #define OPC2_MASK 0x7 1384 #define OPC2_OFFSET 5 1385 u32 aarch32_insn_mcr_extract_opc2(u32 insn) 1386 { 1387 return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET; 1388 } 1389 1390 #define CRM_MASK 0xf 1391 u32 aarch32_insn_mcr_extract_crm(u32 insn) 1392 { 1393 return insn & CRM_MASK; 1394 } 1395 1396 static bool range_of_ones(u64 val) 1397 { 1398 /* Doesn't handle full ones or full zeroes */ 1399 u64 sval = val >> __ffs64(val); 1400 1401 /* One of Sean Eron Anderson's bithack tricks */ 1402 return ((sval + 1) & (sval)) == 0; 1403 } 1404 1405 static u32 aarch64_encode_immediate(u64 imm, 1406 enum aarch64_insn_variant variant, 1407 u32 insn) 1408 { 1409 unsigned int immr, imms, n, ones, ror, esz, tmp; 1410 u64 mask; 1411 1412 switch (variant) { 1413 case AARCH64_INSN_VARIANT_32BIT: 1414 esz = 32; 1415 break; 1416 case AARCH64_INSN_VARIANT_64BIT: 1417 insn |= AARCH64_INSN_SF_BIT; 1418 esz = 64; 1419 break; 1420 default: 1421 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1422 return AARCH64_BREAK_FAULT; 1423 } 1424 1425 mask = GENMASK(esz - 1, 0); 1426 1427 /* Can't encode full zeroes, full ones, or value wider than the mask */ 1428 if (!imm || imm == mask || imm & ~mask) 1429 return AARCH64_BREAK_FAULT; 1430 1431 /* 1432 * Inverse of Replicate(). Try to spot a repeating pattern 1433 * with a pow2 stride. 1434 */ 1435 for (tmp = esz / 2; tmp >= 2; tmp /= 2) { 1436 u64 emask = BIT(tmp) - 1; 1437 1438 if ((imm & emask) != ((imm >> tmp) & emask)) 1439 break; 1440 1441 esz = tmp; 1442 mask = emask; 1443 } 1444 1445 /* N is only set if we're encoding a 64bit value */ 1446 n = esz == 64; 1447 1448 /* Trim imm to the element size */ 1449 imm &= mask; 1450 1451 /* That's how many ones we need to encode */ 1452 ones = hweight64(imm); 1453 1454 /* 1455 * imms is set to (ones - 1), prefixed with a string of ones 1456 * and a zero if they fit. Cap it to 6 bits. 1457 */ 1458 imms = ones - 1; 1459 imms |= 0xf << ffs(esz); 1460 imms &= BIT(6) - 1; 1461 1462 /* Compute the rotation */ 1463 if (range_of_ones(imm)) { 1464 /* 1465 * Pattern: 0..01..10..0 1466 * 1467 * Compute how many rotate we need to align it right 1468 */ 1469 ror = __ffs64(imm); 1470 } else { 1471 /* 1472 * Pattern: 0..01..10..01..1 1473 * 1474 * Fill the unused top bits with ones, and check if 1475 * the result is a valid immediate (all ones with a 1476 * contiguous ranges of zeroes). 1477 */ 1478 imm |= ~mask; 1479 if (!range_of_ones(~imm)) 1480 return AARCH64_BREAK_FAULT; 1481 1482 /* 1483 * Compute the rotation to get a continuous set of 1484 * ones, with the first bit set at position 0 1485 */ 1486 ror = fls64(~imm); 1487 } 1488 1489 /* 1490 * immr is the number of bits we need to rotate back to the 1491 * original set of ones. Note that this is relative to the 1492 * element size... 1493 */ 1494 immr = (esz - ror) % esz; 1495 1496 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n); 1497 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr); 1498 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms); 1499 } 1500 1501 u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type, 1502 enum aarch64_insn_variant variant, 1503 enum aarch64_insn_register Rn, 1504 enum aarch64_insn_register Rd, 1505 u64 imm) 1506 { 1507 u32 insn; 1508 1509 switch (type) { 1510 case AARCH64_INSN_LOGIC_AND: 1511 insn = aarch64_insn_get_and_imm_value(); 1512 break; 1513 case AARCH64_INSN_LOGIC_ORR: 1514 insn = aarch64_insn_get_orr_imm_value(); 1515 break; 1516 case AARCH64_INSN_LOGIC_EOR: 1517 insn = aarch64_insn_get_eor_imm_value(); 1518 break; 1519 case AARCH64_INSN_LOGIC_AND_SETFLAGS: 1520 insn = aarch64_insn_get_ands_imm_value(); 1521 break; 1522 default: 1523 pr_err("%s: unknown logical encoding %d\n", __func__, type); 1524 return AARCH64_BREAK_FAULT; 1525 } 1526 1527 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd); 1528 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn); 1529 return aarch64_encode_immediate(imm, variant, insn); 1530 } 1531 1532 u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant, 1533 enum aarch64_insn_register Rm, 1534 enum aarch64_insn_register Rn, 1535 enum aarch64_insn_register Rd, 1536 u8 lsb) 1537 { 1538 u32 insn; 1539 1540 insn = aarch64_insn_get_extr_value(); 1541 1542 switch (variant) { 1543 case AARCH64_INSN_VARIANT_32BIT: 1544 if (lsb > 31) 1545 return AARCH64_BREAK_FAULT; 1546 break; 1547 case AARCH64_INSN_VARIANT_64BIT: 1548 if (lsb > 63) 1549 return AARCH64_BREAK_FAULT; 1550 insn |= AARCH64_INSN_SF_BIT; 1551 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1); 1552 break; 1553 default: 1554 pr_err("%s: unknown variant encoding %d\n", __func__, variant); 1555 return AARCH64_BREAK_FAULT; 1556 } 1557 1558 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb); 1559 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd); 1560 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn); 1561 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm); 1562 } 1563 1564 static u32 __get_barrier_crm_val(enum aarch64_insn_mb_type type) 1565 { 1566 switch (type) { 1567 case AARCH64_INSN_MB_SY: 1568 return 0xf; 1569 case AARCH64_INSN_MB_ST: 1570 return 0xe; 1571 case AARCH64_INSN_MB_LD: 1572 return 0xd; 1573 case AARCH64_INSN_MB_ISH: 1574 return 0xb; 1575 case AARCH64_INSN_MB_ISHST: 1576 return 0xa; 1577 case AARCH64_INSN_MB_ISHLD: 1578 return 0x9; 1579 case AARCH64_INSN_MB_NSH: 1580 return 0x7; 1581 case AARCH64_INSN_MB_NSHST: 1582 return 0x6; 1583 case AARCH64_INSN_MB_NSHLD: 1584 return 0x5; 1585 default: 1586 pr_err("%s: unknown barrier type %d\n", __func__, type); 1587 return AARCH64_BREAK_FAULT; 1588 } 1589 } 1590 1591 u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type) 1592 { 1593 u32 opt; 1594 u32 insn; 1595 1596 opt = __get_barrier_crm_val(type); 1597 if (opt == AARCH64_BREAK_FAULT) 1598 return AARCH64_BREAK_FAULT; 1599 1600 insn = aarch64_insn_get_dmb_value(); 1601 insn &= ~GENMASK(11, 8); 1602 insn |= (opt << 8); 1603 1604 return insn; 1605 } 1606 1607 u32 aarch64_insn_gen_dsb(enum aarch64_insn_mb_type type) 1608 { 1609 u32 opt, insn; 1610 1611 opt = __get_barrier_crm_val(type); 1612 if (opt == AARCH64_BREAK_FAULT) 1613 return AARCH64_BREAK_FAULT; 1614 1615 insn = aarch64_insn_get_dsb_base_value(); 1616 insn &= ~GENMASK(11, 8); 1617 insn |= (opt << 8); 1618 1619 return insn; 1620 } 1621 1622 u32 aarch64_insn_gen_mrs(enum aarch64_insn_register result, 1623 enum aarch64_insn_system_register sysreg) 1624 { 1625 u32 insn = aarch64_insn_get_mrs_value(); 1626 1627 insn &= ~GENMASK(19, 0); 1628 insn |= sysreg << 5; 1629 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, 1630 insn, result); 1631 } 1632