1 //===- RISCV.cpp ----------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "InputFiles.h" 10 #include "OutputSections.h" 11 #include "Symbols.h" 12 #include "SyntheticSections.h" 13 #include "Target.h" 14 #include "llvm/Support/TimeProfiler.h" 15 16 using namespace llvm; 17 using namespace llvm::object; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 using namespace lld; 21 using namespace lld::elf; 22 23 namespace { 24 25 class RISCV final : public TargetInfo { 26 public: 27 RISCV(); 28 uint32_t calcEFlags() const override; 29 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; 30 void writeGotHeader(uint8_t *buf) const override; 31 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 32 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override; 33 void writePltHeader(uint8_t *buf) const override; 34 void writePlt(uint8_t *buf, const Symbol &sym, 35 uint64_t pltEntryAddr) const override; 36 RelType getDynRel(RelType type) const override; 37 RelExpr getRelExpr(RelType type, const Symbol &s, 38 const uint8_t *loc) const override; 39 void relocate(uint8_t *loc, const Relocation &rel, 40 uint64_t val) const override; 41 bool relaxOnce(int pass) const override; 42 }; 43 44 } // end anonymous namespace 45 46 const uint64_t dtpOffset = 0x800; 47 48 enum Op { 49 ADDI = 0x13, 50 AUIPC = 0x17, 51 JALR = 0x67, 52 LD = 0x3003, 53 LW = 0x2003, 54 SRLI = 0x5013, 55 SUB = 0x40000033, 56 }; 57 58 enum Reg { 59 X_RA = 1, 60 X_TP = 4, 61 X_T0 = 5, 62 X_T1 = 6, 63 X_T2 = 7, 64 X_T3 = 28, 65 }; 66 67 static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; } 68 static uint32_t lo12(uint32_t val) { return val & 4095; } 69 70 static uint32_t itype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t imm) { 71 return op | (rd << 7) | (rs1 << 15) | (imm << 20); 72 } 73 static uint32_t rtype(uint32_t op, uint32_t rd, uint32_t rs1, uint32_t rs2) { 74 return op | (rd << 7) | (rs1 << 15) | (rs2 << 20); 75 } 76 static uint32_t utype(uint32_t op, uint32_t rd, uint32_t imm) { 77 return op | (rd << 7) | (imm << 12); 78 } 79 80 // Extract bits v[begin:end], where range is inclusive, and begin must be < 63. 81 static uint32_t extractBits(uint64_t v, uint32_t begin, uint32_t end) { 82 return (v & ((1ULL << (begin + 1)) - 1)) >> end; 83 } 84 85 static uint32_t setLO12_I(uint32_t insn, uint32_t imm) { 86 return (insn & 0xfffff) | (imm << 20); 87 } 88 static uint32_t setLO12_S(uint32_t insn, uint32_t imm) { 89 return (insn & 0x1fff07f) | (extractBits(imm, 11, 5) << 25) | 90 (extractBits(imm, 4, 0) << 7); 91 } 92 93 RISCV::RISCV() { 94 copyRel = R_RISCV_COPY; 95 pltRel = R_RISCV_JUMP_SLOT; 96 relativeRel = R_RISCV_RELATIVE; 97 iRelativeRel = R_RISCV_IRELATIVE; 98 if (config->is64) { 99 symbolicRel = R_RISCV_64; 100 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD64; 101 tlsOffsetRel = R_RISCV_TLS_DTPREL64; 102 tlsGotRel = R_RISCV_TLS_TPREL64; 103 } else { 104 symbolicRel = R_RISCV_32; 105 tlsModuleIndexRel = R_RISCV_TLS_DTPMOD32; 106 tlsOffsetRel = R_RISCV_TLS_DTPREL32; 107 tlsGotRel = R_RISCV_TLS_TPREL32; 108 } 109 gotRel = symbolicRel; 110 111 // .got[0] = _DYNAMIC 112 gotHeaderEntriesNum = 1; 113 114 // .got.plt[0] = _dl_runtime_resolve, .got.plt[1] = link_map 115 gotPltHeaderEntriesNum = 2; 116 117 pltHeaderSize = 32; 118 pltEntrySize = 16; 119 ipltEntrySize = 16; 120 } 121 122 static uint32_t getEFlags(InputFile *f) { 123 if (config->is64) 124 return cast<ObjFile<ELF64LE>>(f)->getObj().getHeader().e_flags; 125 return cast<ObjFile<ELF32LE>>(f)->getObj().getHeader().e_flags; 126 } 127 128 uint32_t RISCV::calcEFlags() const { 129 // If there are only binary input files (from -b binary), use a 130 // value of 0 for the ELF header flags. 131 if (ctx->objectFiles.empty()) 132 return 0; 133 134 uint32_t target = getEFlags(ctx->objectFiles.front()); 135 136 for (InputFile *f : ctx->objectFiles) { 137 uint32_t eflags = getEFlags(f); 138 if (eflags & EF_RISCV_RVC) 139 target |= EF_RISCV_RVC; 140 141 if ((eflags & EF_RISCV_FLOAT_ABI) != (target & EF_RISCV_FLOAT_ABI)) 142 error(toString(f) + 143 ": cannot link object files with different floating-point ABI"); 144 145 if ((eflags & EF_RISCV_RVE) != (target & EF_RISCV_RVE)) 146 error(toString(f) + 147 ": cannot link object files with different EF_RISCV_RVE"); 148 } 149 150 return target; 151 } 152 153 int64_t RISCV::getImplicitAddend(const uint8_t *buf, RelType type) const { 154 switch (type) { 155 default: 156 internalLinkerError(getErrorLocation(buf), 157 "cannot read addend for relocation " + toString(type)); 158 return 0; 159 case R_RISCV_32: 160 case R_RISCV_TLS_DTPMOD32: 161 case R_RISCV_TLS_DTPREL32: 162 return SignExtend64<32>(read32le(buf)); 163 case R_RISCV_64: 164 return read64le(buf); 165 case R_RISCV_RELATIVE: 166 case R_RISCV_IRELATIVE: 167 return config->is64 ? read64le(buf) : read32le(buf); 168 case R_RISCV_NONE: 169 case R_RISCV_JUMP_SLOT: 170 // These relocations are defined as not having an implicit addend. 171 return 0; 172 } 173 } 174 175 void RISCV::writeGotHeader(uint8_t *buf) const { 176 if (config->is64) 177 write64le(buf, mainPart->dynamic->getVA()); 178 else 179 write32le(buf, mainPart->dynamic->getVA()); 180 } 181 182 void RISCV::writeGotPlt(uint8_t *buf, const Symbol &s) const { 183 if (config->is64) 184 write64le(buf, in.plt->getVA()); 185 else 186 write32le(buf, in.plt->getVA()); 187 } 188 189 void RISCV::writeIgotPlt(uint8_t *buf, const Symbol &s) const { 190 if (config->writeAddends) { 191 if (config->is64) 192 write64le(buf, s.getVA()); 193 else 194 write32le(buf, s.getVA()); 195 } 196 } 197 198 void RISCV::writePltHeader(uint8_t *buf) const { 199 // 1: auipc t2, %pcrel_hi(.got.plt) 200 // sub t1, t1, t3 201 // l[wd] t3, %pcrel_lo(1b)(t2); t3 = _dl_runtime_resolve 202 // addi t1, t1, -pltHeaderSize-12; t1 = &.plt[i] - &.plt[0] 203 // addi t0, t2, %pcrel_lo(1b) 204 // srli t1, t1, (rv64?1:2); t1 = &.got.plt[i] - &.got.plt[0] 205 // l[wd] t0, Wordsize(t0); t0 = link_map 206 // jr t3 207 uint32_t offset = in.gotPlt->getVA() - in.plt->getVA(); 208 uint32_t load = config->is64 ? LD : LW; 209 write32le(buf + 0, utype(AUIPC, X_T2, hi20(offset))); 210 write32le(buf + 4, rtype(SUB, X_T1, X_T1, X_T3)); 211 write32le(buf + 8, itype(load, X_T3, X_T2, lo12(offset))); 212 write32le(buf + 12, itype(ADDI, X_T1, X_T1, -target->pltHeaderSize - 12)); 213 write32le(buf + 16, itype(ADDI, X_T0, X_T2, lo12(offset))); 214 write32le(buf + 20, itype(SRLI, X_T1, X_T1, config->is64 ? 1 : 2)); 215 write32le(buf + 24, itype(load, X_T0, X_T0, config->wordsize)); 216 write32le(buf + 28, itype(JALR, 0, X_T3, 0)); 217 } 218 219 void RISCV::writePlt(uint8_t *buf, const Symbol &sym, 220 uint64_t pltEntryAddr) const { 221 // 1: auipc t3, %pcrel_hi(f@.got.plt) 222 // l[wd] t3, %pcrel_lo(1b)(t3) 223 // jalr t1, t3 224 // nop 225 uint32_t offset = sym.getGotPltVA() - pltEntryAddr; 226 write32le(buf + 0, utype(AUIPC, X_T3, hi20(offset))); 227 write32le(buf + 4, itype(config->is64 ? LD : LW, X_T3, X_T3, lo12(offset))); 228 write32le(buf + 8, itype(JALR, X_T1, X_T3, 0)); 229 write32le(buf + 12, itype(ADDI, 0, 0, 0)); 230 } 231 232 RelType RISCV::getDynRel(RelType type) const { 233 return type == target->symbolicRel ? type 234 : static_cast<RelType>(R_RISCV_NONE); 235 } 236 237 RelExpr RISCV::getRelExpr(const RelType type, const Symbol &s, 238 const uint8_t *loc) const { 239 switch (type) { 240 case R_RISCV_NONE: 241 return R_NONE; 242 case R_RISCV_32: 243 case R_RISCV_64: 244 case R_RISCV_HI20: 245 case R_RISCV_LO12_I: 246 case R_RISCV_LO12_S: 247 case R_RISCV_RVC_LUI: 248 return R_ABS; 249 case R_RISCV_ADD8: 250 case R_RISCV_ADD16: 251 case R_RISCV_ADD32: 252 case R_RISCV_ADD64: 253 case R_RISCV_SET6: 254 case R_RISCV_SET8: 255 case R_RISCV_SET16: 256 case R_RISCV_SET32: 257 case R_RISCV_SUB6: 258 case R_RISCV_SUB8: 259 case R_RISCV_SUB16: 260 case R_RISCV_SUB32: 261 case R_RISCV_SUB64: 262 return R_RISCV_ADD; 263 case R_RISCV_JAL: 264 case R_RISCV_BRANCH: 265 case R_RISCV_PCREL_HI20: 266 case R_RISCV_RVC_BRANCH: 267 case R_RISCV_RVC_JUMP: 268 case R_RISCV_32_PCREL: 269 return R_PC; 270 case R_RISCV_CALL: 271 case R_RISCV_CALL_PLT: 272 return R_PLT_PC; 273 case R_RISCV_GOT_HI20: 274 return R_GOT_PC; 275 case R_RISCV_PCREL_LO12_I: 276 case R_RISCV_PCREL_LO12_S: 277 return R_RISCV_PC_INDIRECT; 278 case R_RISCV_TLS_GD_HI20: 279 return R_TLSGD_PC; 280 case R_RISCV_TLS_GOT_HI20: 281 config->hasTlsIe = true; 282 return R_GOT_PC; 283 case R_RISCV_TPREL_HI20: 284 case R_RISCV_TPREL_LO12_I: 285 case R_RISCV_TPREL_LO12_S: 286 return R_TPREL; 287 case R_RISCV_ALIGN: 288 return R_RELAX_HINT; 289 case R_RISCV_TPREL_ADD: 290 case R_RISCV_RELAX: 291 return config->relax ? R_RELAX_HINT : R_NONE; 292 default: 293 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 294 ") against symbol " + toString(s)); 295 return R_NONE; 296 } 297 } 298 299 void RISCV::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { 300 const unsigned bits = config->wordsize * 8; 301 302 switch (rel.type) { 303 case R_RISCV_32: 304 write32le(loc, val); 305 return; 306 case R_RISCV_64: 307 write64le(loc, val); 308 return; 309 310 case R_RISCV_RVC_BRANCH: { 311 checkInt(loc, val, 9, rel); 312 checkAlignment(loc, val, 2, rel); 313 uint16_t insn = read16le(loc) & 0xE383; 314 uint16_t imm8 = extractBits(val, 8, 8) << 12; 315 uint16_t imm4_3 = extractBits(val, 4, 3) << 10; 316 uint16_t imm7_6 = extractBits(val, 7, 6) << 5; 317 uint16_t imm2_1 = extractBits(val, 2, 1) << 3; 318 uint16_t imm5 = extractBits(val, 5, 5) << 2; 319 insn |= imm8 | imm4_3 | imm7_6 | imm2_1 | imm5; 320 321 write16le(loc, insn); 322 return; 323 } 324 325 case R_RISCV_RVC_JUMP: { 326 checkInt(loc, val, 12, rel); 327 checkAlignment(loc, val, 2, rel); 328 uint16_t insn = read16le(loc) & 0xE003; 329 uint16_t imm11 = extractBits(val, 11, 11) << 12; 330 uint16_t imm4 = extractBits(val, 4, 4) << 11; 331 uint16_t imm9_8 = extractBits(val, 9, 8) << 9; 332 uint16_t imm10 = extractBits(val, 10, 10) << 8; 333 uint16_t imm6 = extractBits(val, 6, 6) << 7; 334 uint16_t imm7 = extractBits(val, 7, 7) << 6; 335 uint16_t imm3_1 = extractBits(val, 3, 1) << 3; 336 uint16_t imm5 = extractBits(val, 5, 5) << 2; 337 insn |= imm11 | imm4 | imm9_8 | imm10 | imm6 | imm7 | imm3_1 | imm5; 338 339 write16le(loc, insn); 340 return; 341 } 342 343 case R_RISCV_RVC_LUI: { 344 int64_t imm = SignExtend64(val + 0x800, bits) >> 12; 345 checkInt(loc, imm, 6, rel); 346 if (imm == 0) { // `c.lui rd, 0` is illegal, convert to `c.li rd, 0` 347 write16le(loc, (read16le(loc) & 0x0F83) | 0x4000); 348 } else { 349 uint16_t imm17 = extractBits(val + 0x800, 17, 17) << 12; 350 uint16_t imm16_12 = extractBits(val + 0x800, 16, 12) << 2; 351 write16le(loc, (read16le(loc) & 0xEF83) | imm17 | imm16_12); 352 } 353 return; 354 } 355 356 case R_RISCV_JAL: { 357 checkInt(loc, val, 21, rel); 358 checkAlignment(loc, val, 2, rel); 359 360 uint32_t insn = read32le(loc) & 0xFFF; 361 uint32_t imm20 = extractBits(val, 20, 20) << 31; 362 uint32_t imm10_1 = extractBits(val, 10, 1) << 21; 363 uint32_t imm11 = extractBits(val, 11, 11) << 20; 364 uint32_t imm19_12 = extractBits(val, 19, 12) << 12; 365 insn |= imm20 | imm10_1 | imm11 | imm19_12; 366 367 write32le(loc, insn); 368 return; 369 } 370 371 case R_RISCV_BRANCH: { 372 checkInt(loc, val, 13, rel); 373 checkAlignment(loc, val, 2, rel); 374 375 uint32_t insn = read32le(loc) & 0x1FFF07F; 376 uint32_t imm12 = extractBits(val, 12, 12) << 31; 377 uint32_t imm10_5 = extractBits(val, 10, 5) << 25; 378 uint32_t imm4_1 = extractBits(val, 4, 1) << 8; 379 uint32_t imm11 = extractBits(val, 11, 11) << 7; 380 insn |= imm12 | imm10_5 | imm4_1 | imm11; 381 382 write32le(loc, insn); 383 return; 384 } 385 386 // auipc + jalr pair 387 case R_RISCV_CALL: 388 case R_RISCV_CALL_PLT: { 389 int64_t hi = SignExtend64(val + 0x800, bits) >> 12; 390 checkInt(loc, hi, 20, rel); 391 if (isInt<20>(hi)) { 392 relocateNoSym(loc, R_RISCV_PCREL_HI20, val); 393 relocateNoSym(loc + 4, R_RISCV_PCREL_LO12_I, val); 394 } 395 return; 396 } 397 398 case R_RISCV_GOT_HI20: 399 case R_RISCV_PCREL_HI20: 400 case R_RISCV_TLS_GD_HI20: 401 case R_RISCV_TLS_GOT_HI20: 402 case R_RISCV_TPREL_HI20: 403 case R_RISCV_HI20: { 404 uint64_t hi = val + 0x800; 405 checkInt(loc, SignExtend64(hi, bits) >> 12, 20, rel); 406 write32le(loc, (read32le(loc) & 0xFFF) | (hi & 0xFFFFF000)); 407 return; 408 } 409 410 case R_RISCV_PCREL_LO12_I: 411 case R_RISCV_TPREL_LO12_I: 412 case R_RISCV_LO12_I: { 413 uint64_t hi = (val + 0x800) >> 12; 414 uint64_t lo = val - (hi << 12); 415 write32le(loc, setLO12_I(read32le(loc), lo & 0xfff)); 416 return; 417 } 418 419 case R_RISCV_PCREL_LO12_S: 420 case R_RISCV_TPREL_LO12_S: 421 case R_RISCV_LO12_S: { 422 uint64_t hi = (val + 0x800) >> 12; 423 uint64_t lo = val - (hi << 12); 424 write32le(loc, setLO12_S(read32le(loc), lo)); 425 return; 426 } 427 428 case R_RISCV_ADD8: 429 *loc += val; 430 return; 431 case R_RISCV_ADD16: 432 write16le(loc, read16le(loc) + val); 433 return; 434 case R_RISCV_ADD32: 435 write32le(loc, read32le(loc) + val); 436 return; 437 case R_RISCV_ADD64: 438 write64le(loc, read64le(loc) + val); 439 return; 440 case R_RISCV_SUB6: 441 *loc = (*loc & 0xc0) | (((*loc & 0x3f) - val) & 0x3f); 442 return; 443 case R_RISCV_SUB8: 444 *loc -= val; 445 return; 446 case R_RISCV_SUB16: 447 write16le(loc, read16le(loc) - val); 448 return; 449 case R_RISCV_SUB32: 450 write32le(loc, read32le(loc) - val); 451 return; 452 case R_RISCV_SUB64: 453 write64le(loc, read64le(loc) - val); 454 return; 455 case R_RISCV_SET6: 456 *loc = (*loc & 0xc0) | (val & 0x3f); 457 return; 458 case R_RISCV_SET8: 459 *loc = val; 460 return; 461 case R_RISCV_SET16: 462 write16le(loc, val); 463 return; 464 case R_RISCV_SET32: 465 case R_RISCV_32_PCREL: 466 write32le(loc, val); 467 return; 468 469 case R_RISCV_TLS_DTPREL32: 470 write32le(loc, val - dtpOffset); 471 break; 472 case R_RISCV_TLS_DTPREL64: 473 write64le(loc, val - dtpOffset); 474 break; 475 476 case R_RISCV_RELAX: 477 return; // Ignored (for now) 478 479 default: 480 llvm_unreachable("unknown relocation"); 481 } 482 } 483 484 namespace { 485 struct SymbolAnchor { 486 uint64_t offset; 487 Defined *d; 488 bool end; // true for the anchor of st_value+st_size 489 }; 490 } // namespace 491 492 struct elf::RISCVRelaxAux { 493 // This records symbol start and end offsets which will be adjusted according 494 // to the nearest relocDeltas element. 495 SmallVector<SymbolAnchor, 0> anchors; 496 // For relocations[i], the actual offset is r_offset - (i ? relocDeltas[i-1] : 497 // 0). 498 std::unique_ptr<uint32_t[]> relocDeltas; 499 // For relocations[i], the actual type is relocTypes[i]. 500 std::unique_ptr<RelType[]> relocTypes; 501 SmallVector<uint32_t, 0> writes; 502 }; 503 504 static void initSymbolAnchors() { 505 SmallVector<InputSection *, 0> storage; 506 for (OutputSection *osec : outputSections) { 507 if (!(osec->flags & SHF_EXECINSTR)) 508 continue; 509 for (InputSection *sec : getInputSections(*osec, storage)) { 510 sec->relaxAux = make<RISCVRelaxAux>(); 511 if (sec->relocations.size()) { 512 sec->relaxAux->relocDeltas = 513 std::make_unique<uint32_t[]>(sec->relocations.size()); 514 sec->relaxAux->relocTypes = 515 std::make_unique<RelType[]>(sec->relocations.size()); 516 } 517 } 518 } 519 // Store anchors (st_value and st_value+st_size) for symbols relative to text 520 // sections. 521 for (InputFile *file : ctx->objectFiles) 522 for (Symbol *sym : file->getSymbols()) { 523 auto *d = dyn_cast<Defined>(sym); 524 if (!d || d->file != file) 525 continue; 526 if (auto *sec = dyn_cast_or_null<InputSection>(d->section)) 527 if (sec->flags & SHF_EXECINSTR && sec->relaxAux) { 528 // If sec is discarded, relaxAux will be nullptr. 529 sec->relaxAux->anchors.push_back({d->value, d, false}); 530 sec->relaxAux->anchors.push_back({d->value + d->size, d, true}); 531 } 532 } 533 // Sort anchors by offset so that we can find the closest relocation 534 // efficiently. For a zero size symbol, ensure that its start anchor precedes 535 // its end anchor. For two symbols with anchors at the same offset, their 536 // order does not matter. 537 for (OutputSection *osec : outputSections) { 538 if (!(osec->flags & SHF_EXECINSTR)) 539 continue; 540 for (InputSection *sec : getInputSections(*osec, storage)) { 541 llvm::sort(sec->relaxAux->anchors, [](auto &a, auto &b) { 542 return std::make_pair(a.offset, a.end) < 543 std::make_pair(b.offset, b.end); 544 }); 545 } 546 } 547 } 548 549 // Relax R_RISCV_CALL/R_RISCV_CALL_PLT auipc+jalr to c.j, c.jal, or jal. 550 static void relaxCall(const InputSection &sec, size_t i, uint64_t loc, 551 Relocation &r, uint32_t &remove) { 552 const bool rvc = config->eflags & EF_RISCV_RVC; 553 const Symbol &sym = *r.sym; 554 const uint64_t insnPair = read64le(sec.rawData.data() + r.offset); 555 const uint32_t rd = extractBits(insnPair, 32 + 11, 32 + 7); 556 const uint64_t dest = 557 (r.expr == R_PLT_PC ? sym.getPltVA() : sym.getVA()) + r.addend; 558 const int64_t displace = dest - loc; 559 560 if (rvc && isInt<12>(displace) && rd == 0) { 561 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP; 562 sec.relaxAux->writes.push_back(0xa001); // c.j 563 remove = 6; 564 } else if (rvc && isInt<12>(displace) && rd == X_RA && 565 !config->is64) { // RV32C only 566 sec.relaxAux->relocTypes[i] = R_RISCV_RVC_JUMP; 567 sec.relaxAux->writes.push_back(0x2001); // c.jal 568 remove = 6; 569 } else if (isInt<21>(displace)) { 570 sec.relaxAux->relocTypes[i] = R_RISCV_JAL; 571 sec.relaxAux->writes.push_back(0x6f | rd << 7); // jal 572 remove = 4; 573 } 574 } 575 576 // Relax local-exec TLS when hi20 is zero. 577 static void relaxTlsLe(const InputSection &sec, size_t i, uint64_t loc, 578 Relocation &r, uint32_t &remove) { 579 uint64_t val = r.sym->getVA(r.addend); 580 if (hi20(val) != 0) 581 return; 582 uint32_t insn = read32le(sec.rawData.data() + r.offset); 583 switch (r.type) { 584 case R_RISCV_TPREL_HI20: 585 case R_RISCV_TPREL_ADD: 586 // Remove lui rd, %tprel_hi(x) and add rd, rd, tp, %tprel_add(x). 587 sec.relaxAux->relocTypes[i] = R_RISCV_RELAX; 588 remove = 4; 589 break; 590 case R_RISCV_TPREL_LO12_I: 591 // addi rd, rd, %tprel_lo(x) => addi rd, tp, st_value(x) 592 sec.relaxAux->relocTypes[i] = R_RISCV_32; 593 insn = (insn & ~(31 << 15)) | (X_TP << 15); 594 sec.relaxAux->writes.push_back(setLO12_I(insn, val)); 595 break; 596 case R_RISCV_TPREL_LO12_S: 597 // sw rs, %tprel_lo(x)(rd) => sw rs, st_value(x)(rd) 598 sec.relaxAux->relocTypes[i] = R_RISCV_32; 599 insn = (insn & ~(31 << 15)) | (X_TP << 15); 600 sec.relaxAux->writes.push_back(setLO12_S(insn, val)); 601 break; 602 } 603 } 604 605 static bool relax(InputSection &sec) { 606 const uint64_t secAddr = sec.getVA(); 607 auto &aux = *sec.relaxAux; 608 bool changed = false; 609 610 // Get st_value delta for symbols relative to this section from the previous 611 // iteration. 612 DenseMap<const Defined *, uint64_t> valueDelta; 613 ArrayRef<SymbolAnchor> sa = makeArrayRef(aux.anchors); 614 uint32_t delta = 0; 615 for (auto it : llvm::enumerate(sec.relocations)) { 616 for (; sa.size() && sa[0].offset <= it.value().offset; sa = sa.slice(1)) 617 if (!sa[0].end) 618 valueDelta[sa[0].d] = delta; 619 delta = aux.relocDeltas[it.index()]; 620 } 621 for (const SymbolAnchor &sa : sa) 622 if (!sa.end) 623 valueDelta[sa.d] = delta; 624 sa = makeArrayRef(aux.anchors); 625 delta = 0; 626 627 std::fill_n(aux.relocTypes.get(), sec.relocations.size(), R_RISCV_NONE); 628 aux.writes.clear(); 629 for (auto it : llvm::enumerate(sec.relocations)) { 630 Relocation &r = it.value(); 631 const size_t i = it.index(); 632 const uint64_t loc = secAddr + r.offset - delta; 633 uint32_t &cur = aux.relocDeltas[i], remove = 0; 634 switch (r.type) { 635 case R_RISCV_ALIGN: { 636 const uint64_t nextLoc = loc + r.addend; 637 const uint64_t align = PowerOf2Ceil(r.addend + 2); 638 // All bytes beyond the alignment boundary should be removed. 639 remove = nextLoc - ((loc + align - 1) & -align); 640 assert(static_cast<int32_t>(remove) >= 0 && 641 "R_RISCV_ALIGN needs expanding the content"); 642 break; 643 } 644 case R_RISCV_CALL: 645 case R_RISCV_CALL_PLT: 646 if (i + 1 != sec.relocations.size() && 647 sec.relocations[i + 1].type == R_RISCV_RELAX) 648 relaxCall(sec, i, loc, r, remove); 649 break; 650 case R_RISCV_TPREL_HI20: 651 case R_RISCV_TPREL_ADD: 652 case R_RISCV_TPREL_LO12_I: 653 case R_RISCV_TPREL_LO12_S: 654 if (i + 1 != sec.relocations.size() && 655 sec.relocations[i + 1].type == R_RISCV_RELAX) 656 relaxTlsLe(sec, i, loc, r, remove); 657 break; 658 } 659 660 // For all anchors whose offsets are <= r.offset, they are preceded by 661 // the previous relocation whose `relocDeltas` value equals `delta`. 662 // Decrease their st_value and update their st_size. 663 for (; sa.size() && sa[0].offset <= r.offset; sa = sa.slice(1)) { 664 if (sa[0].end) 665 sa[0].d->size = sa[0].offset - delta - sa[0].d->value; 666 else 667 sa[0].d->value -= delta - valueDelta.find(sa[0].d)->second; 668 } 669 delta += remove; 670 if (delta != cur) { 671 cur = delta; 672 changed = true; 673 } 674 } 675 676 for (const SymbolAnchor &a : sa) { 677 if (a.end) 678 a.d->size = a.offset - delta - a.d->value; 679 else 680 a.d->value -= delta - valueDelta.find(a.d)->second; 681 } 682 // Inform assignAddresses that the size has changed. 683 if (!isUInt<16>(delta)) 684 fatal("section size decrease is too large"); 685 sec.bytesDropped = delta; 686 return changed; 687 } 688 689 // When relaxing just R_RISCV_ALIGN, relocDeltas is usually changed only once in 690 // the absence of a linker script. For call and load/store R_RISCV_RELAX, code 691 // shrinkage may reduce displacement and make more relocations eligible for 692 // relaxation. Code shrinkage may increase displacement to a call/load/store 693 // target at a higher fixed address, invalidating an earlier relaxation. Any 694 // change in section sizes can have cascading effect and require another 695 // relaxation pass. 696 bool RISCV::relaxOnce(int pass) const { 697 llvm::TimeTraceScope timeScope("RISC-V relaxOnce"); 698 if (config->relocatable) 699 return false; 700 701 if (pass == 0) 702 initSymbolAnchors(); 703 704 SmallVector<InputSection *, 0> storage; 705 bool changed = false; 706 for (OutputSection *osec : outputSections) { 707 if (!(osec->flags & SHF_EXECINSTR)) 708 continue; 709 for (InputSection *sec : getInputSections(*osec, storage)) 710 changed |= relax(*sec); 711 } 712 return changed; 713 } 714 715 void elf::riscvFinalizeRelax(int passes) { 716 llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation"); 717 log("relaxation passes: " + Twine(passes)); 718 SmallVector<InputSection *, 0> storage; 719 for (OutputSection *osec : outputSections) { 720 if (!(osec->flags & SHF_EXECINSTR)) 721 continue; 722 for (InputSection *sec : getInputSections(*osec, storage)) { 723 RISCVRelaxAux &aux = *sec->relaxAux; 724 if (!aux.relocDeltas) 725 continue; 726 727 auto &rels = sec->relocations; 728 ArrayRef<uint8_t> old = sec->rawData; 729 size_t newSize = 730 old.size() - aux.relocDeltas[sec->relocations.size() - 1]; 731 size_t writesIdx = 0; 732 uint8_t *p = context().bAlloc.Allocate<uint8_t>(newSize); 733 uint64_t offset = 0; 734 int64_t delta = 0; 735 sec->rawData = makeArrayRef(p, newSize); 736 sec->bytesDropped = 0; 737 738 // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite 739 // instructions for relaxed relocations. 740 for (size_t i = 0, e = rels.size(); i != e; ++i) { 741 uint32_t remove = aux.relocDeltas[i] - delta; 742 delta = aux.relocDeltas[i]; 743 if (remove == 0 && aux.relocTypes[i] == R_RISCV_NONE) 744 continue; 745 746 // Copy from last location to the current relocated location. 747 const Relocation &r = rels[i]; 748 uint64_t size = r.offset - offset; 749 memcpy(p, old.data() + offset, size); 750 p += size; 751 752 // For R_RISCV_ALIGN, we will place `offset` in a location (among NOPs) 753 // to satisfy the alignment requirement. If `remove` is a multiple of 4, 754 // it is as if we have skipped some NOPs. Otherwise we are in the middle 755 // of a 4-byte NOP, and we need to rewrite the NOP sequence. 756 int64_t skip = 0; 757 if (r.type == R_RISCV_ALIGN) { 758 if (remove % 4 != 0) { 759 skip = r.addend - remove; 760 int64_t j = 0; 761 for (; j + 4 <= skip; j += 4) 762 write32le(p + j, 0x00000013); // nop 763 if (j != skip) { 764 assert(j + 2 == skip); 765 write16le(p + j, 0x0001); // c.nop 766 } 767 } 768 } else if (RelType newType = aux.relocTypes[i]) { 769 switch (newType) { 770 case R_RISCV_RELAX: 771 // Used by relaxTlsLe to indicate the relocation is ignored. 772 break; 773 case R_RISCV_RVC_JUMP: 774 skip = 2; 775 write16le(p, aux.writes[writesIdx++]); 776 break; 777 case R_RISCV_JAL: 778 skip = 4; 779 write32le(p, aux.writes[writesIdx++]); 780 break; 781 case R_RISCV_32: 782 // Used by relaxTlsLe to write a uint32_t then suppress the handling 783 // in relocateAlloc. 784 skip = 4; 785 write32le(p, aux.writes[writesIdx++]); 786 aux.relocTypes[i] = R_RISCV_NONE; 787 break; 788 default: 789 llvm_unreachable("unsupported type"); 790 } 791 } 792 793 p += skip; 794 offset = r.offset + skip + remove; 795 } 796 memcpy(p, old.data() + offset, old.size() - offset); 797 798 // Subtract the previous relocDeltas value from the relocation offset. 799 // For a pair of R_RISCV_CALL/R_RISCV_RELAX with the same offset, decrease 800 // their r_offset by the same delta. 801 delta = 0; 802 for (size_t i = 0, e = rels.size(); i != e;) { 803 uint64_t cur = rels[i].offset; 804 do { 805 rels[i].offset -= delta; 806 if (aux.relocTypes[i] != R_RISCV_NONE) 807 rels[i].type = aux.relocTypes[i]; 808 } while (++i != e && rels[i].offset == cur); 809 delta = aux.relocDeltas[i - 1]; 810 } 811 } 812 } 813 } 814 815 TargetInfo *elf::getRISCVTargetInfo() { 816 static RISCV target; 817 return ⌖ 818 } 819